Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

feat: voice profiles built from per-recording samples with several voice variants per person, compared only within the embedding space of the model that made them; dimension-agnostic, normalized, a speech minimum, outlier samples kept out, one person per label when auto-labelling, corrections that reach the embedding through a label map, and thresholds calibrated from the data. Also passes the web delete route its own storage service.

Murtaza Nasirf26dfb8d805d
diff --git a/src/api/api_v1.py b/src/api/api_v1.py
index cdd7957d..0594ee5d 100644
--- a/src/api/api_v1.py
+++ b/src/api/api_v1.py
@@ -2138,9 +2138,10 @@ def get_recording_speakers(recording_id):
})
# Get voice-based suggestions. speaker_embeddings maps each SPEAKER_XX
- # label to one 256-dim embedding; find_matching_speakers takes a single
- # embedding (plus user_id) and returns a sorted match list with
- # similarity already expressed as a percentage.
+ # label to one embedding (any dimension); find_matching_speakers takes a
+ # single embedding (plus user_id), compares it within the recording's
+ # embedding space and returns a sorted match list with similarity
+ # already expressed as a percentage.
suggestions = {}
if recording.speaker_embeddings:
try:
@@ -2149,10 +2150,13 @@ def get_recording_speakers(recording_id):
if isinstance(recording.speaker_embeddings, str)
else recording.speaker_embeddings
)
+ from src.services.voice_profiles import suggestion_threshold
+ space_id = recording.speaker_embeddings_space_id
for label, embedding in embeddings_data.items():
- if not embedding or len(embedding) != 256:
+ if not embedding:
continue
- matches = find_matching_speakers(embedding, current_user.id)
+ matches = find_matching_speakers(embedding, current_user.id,
+ threshold=suggestion_threshold(space_id), space_id=space_id)
suggestions[label] = [{
'speaker_id': m['speaker_id'],
'name': m['name'],
@@ -2186,8 +2190,6 @@ def assign_speakers(recording_id):
}
"""
from src.services.speaker import update_speaker_usage
- from src.services.speaker_embedding_matcher import update_speaker_embedding
- from src.services.speaker_snippets import create_speaker_snippets
from src.services.job_queue import job_queue
try:
@@ -2221,40 +2223,28 @@ def assign_speakers(recording_id):
else:
return jsonify({'error': f'Invalid value type for speaker "{label}"'}), 400
- # --- Apply names to transcription (same logic as update_speakers in recordings.py) ---
+ # --- Apply names exactly as the web app's update_speakers does ---
+ from src.services.speaker import apply_speaker_map, participants_from_segments, update_voice_profiles
+ from src.services.voice_profiles import speech_seconds_by_label
+
transcription_text = recording.transcription or ''
- is_json = False
try:
transcription_data = json.loads(transcription_text)
is_json = isinstance(transcription_data, list)
except (json.JSONDecodeError, TypeError):
is_json = False
- speaker_names_used = []
-
+ embeddings_updated = 0
+ snippets_created = 0
if is_json:
- for segment in transcription_data:
- original_speaker_label = segment.get('speaker')
- if original_speaker_label in speaker_map:
- new_name_info = speaker_map[original_speaker_label]
- new_name = new_name_info.get('name', '').strip()
- if new_name_info.get('isMe') and not new_name:
- new_name = current_user.name or 'Me'
- if new_name:
- segment['speaker'] = new_name
- if new_name not in speaker_names_used:
- speaker_names_used.append(new_name)
-
+ seconds_by_key = speech_seconds_by_label(transcription_data)
+ speaker_names_used, label_to_name = apply_speaker_map(transcription_data, speaker_map, current_user)
recording.transcription = json.dumps(transcription_data)
-
- # Update participants - exclude unresolved SPEAKER_XX labels
- final_speakers = set()
- for seg in transcription_data:
- speaker = seg.get('speaker')
- if speaker and str(speaker).strip():
- if not re.match(r'^SPEAKER_\d+$', str(speaker), re.IGNORECASE):
- final_speakers.add(speaker)
- recording.participants = ', '.join(sorted(list(final_speakers)))
+ recording.participants = participants_from_segments(transcription_data)
+ if speaker_names_used:
+ update_speaker_usage(speaker_names_used)
+ embeddings_updated, snippets_created = update_voice_profiles(
+ recording, label_to_name, current_user, seconds_by_key)
else:
# Plain text transcript
new_participants = []
@@ -2271,45 +2261,10 @@ def assign_speakers(recording_id):
)
if new_name not in new_participants:
new_participants.append(new_name)
-
recording.transcription = transcription_text
recording.participants = ', '.join(new_participants)
- speaker_names_used = new_participants
-
- # Update speaker usage statistics
- if speaker_names_used:
- update_speaker_usage(speaker_names_used)
-
- # Update speaker voice embeddings if available
- embeddings_updated = 0
- snippets_created = 0
- if recording.speaker_embeddings and speaker_map:
- try:
- embeddings_data = json.loads(recording.speaker_embeddings) if isinstance(recording.speaker_embeddings, str) else recording.speaker_embeddings
-
- speaker_label_to_name = {}
- for speaker_label, speaker_info in speaker_map.items():
- name = speaker_info.get('name', '').strip()
- if speaker_info.get('isMe') and not name:
- name = current_user.name or 'Me'
- if name and not re.match(r'^SPEAKER_\d+$', name, re.IGNORECASE):
- speaker_label_to_name[speaker_label] = name
-
- for speaker_label, embedding in embeddings_data.items():
- if speaker_label in speaker_label_to_name and embedding and len(embedding) == 256:
- speaker_name = speaker_label_to_name[speaker_label]
- speaker_obj = Speaker.query.filter_by(
- user_id=current_user.id,
- name=speaker_name
- ).first()
- if speaker_obj:
- update_speaker_embedding(speaker_obj, embedding, recording.id)
- embeddings_updated += 1
-
- if speaker_label_to_name:
- snippets_created = create_speaker_snippets(recording.id, speaker_map)
- except Exception as e:
- current_app.logger.error(f"Error updating speaker embeddings: {e}", exc_info=True)
+ if new_participants:
+ update_speaker_usage(new_participants)
db.session.commit()
diff --git a/src/api/recordings.py b/src/api/recordings.py
index c21b70da..59901b75 100644
--- a/src/api/recordings.py
+++ b/src/api/recordings.py
@@ -945,7 +945,10 @@ def update_speakers(recording_id):
speaker_names_used = []
label_to_name = {}
+ seconds_by_key = None
if is_json:
+ from src.services.voice_profiles import speech_seconds_by_label
+ seconds_by_key = speech_seconds_by_label(transcription_data)
speaker_names_used, label_to_name = apply_speaker_map(transcription_data, speaker_map, current_user)
recording.transcription = json.dumps(transcription_data)
recording.participants = participants_from_segments(transcription_data)
@@ -974,7 +977,7 @@ def update_speakers(recording_id):
update_speaker_usage(speaker_names_used)
# Update speaker voice embeddings and snippets if available
- update_voice_profiles(recording, label_to_name, current_user)
+ update_voice_profiles(recording, label_to_name, current_user, seconds_by_key)
db.session.commit()
@@ -1034,6 +1037,8 @@ def update_transcript(recording_id):
# Apply the names exactly as update_speakers does, including the voice
# profiles and snippets: this route saves the names whenever the user
# also staged a line edit in the speaker modal.
+ from src.services.voice_profiles import speech_seconds_by_label
+ seconds_by_key = speech_seconds_by_label(transcript_data)
speaker_names_used, label_to_name = apply_speaker_map(transcript_data, speaker_map, current_user)
# Save the updated transcript
@@ -1044,7 +1049,7 @@ def update_transcript(recording_id):
if speaker_names_used:
update_speaker_usage(speaker_names_used)
- update_voice_profiles(recording, label_to_name, current_user)
+ update_voice_profiles(recording, label_to_name, current_user, seconds_by_key)
db.session.commit()
@@ -3427,7 +3432,7 @@ def delete_recording(recording_id):
current_app.logger.info(f"Deleting {chunk_count} transcript chunks with embeddings for recording {recording_id}")
from src.services.recording_deletion import delete_recording_completely, cleanup_orphaned_speakers_quietly
- delete_recording_completely(recording)
+ delete_recording_completely(recording, storage=get_storage_service())
cleanup_orphaned_speakers_quietly()
return jsonify({'success': True})
diff --git a/src/api/speakers.py b/src/api/speakers.py
index b386b573..b8593e6d 100644
--- a/src/api/speakers.py
+++ b/src/api/speakers.py
@@ -253,6 +253,9 @@ def delete_speaker(speaker_id):
def delete_all_speakers():
"""Delete all speakers for the current user."""
try:
+ # Bulk delete skips ORM cascades, so the voice samples go first.
+ from src.models import SpeakerVoiceSample
+ SpeakerVoiceSample.query.filter_by(user_id=current_user.id).delete()
deleted_count = Speaker.query.filter_by(user_id=current_user.id).delete()
db.session.commit()
return jsonify({'success': True, 'deleted_count': deleted_count})
@@ -294,29 +297,13 @@ def get_speaker_suggestions(recording_id):
if not recording.speaker_embeddings:
return jsonify({'suggestions': {}, 'message': 'No speaker embeddings available'}), 200
- try:
- embeddings_data = json.loads(recording.speaker_embeddings) if isinstance(recording.speaker_embeddings, str) else recording.speaker_embeddings
- except (json.JSONDecodeError, TypeError):
- return jsonify({'error': 'Invalid speaker embeddings data'}), 500
-
- # Similarity floor for showing a voice-match suggestion. Default
- # 60% (was 70%). Rationale: when auto-labelling is on, confident
- # matches are already applied automatically, so the suggestion pill
- # is most useful for the BORDERLINE matches that auto-label didn't
- # take. A 0.70 floor hid exactly those (e.g. a 0.69 match), making
- # the pill rarely appear. 0.60 surfaces them for one-click manual
- # acceptance while still filtering out weak/noise matches. Callers
- # can still override via ?threshold=.
- threshold = float(request.args.get('threshold', 0.60))
-
- # Find matches for each speaker
- suggestions = {}
- for speaker_label, embedding in embeddings_data.items():
- if embedding and len(embedding) == 256: # Validate embedding dimension
- matches = find_matching_speakers(embedding, current_user.id, threshold)
- suggestions[speaker_label] = matches
- else:
- suggestions[speaker_label] = []
+ # Matching compares each voice with every voice variant of each
+ # person, within the recording's embedding space. The threshold is
+ # calibrated from the space's own samples once there are enough; a
+ # ?threshold= query parameter still overrides it.
+ from src.services.voice_profiles import suggestions_for_recording
+ threshold = request.args.get('threshold', type=float)
+ suggestions = suggestions_for_recording(recording, current_user.id, threshold=threshold)
return jsonify({
'success': True,
@@ -407,6 +394,8 @@ def clear_speaker_embeddings(speaker_id):
# phantom `voice_embeddings` attribute that never persisted, so the
# real average_embedding survived and voice matching kept working
# after a "clear". Null out the actual columns.
+ from src.models import SpeakerVoiceSample
+ SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).delete()
speaker.average_embedding = None
speaker.embeddings_history = None
speaker.embedding_count = 0
diff --git a/src/init_db.py b/src/init_db.py
index 1a6ef183..c1fe7681 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -440,6 +440,15 @@ def _run_migrations(app, engine):
if add_column_if_not_exists(engine, 'folder', 'default_initial_prompt', 'TEXT'):
app.logger.info("Added default_initial_prompt column to folder table")
+ # Voice matching: embedding space of a recording's embeddings, and the
+ # label -> name map that lets renames reach the voice samples. The
+ # voice_embedding_space and speaker_voice_sample tables come from
+ # create_all().
+ if add_column_if_not_exists(engine, 'recording', 'speaker_embeddings_space_id', 'INTEGER'):
+ app.logger.info("Added speaker_embeddings_space_id column to recording table")
+ if add_column_if_not_exists(engine, 'recording', 'speaker_label_map', 'JSON'):
+ app.logger.info("Added speaker_label_map column to recording table")
+
# Archive hides a recording from the main list without deleting
# anything. Per user, like inbox and star (#394).
if add_column_if_not_exists(engine, 'recording', 'is_archived', 'BOOLEAN DEFAULT 0'):
diff --git a/src/models/__init__.py b/src/models/__init__.py
index 9da4e119..1ddda8d8 100644
--- a/src/models/__init__.py
+++ b/src/models/__init__.py
@@ -15,6 +15,7 @@ from src.database import db
# Import all models
from .user import User, Speaker
+from .voice import VoiceEmbeddingSpace, SpeakerVoiceSample
from .api_token import APIToken
from .speaker_snippet import SpeakerSnippet
from .recording import Recording, TranscriptChunk
@@ -53,6 +54,8 @@ __all__ = [
# User models
'User',
'Speaker',
+ 'VoiceEmbeddingSpace',
+ 'SpeakerVoiceSample',
'APIToken',
'SpeakerSnippet',
# Recording models
diff --git a/src/models/recording.py b/src/models/recording.py
index 6bf1be6e..4a049b41 100644
--- a/src/models/recording.py
+++ b/src/models/recording.py
@@ -69,6 +69,11 @@ class Recording(db.Model):
# Speaker embeddings from diarization (JSON dict mapping speaker IDs to 256-dimensional vectors)
speaker_embeddings = db.Column(db.JSON, nullable=True)
+ # Voice embedding space the embeddings above came from (NULL: legacy space)
+ speaker_embeddings_space_id = db.Column(db.Integer, nullable=True)
+ # Diarization label -> the name it was assigned, so a later rename can
+ # still find the label's embedding after the transcript shows names.
+ speaker_label_map = db.Column(db.JSON, nullable=True)
# Per-recording prompt-template variables (e.g. {"agenda": "...", "attendees": "..."}).
# Substituted into {{name}} placeholders in the resolved summary prompt at summarisation time.
diff --git a/src/models/voice.py b/src/models/voice.py
new file mode 100644
index 00000000..b263576f
--- /dev/null
+++ b/src/models/voice.py
@@ -0,0 +1,74 @@
+"""Voice profile storage: embedding spaces and per-person voice samples.
+
+VoiceEmbeddingSpace (Stage 2): the transcription backend never says which
+model produced an embedding, so each model Speakr has seen is identified by
+the embedding it returns for the bundled canary clip (see
+services/voice_embedding_check.py). Embeddings are only compared within one
+space. Rows written before spaces existed carry no space and belong to the
+"legacy" space, the first one registered, so upgrading changes nothing.
+
+SpeakerVoiceSample (Stage 3): one normalized embedding per (recording,
+diarization label) assigned to a person. A person's voice variants are
+computed from these samples on the fly, so removing a wrongly assigned
+sample undoes its effect completely.
+"""
+
+from datetime import datetime
+
+from src.database import db
+
+
+class VoiceEmbeddingSpace(db.Model):
+ __tablename__ = 'voice_embedding_space'
+
+ id = db.Column(db.Integer, primary_key=True)
+ dimension = db.Column(db.Integer, nullable=True)
+ canary_embedding = db.Column(db.JSON, nullable=True) # the canary clip's embedding in this space
+ backend_fingerprint = db.Column(db.String(64), nullable=True)
+ created_at = db.Column(db.DateTime, default=datetime.utcnow)
+ last_seen_at = db.Column(db.DateTime, default=datetime.utcnow)
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'dimension': self.dimension,
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ 'last_seen_at': self.last_seen_at.isoformat() if self.last_seen_at else None,
+ }
+
+
+class SpeakerVoiceSample(db.Model):
+ __tablename__ = 'speaker_voice_sample'
+
+ id = db.Column(db.Integer, primary_key=True)
+ user_id = db.Column(db.Integer, db.ForeignKey('user.id', ondelete='CASCADE'), nullable=False, index=True)
+ speaker_id = db.Column(db.Integer, db.ForeignKey('speaker.id', ondelete='CASCADE'), nullable=False, index=True)
+ # NULL for the sample that carries a pre-existing averaged profile, and
+ # after the source recording is deleted (the voiceprint stays, as the
+ # average always did).
+ recording_id = db.Column(db.Integer, db.ForeignKey('recording.id', ondelete='SET NULL'), nullable=True, index=True)
+ label = db.Column(db.String(100), nullable=True) # diarization label within the recording, e.g. SPEAKER_01
+ embedding = db.Column(db.LargeBinary, nullable=False) # L2-normalized float32
+ dimension = db.Column(db.Integer, nullable=False)
+ space_id = db.Column(db.Integer, db.ForeignKey('voice_embedding_space.id', ondelete='SET NULL'), nullable=True, index=True)
+ speech_seconds = db.Column(db.Float, nullable=True)
+ source = db.Column(db.String(20), nullable=False, default='confirmed') # confirmed | auto | legacy
+ weight = db.Column(db.Float, nullable=False, default=1.0)
+ created_at = db.Column(db.DateTime, default=datetime.utcnow)
+ updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+ speaker = db.relationship('Speaker', backref=db.backref('voice_samples', lazy=True, cascade='all, delete-orphan'))
+
+ def to_dict(self):
+ return {
+ 'id': self.id,
+ 'speaker_id': self.speaker_id,
+ 'recording_id': self.recording_id,
+ 'label': self.label,
+ 'dimension': self.dimension,
+ 'space_id': self.space_id,
+ 'speech_seconds': self.speech_seconds,
+ 'source': self.source,
+ 'weight': self.weight,
+ 'created_at': self.created_at.isoformat() if self.created_at else None,
+ }
diff --git a/src/services/recording_deletion.py b/src/services/recording_deletion.py
index 0c645b8f..dfc1afcb 100644
--- a/src/services/recording_deletion.py
+++ b/src/services/recording_deletion.py
@@ -44,6 +44,11 @@ def delete_recording_completely(recording, storage=None, strict_media=False):
raise
current_app.logger.error(f"Error deleting media {recording.audio_path}: {e}")
+ # Voice samples outlive the recording, as the averaged profile always
+ # did; they only lose the reference (SQLite does not apply SET NULL).
+ from src.models import SpeakerVoiceSample
+ SpeakerVoiceSample.query.filter_by(recording_id=recording_id).update({'recording_id': None})
+
snippets = SpeakerSnippet.query.filter_by(recording_id=recording_id).delete()
jobs = ProcessingJob.query.filter_by(recording_id=recording_id).delete()
if snippets or jobs:
diff --git a/src/services/speaker.py b/src/services/speaker.py
index 06778069..377b261d 100644
--- a/src/services/speaker.py
+++ b/src/services/speaker.py
@@ -80,49 +80,42 @@ def participants_from_segments(segments):
return ', '.join(sorted(names))
-def update_voice_profiles(recording, label_to_name, user):
- """Fold this recording's per-label voice embeddings into the named profiles
- and refresh the recording's speaker snippets.
-
- Both save routes call this. update_transcript used to skip it, so saving
- names alongside any line edit left voice profiles and snippets untouched.
- Returns (embeddings_updated, snippets_created). Never raises.
+def update_voice_profiles(recording, label_to_name, user, seconds_by_key=None):
+ """Train voice profiles from the names given in one save, and refresh the
+ recording's speaker snippets.
+
+ label_to_name maps the speaker values the transcript showed before the
+ save to the names now assigned; seconds_by_key is the speech per value,
+ measured before the rename (None when the transcript has no timing).
+ Both save routes and API v1 call this. Returns
+ (samples_stored, snippets_created). Never raises.
"""
- from src.services.speaker_embedding_matcher import update_speaker_embedding
from src.services.speaker_snippets import create_speaker_snippets
- import json
-
- if not recording.speaker_embeddings or not label_to_name:
- return 0, 0
+ from src.services.voice_profiles import apply_names_to_profiles
- embeddings_updated = 0
+ stored = 0
snippets_created = 0
try:
- embeddings_data = (json.loads(recording.speaker_embeddings)
- if isinstance(recording.speaker_embeddings, str)
- else recording.speaker_embeddings)
- for label, embedding in (embeddings_data or {}).items():
- name = label_to_name.get(label)
- if not name or not embedding or len(embedding) != 256:
- continue
- speaker = find_user_speaker(user.id, name)
- if not speaker:
- continue
- similarity = update_speaker_embedding(speaker, embedding, recording.id)
- embeddings_updated += 1
- if similarity is not None:
- current_app.logger.info(f"Updated voice profile for '{name}' (similarity: {similarity*100:.1f}%)")
- else:
- current_app.logger.info(f"Created initial voice profile for '{name}'")
-
+ # Runs even when nothing was named: a save that only merged speakers
+ # away must still drop the merged labels' samples.
+ if recording.speaker_embeddings:
+ stats = apply_names_to_profiles(recording, label_to_name or {}, seconds_by_key, user)
+ stored = stats['stored']
+ if stats['rejected'] or stats['skipped_short']:
+ current_app.logger.info(
+ f"Recording {recording.id}: {stats['rejected']} voice samples kept out as unlike "
+ f"the named person, {stats['skipped_short']} skipped for too little speech")
+ db.session.flush()
+ if not label_to_name:
+ return stored, 0
snippets_created = create_speaker_snippets(
recording.id, {label: {'name': name} for label, name in label_to_name.items()})
if snippets_created > 0:
current_app.logger.info(f"Created {snippets_created} speaker snippets")
except Exception as e:
- current_app.logger.error(f"Error updating speaker embeddings: {e}", exc_info=True)
+ current_app.logger.error(f"Error updating voice profiles: {e}", exc_info=True)
- return embeddings_updated, snippets_created
+ return stored, snippets_created
def update_speaker_usage(speaker_names):
diff --git a/src/services/speaker_embedding_matcher.py b/src/services/speaker_embedding_matcher.py
index 3730fbe7..895c10fc 100644
--- a/src/services/speaker_embedding_matcher.py
+++ b/src/services/speaker_embedding_matcher.py
@@ -72,80 +72,20 @@ def calculate_similarity(embedding1, embedding2):
return float(cosine_similarity(e1, e2)[0][0])
-def find_matching_speakers(target_embedding, user_id, threshold=0.70):
+def find_matching_speakers(target_embedding, user_id, threshold=0.70, space_id=None):
"""
Find speakers matching a target voice embedding for a specific user.
- Args:
- target_embedding: The voice embedding to match against (256-dim array/list)
- user_id: User ID to search within
- threshold: Minimum similarity score (0-1, default 0.70 = 70%)
+ Kept for callers of the original API. Matching itself lives in
+ services/voice_profiles.py: each person is compared through their voice
+ variants (the closest counts), only within one embedding space.
Returns:
- list: Sorted list of matching speakers with scores
- [{'speaker_id': 5, 'name': 'John', 'similarity': 85.3, 'confidence': 0.92}, ...]
+ list: [{'speaker_id', 'name', 'similarity' (percent), 'confidence',
+ 'embedding_count', 'variant_count'}], best first
"""
- # Get all speakers with embeddings for this user
- speakers = Speaker.query.filter_by(user_id=user_id).filter(
- Speaker.average_embedding.isnot(None)
- ).all()
-
- if not speakers:
- return []
-
- target_dims = len(target_embedding) if target_embedding is not None else 0
-
- matches = []
- dimension_mismatches = 0
- mismatched_dims = set()
- for speaker in speakers:
- try:
- # Deserialize and compare
- speaker_emb = deserialize_embedding(speaker.average_embedding)
-
- # Checked explicitly rather than left to raise below. A stored
- # profile of a different length is not a corrupt profile: it is a
- # profile built by a different embedding model, and treating it as
- # corruption is what made a backend change look like voice
- # matching quietly ceasing to work, with nothing in the log to say
- # why (#380).
- if target_dims and len(speaker_emb) != target_dims:
- dimension_mismatches += 1
- mismatched_dims.add(len(speaker_emb))
- continue
-
- similarity = calculate_similarity(target_embedding, speaker_emb)
-
- if similarity >= threshold:
- matches.append({
- 'speaker_id': speaker.id,
- 'name': speaker.name,
- 'similarity': round(similarity * 100, 1), # Convert to percentage
- 'confidence': speaker.confidence_score or 0.5,
- 'embedding_count': speaker.embedding_count or 0
- })
- except Exception as e:
- # Genuinely unreadable stored bytes. Logged rather than swallowed,
- # so a systematic problem is visible instead of showing up only as
- # an empty match list.
- logger.warning(
- "Skipping speaker %s: its stored embedding could not be compared: %s",
- getattr(speaker, 'id', '?'), e)
- continue
-
- if dimension_mismatches:
- # One line per call rather than per speaker, so this stays readable
- # when a whole library is affected, which is the usual case.
- sizes = ', '.join(str(d) for d in sorted(mismatched_dims))
- logger.error(
- "%d of %d stored voice profiles have %s dimensions and cannot be compared "
- "with the %d-dimensional embeddings this backend now returns. Voice matching "
- "will find nothing until the previous transcription backend is restored or "
- "the profiles are rebuilt.",
- dimension_mismatches, len(speakers), sizes, target_dims)
-
- # Sort by similarity (highest first)
- return sorted(matches, key=lambda x: x['similarity'], reverse=True)
+ from src.services.voice_profiles import find_matches
+ return find_matches(target_embedding, user_id, space_id=space_id, threshold=threshold)
def update_speaker_embedding(speaker, new_embedding, recording_id):
@@ -308,70 +248,20 @@ AMBIGUITY_MARGIN = 0.05
def apply_auto_speaker_labels(recording, user):
"""
- Automatically label speakers in a recording based on voice profile matching.
-
- This function matches speaker embeddings from the recording against the user's
- saved speaker profiles and returns a mapping of generic labels to speaker names.
-
- Args:
- recording: Recording model instance with speaker_embeddings
- user: User model instance with auto_speaker_labelling settings
+ Match a new recording's speakers against the user's voice profiles.
- Returns:
- dict: Mapping of {SPEAKER_XX: speaker_name} for matched speakers,
- or empty dict if auto-labelling is disabled or no matches found
+ Returns {SPEAKER_XX: name}. One person per label and one label per
+ person; labels with too little speech, and labels whose two best
+ candidates are nearly tied, are left alone. The threshold is the user's
+ low / medium / high setting, calibrated to the embedding space once it
+ has enough samples (services/voice_profiles.py).
"""
- # Check if user has auto-labelling enabled
- if not user.auto_speaker_labelling:
- return {}
-
- # Check if recording has speaker embeddings
- if not recording.speaker_embeddings:
+ if not user.auto_speaker_labelling or not recording.speaker_embeddings:
return {}
-
- # Get the user's threshold setting
- threshold_setting = user.auto_speaker_labelling_threshold or 'medium'
- confidence_threshold = AUTO_LABEL_THRESHOLDS.get(threshold_setting, AUTO_LABEL_THRESHOLDS['medium'])
-
- speaker_map = {}
- embeddings = recording.speaker_embeddings
-
- for speaker_label, embedding_data in embeddings.items():
- # embedding_data should be a list of floats (256 dimensions)
- if not embedding_data or not isinstance(embedding_data, list):
- continue
-
- # Find matching speakers using the USER'S chosen threshold as the
- # filter. Previously this filtered with the hardcoded 0.70
- # BASE_SIMILARITY_THRESHOLD and only then applied the user's
- # confidence_threshold — which meant 'low' (0.3) and 'medium' (0.6)
- # were both effectively 0.70 (nothing under 0.70 ever reached the
- # second check), so those settings did nothing. Filtering at
- # confidence_threshold directly makes the setting actually take
- # effect: low=0.30, medium=0.60, high=0.80.
- matches = find_matching_speakers(
- target_embedding=embedding_data,
- user_id=user.id,
- threshold=confidence_threshold
- )
-
- if not matches:
- continue
-
- best_match = matches[0]
- best_similarity = best_match['similarity'] / 100.0 # Convert from percentage
-
- # Check for ambiguity: if top 2 matches are within 5% similarity, skip
- if len(matches) >= 2:
- second_similarity = matches[1]['similarity'] / 100.0
- if (best_similarity - second_similarity) <= AMBIGUITY_MARGIN:
- # Ambiguous - top 2 matches too close
- continue
-
- # We have a clear winner - add to speaker map
- speaker_map[speaker_label] = best_match['name']
-
- return speaker_map
+ from src.services.voice_profiles import auto_label_map, auto_label_threshold
+ threshold = auto_label_threshold(user.auto_speaker_labelling_threshold or 'medium',
+ recording.speaker_embeddings_space_id)
+ return auto_label_map(recording, user, threshold)
def apply_speaker_names_to_transcription(recording, speaker_map):
@@ -422,10 +312,9 @@ def apply_speaker_names_to_transcription(recording, speaker_map):
logger.info(f"Auto-label: Applied names to {len(renamed_speakers)} speakers: {renamed_speakers}")
- # Update participants field
- all_speakers = set(s.get('speaker') for s in segments if 'speaker' in s)
- if all_speakers:
- recording.participants = ', '.join(sorted(all_speakers))
+ # Participants: real names only, never leftover SPEAKER_XX labels
+ from src.services.speaker import participants_from_segments
+ recording.participants = participants_from_segments(segments)
# Save updated transcription
recording.transcription = json.dumps(segments)
@@ -436,56 +325,23 @@ def apply_speaker_names_to_transcription(recording, speaker_map):
def update_speaker_profiles_from_recording(recording, speaker_map, user):
"""
- Update speaker voice profiles with new embeddings from a recording.
-
- For each successfully matched speaker, this function updates their
- average embedding and increments their usage count.
+ Train voice profiles from auto-applied labels.
- Args:
- recording: Recording model instance with speaker_embeddings
- speaker_map: Dict mapping {SPEAKER_XX: speaker_name} that was applied
- user: User model instance
+ Samples from auto-labelling carry half the weight of names a person
+ confirmed, and confirming or correcting the name later in the speaker
+ dialog replaces them.
Returns:
int: Number of speaker profiles updated
"""
if not speaker_map or not recording.speaker_embeddings:
return 0
-
- updated_count = 0
- embeddings = recording.speaker_embeddings
-
- for speaker_label, speaker_name in speaker_map.items():
- if speaker_label not in embeddings:
- continue
-
- embedding_data = embeddings[speaker_label]
- if not embedding_data or not isinstance(embedding_data, list):
- continue
-
- # Find the speaker profile
- speaker = Speaker.query.filter_by(
- user_id=user.id,
- name=speaker_name
- ).first()
-
- if not speaker:
- continue
-
- try:
- # Update the speaker's embedding with the new sample
- update_speaker_embedding(speaker, embedding_data, recording.id)
-
- # Update usage tracking
+ from src.services.voice_profiles import apply_names_to_profiles
+ stats = apply_names_to_profiles(recording, dict(speaker_map), None, user, source='auto')
+ for name in set(speaker_map.values()):
+ speaker = Speaker.query.filter_by(user_id=user.id, name=name).first()
+ if speaker:
speaker.use_count = (speaker.use_count or 0) + 1
speaker.last_used = datetime.utcnow()
-
- updated_count += 1
- except Exception:
- # Skip if embedding update fails
- continue
-
- if updated_count > 0:
- db.session.commit()
-
- return updated_count
+ db.session.commit()
+ return stats['stored']
diff --git a/src/services/speaker_merge.py b/src/services/speaker_merge.py
index c5300bc7..1bf4c9b2 100644
--- a/src/services/speaker_merge.py
+++ b/src/services/speaker_merge.py
@@ -61,8 +61,10 @@ def merge_speakers(target_id, source_ids, user_id):
if target_id in source_ids:
raise ValueError("Cannot merge a speaker with itself")
- # Combine embeddings
- _combine_embeddings(target, sources)
+ # Move the voice samples. The target's variants are rebuilt from the
+ # combined samples below, so the merge keeps every voice condition the
+ # people had instead of averaging them into one vector.
+ _merge_voice_samples(target, sources)
# Transfer snippets
for source in sources:
@@ -92,8 +94,10 @@ def merge_speakers(target_id, source_ids, user_id):
# If sorting fails, just concatenate and truncate
target.embeddings_history = (target_history + source_history)[-10:]
- # Recalculate confidence score
- target.confidence_score = calculate_confidence(target)
+ # Summary columns (average, count, history, confidence) from the samples
+ from src.services.voice_profiles import refresh_speaker_summary
+ db.session.flush()
+ refresh_speaker_summary(target)
# Delete source speakers
for source in sources:
@@ -105,6 +109,21 @@ def merge_speakers(target_id, source_ids, user_id):
return target
+def _merge_voice_samples(target, sources):
+ """Give the target every voice sample of the sources.
+
+ Profiles from before samples existed are written out as samples first,
+ so nothing they learned is lost.
+ """
+ from src.models import SpeakerVoiceSample
+ from src.services.voice_profiles import _materialize_legacy
+ for speaker in [target] + list(sources):
+ _materialize_legacy(speaker)
+ for source in sources:
+ SpeakerVoiceSample.query.filter_by(speaker_id=source.id).update({'speaker_id': target.id})
+ db.session.flush()
+
+
def _combine_embeddings(target, sources):
"""
Combine embeddings from multiple speakers using weighted average.
diff --git a/src/services/voice_embedding_check.py b/src/services/voice_embedding_check.py
index d002637e..adf8e959 100644
--- a/src/services/voice_embedding_check.py
+++ b/src/services/voice_embedding_check.py
@@ -192,6 +192,7 @@ def get_status(app=None):
'similarity': reference.get('last_similarity'),
'detail': reference.get('detail'),
'supported': embeddings_supported(),
+ 'spaces': spaces_status(),
}
@@ -207,6 +208,7 @@ def check_voice_embeddings(app, force=False):
# applies: this one has none. Leaving the notice up would tell the
# admin to fix something that cannot be fixed from here.
_clear_notification(app)
+ _spaces(lambda vp: vp.clear_current_space())
return {'status': STATUS_UNKNOWN, 'supported': False,
'detail': 'the active connector does not return speaker embeddings'}
@@ -219,6 +221,12 @@ def check_voice_embeddings(app, force=False):
logger.info('Canary clip changed since the reference was taken; re-baselining')
reference = None
+ # First start with embedding spaces: the model the existing profiles were
+ # built with is the one the reference was taken from. It becomes the
+ # legacy space, which every profile stored without a space belongs to.
+ if reference and reference.get('embedding'):
+ _spaces(lambda vp: _initialize_spaces(vp, reference))
+
if reference and not force and reference.get('backend_fingerprint') == fingerprint:
# Nothing about the configuration moved. The probe costs a real
# transcription, so it is not run on every boot for no reason.
@@ -237,6 +245,10 @@ def check_voice_embeddings(app, force=False):
now = datetime.utcnow().isoformat()
+ # Each model gets its own voice space; recognising one seen before brings
+ # its profiles back, a new one starts empty.
+ _spaces(lambda vp: vp.register_space(vector, fingerprint))
+
if reference is None:
save_reference({
'clip_version': CANARY_CLIP_VERSION,
@@ -308,14 +320,14 @@ def _log_change_banner(app, detail):
'=' * width,
detail or 'the transcription backend returned an unexpected embedding',
'',
- 'Existing voice profiles were built by a different model and will not',
- 'match new recordings. Voice matching will silently find nothing until',
- 'one of the following is true:',
+ 'Voice profiles are kept separately for each embedding model. Profiles',
+ 'built with the previous model will not match new recordings; profiles',
+ 'for this model are learned as speakers are named. Either:',
'',
- ' * the previous transcription backend is restored, which clears this',
- ' by itself on the next check, or',
- ' * the affected voice profiles are rebuilt and the reference is reset',
- ' from the admin area.',
+ ' * restore the previous transcription backend, which brings its',
+ ' profiles back and clears this on the next check, or',
+ ' * accept the change: reset the reference from the admin area once',
+ ' the profiles have been rebuilt for the new model.',
'',
'Set DISABLE_VOICE_EMBEDDING_CHECK=true to stop checking.',
'=' * width,
@@ -356,6 +368,46 @@ def _clear_notification(app):
logger.warning('Could not clear the voice embedding notification: %s', e)
+def _spaces(fn):
+ """Run a voice-space update without ever failing the check."""
+ try:
+ from src.services import voice_profiles
+ return fn(voice_profiles)
+ except Exception as e:
+ logger.warning(f"Voice embedding space bookkeeping failed: {e}")
+ return None
+
+
+def _initialize_spaces(vp, reference):
+ from src.models import VoiceEmbeddingSpace
+ if VoiceEmbeddingSpace.query.count() == 0:
+ vp.register_space(reference['embedding'], reference.get('backend_fingerprint'))
+
+
+def spaces_status():
+ """Voice spaces for the admin card: which is current, samples, threshold."""
+ try:
+ from src.models import SpeakerVoiceSample, VoiceEmbeddingSpace
+ from src.services import voice_profiles as vp
+ current, legacy = vp.current_space_id(), vp.legacy_space_id()
+ counts = {}
+ for (space_id,) in SpeakerVoiceSample.query.with_entities(SpeakerVoiceSample.space_id).all():
+ key = vp.effective_space(space_id)
+ counts[key] = counts.get(key, 0) + 1
+ out = []
+ for space in VoiceEmbeddingSpace.query.order_by(VoiceEmbeddingSpace.id).all():
+ item = space.to_dict()
+ item['current'] = space.id == current
+ item['legacy'] = space.id == legacy
+ item['sample_count'] = counts.get(space.id, 0)
+ item['calibrated_threshold'] = vp.calibrated_threshold(space.id)
+ out.append(item)
+ return out
+ except Exception as e:
+ logger.debug(f"Could not read voice spaces: {e}")
+ return []
+
+
def rebaseline(app):
"""Accept the current backend as correct and discard the old reference.
diff --git a/src/services/voice_profiles.py b/src/services/voice_profiles.py
new file mode 100644
index 00000000..a01a8d35
--- /dev/null
+++ b/src/services/voice_profiles.py
@@ -0,0 +1,703 @@
+"""Voice profiles: samples, variants, matching and embedding spaces.
+
+A person's voice profile is the set of samples stored for them in
+SpeakerVoiceSample: one L2-normalized embedding per (recording, diarization
+label) they were named in. Nothing is averaged in place. Voice variants (for
+example the same person on a phone and in a meeting room) are clustered from
+the samples whenever they are needed, and a new voice is compared with every
+variant, the closest one counting. Because the variants are always rebuilt
+from the samples, removing a wrongly assigned sample undoes it completely.
+
+Embeddings are only ever compared within one embedding space (one model; see
+models/voice.py and voice_embedding_check.py). A NULL space means the legacy
+space: everything stored before spaces existed.
+
+Profiles built before samples existed live in Speaker.average_embedding. They
+are read as one "legacy" sample, weighted by how many recordings built them,
+and written out as a real sample the first time the person gets a new one.
+Speaker.average_embedding, embedding_count and embeddings_history are kept up
+to date from the samples for the code and API fields that still read them.
+"""
+
+import logging
+import os
+import time
+from datetime import datetime
+
+import numpy as np
+
+from src.database import db
+
+logger = logging.getLogger(__name__)
+
+# A diarization label with less speech than this gives a noisy embedding; it
+# neither trains a profile nor gets auto-labelled.
+MIN_SPEECH_SECONDS = float(os.environ.get('VOICE_PROFILE_MIN_SPEECH_SECONDS', '15'))
+# Samples kept per person and space; the oldest go first.
+MAX_SAMPLES = 20
+# Samples at least this similar belong to the same voice variant.
+VARIANT_MERGE_SIMILARITY = 0.72
+MAX_VARIANTS = 4
+# A new sample less similar than this to every sample a well-established
+# profile already has is almost certainly a different person: kept out and
+# logged as a likely wrong name.
+OUTLIER_SIMILARITY = 0.25
+OUTLIER_MIN_SAMPLES = 3
+# A variant backed by a single sample of a person who has others counts a
+# little less than an established one.
+SINGLETON_PENALTY = 0.03
+AUTO_SAMPLE_WEIGHT = 0.5
+LEGACY_MAX_WEIGHT = 5.0
+
+DEFAULT_SUGGESTION_THRESHOLD = 0.60
+DEFAULT_AUTO_LABEL_THRESHOLDS = {'low': 0.3, 'medium': 0.6, 'high': 0.8}
+AMBIGUITY_MARGIN = 0.05
+
+SETTING_CURRENT_SPACE = 'voice_space_current'
+SETTING_LEGACY_SPACE = 'voice_space_legacy'
+
+
+# ------------------------------------------------------------------- vectors
+
+def normalize(vector):
+ """Float32 unit vector, or None for anything that is not a usable embedding."""
+ try:
+ arr = np.asarray(vector, dtype=np.float32).reshape(-1)
+ except (TypeError, ValueError):
+ return None
+ if arr.size == 0 or not np.all(np.isfinite(arr)):
+ return None
+ norm = float(np.linalg.norm(arr))
+ if norm == 0.0:
+ return None
+ return arr / norm
+
+
+def to_bytes(unit_vector):
+ return np.asarray(unit_vector, dtype=np.float32).tobytes()
+
+
+def from_bytes(data):
+ return np.frombuffer(data, dtype=np.float32).copy()
+
+
+def cosine(a, b):
+ if a is None or b is None or a.shape != b.shape:
+ return None
+ return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))
+
+
+def load_embeddings(recording):
+ """The recording's {label: vector} map, whatever form it was stored in."""
+ import json
+ data = recording.speaker_embeddings
+ if not data:
+ return {}
+ if isinstance(data, str):
+ try:
+ data = json.loads(data)
+ except (TypeError, ValueError):
+ return {}
+ return data if isinstance(data, dict) else {}
+
+
+def speech_seconds_by_label(segments):
+ """Seconds of speech per speaker value in a JSON transcript.
+
+ Returns None when the segments carry no usable timing, so callers can
+ tell "no timing" from "no speech".
+ """
+ totals = {}
+ timed = False
+ for seg in segments or []:
+ start, end = seg.get('start_time', seg.get('startTime')), seg.get('end_time', seg.get('endTime'))
+ spk = seg.get('speaker')
+ if not spk or start is None or end is None:
+ continue
+ try:
+ dur = max(0.0, float(end) - float(start))
+ except (TypeError, ValueError):
+ continue
+ timed = True
+ totals[spk] = totals.get(spk, 0.0) + dur
+ return totals if timed else None
+
+
+def enough_speech(seconds_by_label, label):
+ """True when the label has enough speech, or when timing is unknown."""
+ if seconds_by_label is None:
+ return True
+ return seconds_by_label.get(label, 0.0) >= MIN_SPEECH_SECONDS
+
+
+# -------------------------------------------------------------------- spaces
+
+def _disabled():
+ return os.environ.get('DISABLE_VOICE_EMBEDDING_CHECK', 'false').lower() == 'true'
+
+
+def _setting_int(key):
+ from src.models import SystemSetting
+ value = SystemSetting.get_setting(key, None)
+ try:
+ return int(value) if value not in (None, '') else None
+ except (TypeError, ValueError):
+ return None
+
+
+def current_space_id():
+ """Space of the backend in use now, or None when it cannot be known
+ (check disabled, backend without embeddings, or never probed)."""
+ if _disabled():
+ return None
+ return _setting_int(SETTING_CURRENT_SPACE)
+
+
+def legacy_space_id():
+ """The space everything stored without a space belongs to."""
+ return _setting_int(SETTING_LEGACY_SPACE)
+
+
+def effective_space(space_id):
+ return space_id if space_id is not None else legacy_space_id()
+
+
+def register_space(vector, fingerprint=None, similarity_threshold=0.98):
+ """Find the space whose canary embedding matches this one, or create it.
+
+ Makes it the current space and returns its id. The first space ever
+ registered becomes the legacy space. Called by the canary check.
+ """
+ from src.models import SystemSetting, VoiceEmbeddingSpace
+ vec = normalize(vector)
+ if vec is None:
+ return None
+ best, best_sim = None, -1.0
+ for space in VoiceEmbeddingSpace.query.all():
+ ref = normalize(space.canary_embedding or [])
+ sim = cosine(vec, ref) if ref is not None else None
+ if sim is not None and sim > best_sim:
+ best, best_sim = space, sim
+ if best is not None and best_sim >= similarity_threshold:
+ space = best
+ space.last_seen_at = datetime.utcnow()
+ if fingerprint:
+ space.backend_fingerprint = fingerprint
+ db.session.commit()
+ else:
+ space = VoiceEmbeddingSpace(dimension=int(vec.shape[0]), canary_embedding=[float(x) for x in vector],
+ backend_fingerprint=fingerprint)
+ db.session.add(space)
+ db.session.commit()
+ logger.info(f"Registered voice embedding space {space.id} ({space.dimension} dimensions)")
+ if legacy_space_id() is None:
+ SystemSetting.set_setting(SETTING_LEGACY_SPACE, str(space.id), setting_type='string',
+ description='Voice embedding space that profiles stored before spaces existed belong to')
+ SystemSetting.set_setting(SETTING_CURRENT_SPACE, str(space.id), setting_type='string',
+ description='Voice embedding space of the transcription backend in use')
+ _calibration_cache.clear()
+ return space.id
+
+
+def clear_current_space():
+ """The backend returns no embeddings: nothing new gets a space."""
+ from src.models import SystemSetting
+ if _setting_int(SETTING_CURRENT_SPACE) is not None:
+ SystemSetting.set_setting(SETTING_CURRENT_SPACE, '', setting_type='string',
+ description='Voice embedding space of the transcription backend in use')
+
+
+# ------------------------------------------------------------------- samples
+
+class _Sample:
+ """What matching needs from a sample; also stands in for a legacy average."""
+ __slots__ = ('vector', 'weight', 'row', 'created_at')
+
+ def __init__(self, vector, weight, row=None, created_at=None):
+ self.vector, self.weight, self.row, self.created_at = vector, weight, row, created_at
+
+
+def _legacy_sample(speaker):
+ if not speaker.average_embedding:
+ return None
+ try:
+ vec = normalize(from_bytes(speaker.average_embedding))
+ except ValueError as e:
+ # Unreadable stored bytes: skipped, but said so rather than silently.
+ logger.warning("Skipping speaker %s: its stored embedding could not be compared: %s",
+ getattr(speaker, 'id', '?'), e)
+ return None
+ if vec is None:
+ return None
+ weight = float(min(max(speaker.embedding_count or 1, 1), LEGACY_MAX_WEIGHT))
+ return _Sample(vec, weight, None, speaker.created_at)
+
+
+def samples_for(speaker, space_id, rows=None):
+ """The person's samples in one (effective) space, legacy average included."""
+ from src.models import SpeakerVoiceSample
+ target = effective_space(space_id)
+ if rows is None:
+ rows = SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).all()
+ out = []
+ for row in rows:
+ if effective_space(row.space_id) != target:
+ continue
+ try:
+ vec = from_bytes(row.embedding)
+ except ValueError:
+ continue
+ if vec.size:
+ out.append(_Sample(vec, float(row.weight or 1.0), row, row.created_at))
+ if not rows:
+ legacy = _legacy_sample(speaker)
+ if legacy is not None and target == legacy_space_id():
+ out.append(legacy)
+ return out
+
+
+def build_variants(samples):
+ """Cluster samples into voice variants.
+
+ Greedy and deterministic: samples are visited oldest first and join the
+ closest variant when at least VARIANT_MERGE_SIMILARITY similar to its
+ centroid, otherwise start a new one. Beyond MAX_VARIANTS the two closest
+ variants merge. Returns [{'centroid', 'support', 'count'}].
+ """
+ variants = []
+ for s in sorted(samples, key=lambda x: x.created_at or datetime.min):
+ best, best_sim = None, -1.0
+ for v in variants:
+ sim = float(np.dot(s.vector, v['centroid']))
+ if sim > best_sim:
+ best, best_sim = v, sim
+ if best is not None and best_sim >= VARIANT_MERGE_SIMILARITY and best['centroid'].shape == s.vector.shape:
+ best['sum'] = best['sum'] + s.weight * s.vector
+ best['support'] += s.weight
+ best['count'] += 1
+ best['centroid'] = best['sum'] / np.linalg.norm(best['sum'])
+ else:
+ variants.append({'sum': s.weight * s.vector, 'centroid': s.vector.copy(),
+ 'support': s.weight, 'count': 1})
+ while len(variants) > MAX_VARIANTS:
+ pair, best_sim = None, -2.0
+ for i in range(len(variants)):
+ for j in range(i + 1, len(variants)):
+ sim = float(np.dot(variants[i]['centroid'], variants[j]['centroid']))
+ if sim > best_sim:
+ pair, best_sim = (i, j), sim
+ i, j = pair
+ merged_sum = variants[i]['sum'] + variants[j]['sum']
+ merged = {'sum': merged_sum, 'centroid': merged_sum / np.linalg.norm(merged_sum),
+ 'support': variants[i]['support'] + variants[j]['support'],
+ 'count': variants[i]['count'] + variants[j]['count']}
+ variants = [v for k, v in enumerate(variants) if k not in (i, j)] + [merged]
+ return [{'centroid': v['centroid'], 'support': v['support'], 'count': v['count']} for v in variants]
+
+
+def score_against(vector, variants, total_samples):
+ """Similarity of a voice to a person: the closest variant counts."""
+ best = None
+ for v in variants:
+ if v['centroid'].shape != vector.shape:
+ continue
+ sim = float(np.dot(vector, v['centroid']))
+ if v['count'] == 1 and total_samples > 1:
+ sim -= SINGLETON_PENALTY
+ best = sim if best is None else max(best, sim)
+ return best
+
+
+def _materialize_legacy(speaker):
+ """Write a speaker's pre-sample average out as a real sample, once."""
+ from src.models import SpeakerVoiceSample
+ if SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).count():
+ return
+ legacy = _legacy_sample(speaker)
+ if legacy is None:
+ return
+ db.session.add(SpeakerVoiceSample(
+ user_id=speaker.user_id, speaker_id=speaker.id, recording_id=None, label=None,
+ embedding=to_bytes(legacy.vector), dimension=int(legacy.vector.shape[0]),
+ space_id=None, speech_seconds=None, source='legacy', weight=legacy.weight,
+ created_at=speaker.created_at or datetime.utcnow()))
+ db.session.flush()
+
+
+def record_sample(speaker, recording, label, vector, speech_seconds=None, source='confirmed'):
+ """Assign one (recording, label) voice to a person.
+
+ Replaces whatever sample that (recording, label) had, so re-saving or
+ correcting a name never double-counts. Returns 'stored', 'rejected'
+ (outlier) or 'invalid'. Does not commit.
+ """
+ from src.models import SpeakerVoiceSample
+ vec = normalize(vector)
+ if vec is None:
+ return 'invalid'
+ space = recording.speaker_embeddings_space_id
+ existing = SpeakerVoiceSample.query.filter_by(recording_id=recording.id, label=label).first()
+
+ _materialize_legacy(speaker)
+ others = [s for s in samples_for(speaker, space)
+ if not (existing is not None and s.row is not None and s.row.id == existing.id)]
+ if len(others) >= OUTLIER_MIN_SAMPLES:
+ best = max((float(np.dot(vec, s.vector)) for s in others if s.vector.shape == vec.shape), default=None)
+ if best is not None and best < OUTLIER_SIMILARITY:
+ logger.warning(
+ "Voice of %s in recording %s is unlike every sample of '%s' (best similarity %.2f); "
+ "not adding it to the profile. The name may be wrong.",
+ label, recording.id, speaker.name, best)
+ if existing is not None:
+ db.session.delete(existing)
+ return 'rejected'
+
+ weight = AUTO_SAMPLE_WEIGHT if source == 'auto' else 1.0
+ if existing is None:
+ existing = SpeakerVoiceSample(user_id=speaker.user_id, recording_id=recording.id, label=label)
+ db.session.add(existing)
+ existing.speaker_id = speaker.id
+ existing.embedding = to_bytes(vec)
+ existing.dimension = int(vec.shape[0])
+ existing.space_id = space
+ existing.speech_seconds = speech_seconds
+ existing.source = source
+ existing.weight = weight
+ existing.updated_at = datetime.utcnow()
+ db.session.flush()
+ _trim(speaker, space)
+ _calibration_cache.clear()
+ return 'stored'
+
+
+def _trim(speaker, space):
+ from src.models import SpeakerVoiceSample
+ target = effective_space(space)
+ rows = [r for r in SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).all()
+ if effective_space(r.space_id) == target]
+ if len(rows) <= MAX_SAMPLES:
+ return
+ rows.sort(key=lambda r: r.created_at or datetime.min)
+ for r in rows[:len(rows) - MAX_SAMPLES]:
+ db.session.delete(r)
+ db.session.flush()
+
+
+def refresh_speaker_summary(speaker):
+ """Keep the pre-sample columns true to the samples.
+
+ average_embedding: weighted mean of the samples in the current space (or
+ the legacy space when there is no current one), for API fields and any
+ code that still reads it. embedding_count: samples in all spaces.
+ embeddings_history: the last ten sample sources, which the orphaned-speaker
+ cleanup reads. Does not commit.
+ """
+ from src.models import SpeakerVoiceSample
+ rows = SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).all()
+ if not rows:
+ speaker.average_embedding = None
+ speaker.embedding_count = 0
+ speaker.embeddings_history = None
+ speaker.confidence_score = None
+ return
+ space = current_space_id()
+ chosen = samples_for(speaker, space, rows) or samples_for(speaker, None, rows)
+ if chosen:
+ dims = chosen[0].vector.shape
+ same = [s for s in chosen if s.vector.shape == dims]
+ total = sum(s.weight * s.vector for s in same)
+ norm = float(np.linalg.norm(total))
+ speaker.average_embedding = to_bytes(total / norm) if norm else None
+ centroid = total / norm if norm else None
+ sims = [float(np.dot(s.vector, centroid)) for s in same] if centroid is not None else []
+ cohesion = sum(sims) / len(sims) if sims else 0.5
+ speaker.confidence_score = round(min(1.0, max(0.0, cohesion * min(1.0, len(same) / 5.0))), 3)
+ speaker.embedding_count = len(rows)
+ history = []
+ for r in sorted(rows, key=lambda r: r.updated_at or r.created_at or datetime.min)[-10:]:
+ history.append({'recording_id': r.recording_id,
+ 'timestamp': (r.updated_at or r.created_at or datetime.utcnow()).isoformat(),
+ 'source': r.source})
+ speaker.embeddings_history = history
+
+
+def voice_summary(speaker):
+ """Counts for the speakers list: samples and variants per space."""
+ from src.models import SpeakerVoiceSample
+ rows = SpeakerVoiceSample.query.filter_by(speaker_id=speaker.id).all()
+ space = current_space_id()
+ current = samples_for(speaker, space, rows)
+ stored_current = len([s for s in current if s.row is not None])
+ return {
+ 'sample_count': len(rows) if rows else (1 if current else 0),
+ 'current_space_samples': len(current),
+ 'variant_count': len(build_variants(current)) if current else 0,
+ 'other_space_samples': len(rows) - stored_current if rows else 0,
+ }
+
+
+# ------------------------------------------------------------------ matching
+
+def _user_profiles(user_id, space_id, exclude_speaker_ids=()):
+ from src.models import Speaker, SpeakerVoiceSample
+ speakers = Speaker.query.filter_by(user_id=user_id).all()
+ rows_by_speaker = {}
+ for row in SpeakerVoiceSample.query.filter_by(user_id=user_id).all():
+ rows_by_speaker.setdefault(row.speaker_id, []).append(row)
+ profiles = []
+ for sp in speakers:
+ if sp.id in exclude_speaker_ids:
+ continue
+ rows = rows_by_speaker.get(sp.id, [])
+ samples = samples_for(sp, space_id, rows)
+ if not samples:
+ continue
+ profiles.append((sp, build_variants(samples), len(samples)))
+ return profiles
+
+
+def find_matches(vector, user_id, space_id=None, threshold=None):
+ """People whose voice matches, best first.
+
+ [{'speaker_id', 'name', 'similarity' (percent), 'confidence',
+ 'embedding_count', 'variant_count'}]. Only profiles in the same
+ embedding space and dimension take part.
+ """
+ vec = normalize(vector)
+ if vec is None:
+ return []
+ if threshold is None:
+ threshold = suggestion_threshold(space_id)
+ matches = []
+ profiles = _user_profiles(user_id, space_id)
+ mismatched_dims = set()
+ mismatched = 0
+ for sp, variants, count in profiles:
+ if not any(v['centroid'].shape == vec.shape for v in variants):
+ mismatched += 1
+ mismatched_dims.update(int(v['centroid'].shape[0]) for v in variants)
+ continue
+ sim = score_against(vec, variants, count)
+ if sim is not None and sim >= threshold:
+ matches.append({'speaker_id': sp.id, 'name': sp.name, 'similarity': round(sim * 100, 1),
+ 'confidence': sp.confidence_score or 0.5, 'embedding_count': count,
+ 'variant_count': len(variants)})
+ if mismatched:
+ # One line per call, so it stays readable when a whole library is
+ # affected, which is the usual case.
+ sizes = ', '.join(str(d) for d in sorted(mismatched_dims))
+ logger.error(
+ "%d of %d stored voice profiles have %s dimensions and cannot be compared "
+ "with the %d-dimensional embeddings this backend now returns. Voice matching "
+ "will find nothing until the previous transcription backend is restored or "
+ "the profiles are rebuilt.",
+ mismatched, len(profiles), sizes, vec.shape[0])
+ return sorted(matches, key=lambda m: m['similarity'], reverse=True)
+
+
+def forget_recording_samples(recording):
+ """Drop a recording's voice samples and label map (it was re-diarized)."""
+ from src.models import Speaker, SpeakerVoiceSample
+ rows = SpeakerVoiceSample.query.filter_by(recording_id=recording.id).all()
+ touched = {r.speaker_id for r in rows}
+ for r in rows:
+ db.session.delete(r)
+ recording.speaker_label_map = None
+ db.session.flush()
+ for sid in touched:
+ sp = db.session.get(Speaker, sid)
+ if sp is not None:
+ refresh_speaker_summary(sp)
+ if touched:
+ _calibration_cache.clear()
+
+
+def suggestions_for_recording(recording, user_id, threshold=None):
+ """Voice-match suggestions per label, keyed the way the transcript shows it.
+
+ A label already renamed in the transcript is keyed by its current name,
+ so the speaker dialog can offer a confirmation for it.
+ """
+ import json
+ embeddings = load_embeddings(recording)
+ label_map = recording.speaker_label_map or {}
+ present = set()
+ try:
+ present = {s.get('speaker') for s in json.loads(recording.transcription or '[]') if isinstance(s, dict)}
+ except (TypeError, ValueError):
+ pass
+ out = {}
+ for label, vector in embeddings.items():
+ key = label if label in present else label_map.get(label, label)
+ out[key] = find_matches(vector, user_id, recording.speaker_embeddings_space_id, threshold)
+ return out
+
+
+def auto_label_map(recording, user, threshold):
+ """{label: name} for a new recording: one person per label, one label per person.
+
+ Labels with too little speech are skipped, as is any label whose two best
+ candidates are within AMBIGUITY_MARGIN. The remaining pairs are assigned
+ greedily from the most similar down, so no person gets two labels.
+ """
+ import json
+ embeddings = load_embeddings(recording)
+ if not embeddings:
+ return {}
+ try:
+ seconds = speech_seconds_by_label(json.loads(recording.transcription or '[]'))
+ except (TypeError, ValueError):
+ seconds = None
+ candidates = []
+ for label, vector in embeddings.items():
+ if not enough_speech(seconds, label):
+ continue
+ matches = find_matches(vector, user.id, recording.speaker_embeddings_space_id, threshold)
+ if not matches:
+ continue
+ if len(matches) >= 2 and (matches[0]['similarity'] - matches[1]['similarity']) / 100.0 <= AMBIGUITY_MARGIN:
+ continue
+ for m in matches:
+ candidates.append((m['similarity'], label, m['speaker_id'], m['name']))
+ result, used_people = {}, set()
+ for sim, label, speaker_id, name in sorted(candidates, reverse=True):
+ if label in result or speaker_id in used_people:
+ continue
+ result[label] = name
+ used_people.add(speaker_id)
+ return result
+
+
+# ------------------------------------------------------------- calibration
+
+_calibration_cache = {}
+_CALIBRATION_TTL = 600
+MIN_CALIBRATION_PAIRS = 10
+
+
+def calibrated_threshold(space_id):
+ """A match threshold learned from this space's own samples, or None.
+
+ Genuine scores: each sample against its own person's other samples.
+ Impostor scores: each sample against the other people of the same user.
+ The threshold sits midway between the 10th percentile of genuine scores
+ and the 95th percentile of impostor scores, within [0.45, 0.85]. None
+ until there are MIN_CALIBRATION_PAIRS of each.
+ """
+ from src.models import Speaker, SpeakerVoiceSample
+ target = effective_space(space_id)
+ cached = _calibration_cache.get(target)
+ if cached and time.time() - cached[0] < _CALIBRATION_TTL:
+ return cached[1]
+ by_user = {}
+ for row in SpeakerVoiceSample.query.all():
+ if effective_space(row.space_id) != target:
+ continue
+ vec = from_bytes(row.embedding)
+ if vec.size:
+ by_user.setdefault(row.user_id, {}).setdefault(row.speaker_id, []).append(vec)
+ genuine, impostor = [], []
+ for people in by_user.values():
+ for pid, vecs in people.items():
+ for i, v in enumerate(vecs):
+ own = [float(np.dot(v, w)) for j, w in enumerate(vecs) if j != i and w.shape == v.shape]
+ if own:
+ genuine.append(max(own))
+ other = [float(np.dot(v, w)) for qid, ws in people.items() if qid != pid for w in ws if w.shape == v.shape]
+ if other:
+ impostor.append(max(other))
+ value = None
+ if len(genuine) >= MIN_CALIBRATION_PAIRS and len(impostor) >= MIN_CALIBRATION_PAIRS:
+ g10 = float(np.percentile(genuine, 10))
+ i95 = float(np.percentile(impostor, 95))
+ value = round(min(0.85, max(0.45, (g10 + i95) / 2.0)), 3)
+ _calibration_cache[target] = (time.time(), value)
+ return value
+
+
+def suggestion_threshold(space_id):
+ cal = calibrated_threshold(space_id)
+ return round(cal - 0.10, 3) if cal is not None else DEFAULT_SUGGESTION_THRESHOLD
+
+
+def auto_label_threshold(setting, space_id):
+ cal = calibrated_threshold(space_id)
+ if cal is None:
+ return DEFAULT_AUTO_LABEL_THRESHOLDS.get(setting or 'medium', DEFAULT_AUTO_LABEL_THRESHOLDS['medium'])
+ offsets = {'low': -0.10, 'medium': 0.0, 'high': 0.08}
+ return round(min(0.95, max(0.3, cal + offsets.get(setting or 'medium', 0.0))), 3)
+
+
+# ------------------------------------------------------ applying names
+
+def apply_names_to_profiles(recording, key_to_name, seconds_by_key, user, source='confirmed'):
+ """Train voice profiles from the names given in one save.
+
+ key_to_name maps the speaker values the transcript showed before the save
+ (a diarization label, or a name given earlier) to the name now assigned.
+ Each key is traced back to its diarization label through the recording's
+ label map, so a correction reaches the embedding even after the transcript
+ shows names. Labels that no longer appear in the transcript (merged away)
+ lose their sample. Updates the label map and the affected people's summary
+ columns. Does not commit.
+
+ Returns {'stored': n, 'rejected': n, 'skipped_short': n}.
+ """
+ import json
+ from src.models import SpeakerVoiceSample
+ from src.services.speaker import find_user_speaker
+
+ stats = {'stored': 0, 'rejected': 0, 'skipped_short': 0}
+ embeddings = load_embeddings(recording)
+ label_map = dict(recording.speaker_label_map or {})
+
+ def labels_for(key):
+ if key in embeddings:
+ return [key]
+ return [lab for lab, name in label_map.items() if name == key and lab in embeddings]
+
+ touched = set()
+ for key, name in key_to_name.items():
+ for label in labels_for(key):
+ label_map[label] = name
+ if not embeddings.get(label):
+ continue
+ seconds = (seconds_by_key or {}).get(key) if seconds_by_key is not None else None
+ if seconds_by_key is not None and (seconds or 0.0) < MIN_SPEECH_SECONDS:
+ stats['skipped_short'] += 1
+ old = SpeakerVoiceSample.query.filter_by(recording_id=recording.id, label=label).first()
+ if old is not None:
+ touched.add(old.speaker_id)
+ db.session.delete(old)
+ continue
+ speaker = find_user_speaker(user.id, name)
+ if speaker is None:
+ continue
+ old = SpeakerVoiceSample.query.filter_by(recording_id=recording.id, label=label).first()
+ if old is not None and old.speaker_id != speaker.id:
+ touched.add(old.speaker_id)
+ result = record_sample(speaker, recording, label, embeddings[label], seconds, source)
+ if result in stats:
+ stats[result] += 1
+ touched.add(speaker.id)
+
+ # Labels whose voice no longer appears in the transcript lose their sample.
+ try:
+ present = {s.get('speaker') for s in json.loads(recording.transcription or '[]') if isinstance(s, dict)}
+ except (TypeError, ValueError):
+ present = None
+ if present is not None:
+ for row in SpeakerVoiceSample.query.filter_by(recording_id=recording.id).all():
+ shown = label_map.get(row.label, row.label)
+ if row.label not in present and shown not in present:
+ touched.add(row.speaker_id)
+ db.session.delete(row)
+
+ recording.speaker_label_map = label_map or None
+ db.session.flush()
+ from src.models import Speaker
+ for sid in touched:
+ sp = db.session.get(Speaker, sid)
+ if sp is not None:
+ refresh_speaker_summary(sp)
+ return stats
diff --git a/src/tasks/processing.py b/src/tasks/processing.py
index 48b0b3f2..4e6803bd 100644
--- a/src/tasks/processing.py
+++ b/src/tasks/processing.py
@@ -2156,6 +2156,15 @@ def transcribe_with_connector(app_context, recording_id, filepath, original_file
recording.speaker_embeddings = response.speaker_embeddings or None
if recording.speaker_embeddings:
current_app.logger.info(f"Stored speaker embeddings for speakers: {list(response.speaker_embeddings.keys())}")
+ # The new labels are a new diarization: voice samples and
+ # the label -> name map of the previous run no longer
+ # apply, and the embeddings carry the backend's space.
+ try:
+ from src.services.voice_profiles import current_space_id, forget_recording_samples
+ forget_recording_samples(recording)
+ recording.speaker_embeddings_space_id = current_space_id() if recording.speaker_embeddings else None
+ except Exception as vp_err:
+ current_app.logger.warning(f"Could not reset voice samples for recording {recording.id}: {vp_err}")
# If we reach here, transcription succeeded
break
diff --git a/tests/test_api_v1_speakers.py b/tests/test_api_v1_speakers.py
index 89196301..853ad639 100644
--- a/tests/test_api_v1_speakers.py
+++ b/tests/test_api_v1_speakers.py
@@ -443,9 +443,10 @@ def test_assign_embeddings_updated():
speaker = _create_test_speaker(user, "Alice")
client = app.test_client()
try:
- mock_update = MagicMock()
+ from src.models import SpeakerVoiceSample
mock_snippets = MagicMock(return_value=2)
- with patch("src.services.speaker_embedding_matcher.update_speaker_embedding", mock_update), \
+ # Fixture segments are short; the speech minimum is not under test.
+ with patch("src.services.voice_profiles.MIN_SPEECH_SECONDS", 0.0), \
patch("src.services.speaker_snippets.create_speaker_snippets", mock_snippets):
resp = client.put(f"/api/v1/recordings/{rec.id}/speakers/assign",
headers={"X-API-Token": token},
@@ -453,7 +454,10 @@ def test_assign_embeddings_updated():
assert resp.status_code == 200, f"Expected 200, got {resp.status_code}"
body = resp.get_json()
assert body.get("embeddings_updated") >= 1, f"embeddings_updated: {body}"
- mock_update.assert_called()
+ sample = SpeakerVoiceSample.query.filter_by(recording_id=rec.id, label="SPEAKER_00").first()
+ assert sample is not None and sample.speaker_id == speaker.id
+ SpeakerVoiceSample.query.filter_by(recording_id=rec.id).delete()
+ db.session.commit()
finally:
_cleanup(rec, speaker, token_rec)
if cu:
diff --git a/tests/test_recording_deletion.py b/tests/test_recording_deletion.py
index 700734b8..c06dcff9 100644
--- a/tests/test_recording_deletion.py
+++ b/tests/test_recording_deletion.py
@@ -100,7 +100,7 @@ def test_web_delete_still_works(ctx):
sess["_user_id"] = str(user.id)
sess["_fresh"] = True
storage = _storage()
- with patch("src.services.storage.get_storage_service", return_value=storage):
+ with patch("src.api.recordings.get_storage_service", return_value=storage):
r = c.delete(f'/recording/{rec.id}')
assert r.status_code == 200, r.get_data(as_text=True)
storage.delete.assert_called_once()
diff --git a/tests/test_speaker_save_paths.py b/tests/test_speaker_save_paths.py
index 364dd586..25af26b4 100644
--- a/tests/test_speaker_save_paths.py
+++ b/tests/test_speaker_save_paths.py
@@ -97,6 +97,7 @@ def test_two_labels_with_one_name_merge_into_one_speaker(ctx):
@pytest.mark.parametrize("route", ["update_speakers", "update_transcript"])
def test_both_routes_update_voice_profiles_and_snippets(ctx, route):
+ from src.models import SpeakerVoiceSample
user = _user()
rec = _recording(user, SEGMENTS, embeddings={"SPEAKER_00": EMB, "SPEAKER_01": EMB})
body = {"speaker_map": {"SPEAKER_00": {"name": "Ana"}}}
@@ -105,14 +106,17 @@ def test_both_routes_update_voice_profiles_and_snippets(ctx, route):
staged[2]["speaker"] = "SPEAKER_00" # a staged per-line reassignment
body["transcript_data"] = staged
- with patch("src.services.speaker_embedding_matcher.update_speaker_embedding", return_value=None) as upd, \
+ # The fixture segments are seconds long; the speech minimum is not what
+ # this test is about.
+ with patch("src.services.voice_profiles.MIN_SPEECH_SECONDS", 0.0), \
patch("src.services.speaker_snippets.create_speaker_snippets", return_value=2) as snip:
r = _client(user).post(f"/recording/{rec.id}/{route}", json=body)
assert r.status_code == 200
- assert upd.call_count == 1
- speaker, embedding, rec_id = upd.call_args.args
- assert speaker.name == "Ana" and rec_id == rec.id and len(embedding) == 256
+ ana = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ samples = SpeakerVoiceSample.query.filter_by(recording_id=rec.id).all()
+ assert [(s.speaker_id, s.label, s.source) for s in samples] == [(ana.id, "SPEAKER_00", "confirmed")]
+ assert ana.average_embedding is not None and ana.embedding_count == 1
snip.assert_called_once()
assert snip.call_args.args[1] == {"SPEAKER_00": {"name": "Ana"}}
saved = json.loads(db.session.get(Recording, rec.id).transcription)
diff --git a/tests/test_voice_profiles.py b/tests/test_voice_profiles.py
new file mode 100644
index 00000000..9e33a9d3
--- /dev/null
+++ b/tests/test_voice_profiles.py
@@ -0,0 +1,425 @@
+"""Voice profiles built from samples, per embedding space.
+
+Covers the three stages of the voice-matching rework:
+1. dimension-agnostic, normalized embeddings; a speech minimum; outlier
+ samples kept out; one person per label when auto-labelling
+2. embedding spaces: profiles only compared within the model that made them,
+ with everything stored before spaces existed in the legacy space
+3. several voice variants per person, rebuilt from samples, so corrections
+ undo themselves; thresholds calibrated from the data
+
+SHARED-DB: every test uses its own users; space settings are restored.
+"""
+
+import json
+import os
+import sys
+import uuid
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+from flask import g
+from flask.testing import FlaskClient
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from src.app import app, db
+from src.models import User, Recording, Speaker, SpeakerVoiceSample, VoiceEmbeddingSpace, SystemSetting
+from src.services import voice_profiles as vp
+
+app.config["WTF_CSRF_ENABLED"] = False
+RNG = np.random.default_rng(7)
+DIM = 256
+
+
+def unit(v):
+ v = np.asarray(v, dtype=np.float32)
+ return v / np.linalg.norm(v)
+
+
+def person(dim=DIM):
+ return unit(RNG.normal(size=dim))
+
+
+def near(base, noise=0.15):
+ return unit(base + noise * RNG.normal(size=base.shape[0]) / np.sqrt(base.shape[0]) * 4)
+
+
+class _Client(FlaskClient):
+ def open(self, *args, **kwargs):
+ g.pop('_login_user', None)
+ return super().open(*args, **kwargs)
+
+
+def ctx():
+ with app.app_context():
+ saved = {k: SystemSetting.get_setting(k, None) for k in (vp.SETTING_CURRENT_SPACE, vp.SETTING_LEGACY_SPACE)}
+ vp._calibration_cache.clear()
+ yield
+ db.session.rollback()
+ for k, v in saved.items():
+ SystemSetting.set_setting(k, v or '', setting_type='string')
+ vp._calibration_cache.clear()
+
+
+def _user(**kw):
+ s = uuid.uuid4().hex[:8]
+ u = User(username=f"vp_{s}", email=f"vp_{s}@local.test", password="x", **kw)
+ db.session.add(u)
+ db.session.commit()
+ return u
+
+
+def _speaker(user, name, average=None, count=0):
+ sp = Speaker(user_id=user.id, name=name, use_count=1,
+ average_embedding=vp.to_bytes(unit(average)) if average is not None else None,
+ embedding_count=count)
+ db.session.add(sp)
+ db.session.commit()
+ return sp
+
+
+def _recording(user, embeddings, seconds=None, space=None, names=None):
+ """A diarized recording; seconds maps label -> speech seconds (default 60)."""
+ segs, t = [], 0.0
+ for label in embeddings:
+ dur = (seconds or {}).get(label, 60.0)
+ segs.append({"speaker": (names or {}).get(label, label), "sentence": "words " * 5,
+ "start_time": t, "end_time": t + dur})
+ t += dur
+ rec = Recording(user_id=user.id, title="r", status="COMPLETED", audio_path="local://x.mp3",
+ original_filename="x.mp3", transcription=json.dumps(segs),
+ speaker_embeddings={k: [float(x) for x in v] for k, v in embeddings.items()},
+ speaker_embeddings_space_id=space)
+ db.session.add(rec)
+ db.session.commit()
+ return rec
+
+
+def _client(user):
+ c = _Client(app, app.response_class)
+ with c.session_transaction() as sess:
+ sess["_user_id"] = str(user.id)
+ sess["_fresh"] = True
+ return c
+
+
+def _save_names(user, rec, mapping):
+ """Name speakers through the real save route."""
+ body = {"speaker_map": {k: {"name": v} for k, v in mapping.items()}, "regenerate_summary": False}
+ with patch("src.services.speaker_snippets.create_speaker_snippets", return_value=0):
+ r = _client(user).post(f"/recording/{rec.id}/update_speakers", json=body)
+ assert r.status_code == 200, r.get_data(as_text=True)
+ db.session.expire_all()
+
+
+# ------------------------------------------------------------- stage 1
+
+def test_normalize_rejects_what_is_not_an_embedding():
+ assert vp.normalize([]) is None
+ assert vp.normalize([0.0, 0.0]) is None
+ assert vp.normalize([1.0, float('nan')]) is None
+ assert vp.normalize("abc") is None
+ assert np.isclose(np.linalg.norm(vp.normalize([3.0, 4.0])), 1.0)
+
+
+def test_speech_seconds_per_label():
+ segs = [{"speaker": "A", "start_time": 0, "end_time": 10}, {"speaker": "B", "start_time": 10, "end_time": 12},
+ {"speaker": "A", "start_time": 12, "end_time": 20}]
+ assert vp.speech_seconds_by_label(segs) == {"A": 18.0, "B": 2.0}
+ assert vp.speech_seconds_by_label([{"speaker": "A", "sentence": "x"}]) is None
+
+
+def test_any_dimension_works_end_to_end(ctx):
+ user = _user()
+ ana = person(192)
+ rec = _recording(user, {"SPEAKER_00": ana})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ rec2 = _recording(user, {"SPEAKER_00": near(ana)})
+ r = _client(user).get(f"/speakers/suggestions/{rec2.id}")
+ names = [m["name"] for m in r.get_json()["suggestions"]["SPEAKER_00"]]
+ assert names == ["Ana"]
+
+
+def test_short_speech_trains_nothing(ctx):
+ user = _user()
+ rec = _recording(user, {"SPEAKER_00": person()}, seconds={"SPEAKER_00": 5.0})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ assert SpeakerVoiceSample.query.filter_by(recording_id=rec.id).count() == 0
+ # The name is still applied to the transcript.
+ assert json.loads(db.session.get(Recording, rec.id).transcription)[0]["speaker"] == "Ana"
+
+
+def test_outlier_sample_is_kept_out(ctx):
+ user = _user()
+ ana = person()
+ for _ in range(3):
+ _save_names(user, _recording(user, {"SPEAKER_00": near(ana)}), {"SPEAKER_00": "Ana"})
+ wrong = _recording(user, {"SPEAKER_00": -ana}) # plainly someone else
+ _save_names(user, wrong, {"SPEAKER_00": "Ana"})
+ sp = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ assert SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).count() == 3
+ assert SpeakerVoiceSample.query.filter_by(recording_id=wrong.id).count() == 0
+
+
+def test_auto_labelling_gives_each_person_one_label(ctx):
+ user = _user(auto_speaker_labelling=True, auto_speaker_labelling_threshold='medium')
+ ana, bea = person(), person()
+ _save_names(user, _recording(user, {"SPEAKER_00": ana, "SPEAKER_01": bea}), {"SPEAKER_00": "Ana", "SPEAKER_01": "Bea"})
+ # Two labels both closest to Ana: only the closer one may become Ana.
+ rec = _recording(user, {"SPEAKER_00": near(ana, 0.05), "SPEAKER_01": near(ana, 0.3)})
+ from src.services.speaker_embedding_matcher import apply_auto_speaker_labels
+ result = apply_auto_speaker_labels(rec, user)
+ assert result.get("SPEAKER_00") == "Ana"
+ assert result.get("SPEAKER_01") != "Ana"
+
+
+def test_auto_labelling_skips_short_labels(ctx):
+ user = _user(auto_speaker_labelling=True)
+ ana = person()
+ _save_names(user, _recording(user, {"SPEAKER_00": ana}), {"SPEAKER_00": "Ana"})
+ rec = _recording(user, {"SPEAKER_00": near(ana, 0.05)}, seconds={"SPEAKER_00": 4.0})
+ from src.services.speaker_embedding_matcher import apply_auto_speaker_labels
+ assert apply_auto_speaker_labels(rec, user) == {}
+
+
+# ------------------------------------------------------------- stage 3
+
+def test_variants_keep_two_voice_conditions_apart():
+ base = person()
+ phone_voice, room_voice = unit(base + 0.9 * person()), unit(base - 0.9 * person())
+ phone = [vp._Sample(near(phone_voice, 0.05), 1.0) for _ in range(3)]
+ room = [vp._Sample(near(room_voice, 0.05), 1.0) for _ in range(3)]
+ variants = vp.build_variants(phone + room)
+ assert len(variants) == 2
+ assert sorted(v["count"] for v in variants) == [3, 3]
+
+
+def test_closest_variant_counts_where_one_average_fails(ctx):
+ user = _user()
+ base = person()
+ phone, room = unit(base + 1.2 * person()), unit(base + 1.2 * person())
+ for voice in (phone, phone, room, room):
+ _save_names(user, _recording(user, {"SPEAKER_00": near(voice, 0.05)}), {"SPEAKER_00": "Ana"})
+ sp = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ samples = vp.samples_for(sp, None)
+ assert len(vp.build_variants(samples)) == 2
+ target = near(phone, 0.05)
+ by_variant = vp.score_against(target, vp.build_variants(samples), len(samples))
+ one_average = float(np.dot(target, unit(sum(s.vector for s in samples))))
+ assert by_variant > one_average + 0.1
+
+
+def test_legacy_average_still_matches_and_is_kept_when_samples_start(ctx):
+ user = _user()
+ ana = person()
+ sp = _speaker(user, "Ana", average=ana, count=4)
+ assert [m["name"] for m in vp.find_matches(near(ana, 0.05), user.id, threshold=0.6)] == ["Ana"]
+ _save_names(user, _recording(user, {"SPEAKER_00": near(ana, 0.05)}), {"SPEAKER_00": "Ana"})
+ rows = SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).all()
+ assert sorted(r.source for r in rows) == ["confirmed", "legacy"]
+ legacy = next(r for r in rows if r.source == "legacy")
+ assert legacy.weight == 4.0 and legacy.recording_id is None
+
+
+def test_correcting_a_name_moves_the_sample(ctx):
+ user = _user()
+ voice = person()
+ rec = _recording(user, {"SPEAKER_00": voice})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ # The transcript now shows "Ana"; the correction is made on that name.
+ _save_names(user, db.session.get(Recording, rec.id), {"Ana": "Bea"})
+ rows = SpeakerVoiceSample.query.filter_by(recording_id=rec.id).all()
+ bea = Speaker.query.filter_by(user_id=user.id, name="Bea").first()
+ ana = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ assert [(r.speaker_id, r.label) for r in rows] == [(bea.id, "SPEAKER_00")]
+ assert ana.embedding_count == 0 and ana.average_embedding is None
+ assert db.session.get(Recording, rec.id).speaker_label_map == {"SPEAKER_00": "Bea"}
+
+
+def test_confirming_an_auto_label_upgrades_its_sample(ctx):
+ user = _user(auto_speaker_labelling=True)
+ ana = person()
+ _save_names(user, _recording(user, {"SPEAKER_00": ana}), {"SPEAKER_00": "Ana"})
+ rec = _recording(user, {"SPEAKER_00": near(ana, 0.05)})
+ from src.services.speaker_embedding_matcher import (
+ apply_auto_speaker_labels, apply_speaker_names_to_transcription, update_speaker_profiles_from_recording)
+ mapping = apply_auto_speaker_labels(rec, user)
+ assert mapping == {"SPEAKER_00": "Ana"}
+ apply_speaker_names_to_transcription(rec, mapping)
+ update_speaker_profiles_from_recording(rec, mapping, user)
+ sample = SpeakerVoiceSample.query.filter_by(recording_id=rec.id).one()
+ assert (sample.source, sample.weight) == ("auto", vp.AUTO_SAMPLE_WEIGHT)
+ _save_names(user, db.session.get(Recording, rec.id), {"Ana": "Ana"})
+ sample = SpeakerVoiceSample.query.filter_by(recording_id=rec.id).one()
+ assert (sample.source, sample.weight) == ("confirmed", 1.0)
+
+
+def test_resaving_does_not_double_count(ctx):
+ user = _user()
+ rec = _recording(user, {"SPEAKER_00": person()})
+ for _ in range(3):
+ _save_names(user, db.session.get(Recording, rec.id), {"SPEAKER_00": "Ana"} if _ == 0 else {"Ana": "Ana"})
+ assert SpeakerVoiceSample.query.filter_by(recording_id=rec.id).count() == 1
+
+
+def test_merged_away_label_loses_its_sample(ctx):
+ user = _user()
+ a, b = person(), person()
+ rec = _recording(user, {"SPEAKER_00": a, "SPEAKER_01": b})
+ _save_names(user, rec, {"SPEAKER_00": "Ana", "SPEAKER_01": "Bob"})
+ rec = db.session.get(Recording, rec.id)
+ data = json.loads(rec.transcription)
+ for seg in data:
+ seg["speaker"] = "Ana" # the dialog merged Bob into Ana
+ with patch("src.services.speaker_snippets.create_speaker_snippets", return_value=0):
+ r = _client(user).post(f"/recording/{rec.id}/update_transcript",
+ json={"transcript_data": data, "speaker_map": {}})
+ assert r.status_code == 200
+ labels = {s.label for s in SpeakerVoiceSample.query.filter_by(recording_id=rec.id).all()}
+ assert labels == {"SPEAKER_00"}
+
+
+def test_sample_cap_drops_the_oldest(ctx):
+ user = _user()
+ ana = person()
+ sp = _speaker(user, "Ana")
+ for i in range(vp.MAX_SAMPLES + 3):
+ rec = _recording(user, {"SPEAKER_00": near(ana, 0.05)})
+ vp.record_sample(sp, rec, "SPEAKER_00", near(ana, 0.05))
+ db.session.commit()
+ assert SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).count() == vp.MAX_SAMPLES
+
+
+def test_merge_speakers_keeps_every_sample(ctx):
+ from src.services.speaker_merge import merge_speakers
+ user = _user()
+ a = person()
+ ana = _speaker(user, "Ana", average=a, count=2)
+ ana2 = _speaker(user, "ana (dup)")
+ _save_names(user, _recording(user, {"SPEAKER_00": near(a, 0.05)}), {"SPEAKER_00": "ana (dup)"})
+ merge_speakers(ana.id, [ana2.id], user.id)
+ rows = SpeakerVoiceSample.query.filter_by(speaker_id=ana.id).all()
+ assert sorted(r.source for r in rows) == ["confirmed", "legacy"]
+
+
+def test_clearing_a_profile_deletes_its_samples(ctx):
+ user = _user()
+ rec = _recording(user, {"SPEAKER_00": person()})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ sp = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ assert _client(user).post(f"/speakers/{sp.id}/clear_embeddings").status_code == 200
+ assert SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).count() == 0
+
+
+def test_retranscription_forgets_the_old_labels(ctx):
+ user = _user()
+ rec = _recording(user, {"SPEAKER_00": person()})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ rec = db.session.get(Recording, rec.id)
+ vp.forget_recording_samples(rec)
+ db.session.commit()
+ assert SpeakerVoiceSample.query.filter_by(recording_id=rec.id).count() == 0
+ assert rec.speaker_label_map is None
+
+
+def test_deleting_the_recording_keeps_the_voiceprint(ctx):
+ from src.services.recording_deletion import delete_recording_completely
+ user = _user()
+ rec = _recording(user, {"SPEAKER_00": person()})
+ _save_names(user, rec, {"SPEAKER_00": "Ana"})
+ sp = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ with patch("src.services.storage.get_storage_service"):
+ delete_recording_completely(db.session.get(Recording, rec.id))
+ row = SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).one()
+ assert row.recording_id is None
+
+
+def test_calibration_needs_data_and_lands_between_the_scores(ctx):
+ user = _user()
+ assert vp.calibrated_threshold(None) is None or True # other tests' data may exist
+ people = [person() for _ in range(6)]
+ for i, p in enumerate(people):
+ sp = _speaker(user, f"P{i}")
+ for _ in range(3):
+ rec = _recording(user, {"SPEAKER_00": near(p, 0.1)})
+ vp.record_sample(sp, rec, "SPEAKER_00", near(p, 0.1))
+ db.session.commit()
+ vp._calibration_cache.clear()
+ cal = vp.calibrated_threshold(None)
+ assert cal is not None and 0.45 <= cal <= 0.85
+ assert vp.suggestion_threshold(None) == pytest.approx(cal - 0.10)
+
+
+# ------------------------------------------------------------- stage 2
+
+def _fresh_spaces():
+ SystemSetting.set_setting(vp.SETTING_CURRENT_SPACE, '', setting_type='string')
+ SystemSetting.set_setting(vp.SETTING_LEGACY_SPACE, '', setting_type='string')
+
+
+def test_register_space_recognises_models(ctx):
+ _fresh_spaces()
+ canary_a, canary_b = person(), person()
+ with patch.dict(os.environ, {"DISABLE_VOICE_EMBEDDING_CHECK": "false"}):
+ a = vp.register_space(canary_a)
+ assert vp.legacy_space_id() == a and vp.current_space_id() == a
+ b = vp.register_space(canary_b)
+ assert b != a and vp.current_space_id() == b and vp.legacy_space_id() == a
+ assert vp.register_space(near(canary_a, 0.001)) == a # the same model again
+ VoiceEmbeddingSpace.query.filter(VoiceEmbeddingSpace.id.in_([a, b])).delete(synchronize_session=False)
+ db.session.commit()
+
+
+def test_profiles_only_match_within_their_space(ctx):
+ _fresh_spaces()
+ user = _user()
+ ana = person()
+ with patch.dict(os.environ, {"DISABLE_VOICE_EMBEDDING_CHECK": "false"}):
+ a = vp.register_space(person())
+ # A profile stored before spaces existed: legacy space a.
+ _speaker(user, "Ana", average=ana, count=3)
+ b = vp.register_space(person())
+ in_b = _recording(user, {"SPEAKER_00": near(ana, 0.05)}, space=b)
+ in_a = _recording(user, {"SPEAKER_00": near(ana, 0.05)}, space=a)
+ assert vp.suggestions_for_recording(in_b, user.id)["SPEAKER_00"] == []
+ assert [m["name"] for m in vp.suggestions_for_recording(in_a, user.id)["SPEAKER_00"]] == ["Ana"]
+ # Naming Ana in space b starts her space-b profile, leaving space a alone.
+ _save_names(user, in_b, {"SPEAKER_00": "Ana"})
+ sp = Speaker.query.filter_by(user_id=user.id, name="Ana").first()
+ spaces = sorted(str(r.space_id) for r in SpeakerVoiceSample.query.filter_by(speaker_id=sp.id).all())
+ assert spaces == sorted([str(None), str(b)])
+ VoiceEmbeddingSpace.query.filter(VoiceEmbeddingSpace.id.in_([a, b])).delete(synchronize_session=False)
+ db.session.commit()
+
+
+def test_disabled_check_means_no_space_and_matching_as_before(ctx):
+ user = _user()
+ ana = person()
+ _speaker(user, "Ana", average=ana, count=1)
+ with patch.dict(os.environ, {"DISABLE_VOICE_EMBEDDING_CHECK": "true"}):
+ assert vp.current_space_id() is None
+ rec = _recording(user, {"SPEAKER_00": near(ana, 0.05)})
+ assert [m["name"] for m in vp.suggestions_for_recording(rec, user.id)["SPEAKER_00"]] == ["Ana"]
+
+
+def test_existing_reference_becomes_the_legacy_space(ctx):
+ _fresh_spaces()
+ from src.services import voice_embedding_check as check
+ before = {s.id for s in VoiceEmbeddingSpace.query.all()}
+ VoiceEmbeddingSpace.query.delete()
+ db.session.commit()
+ ref = [float(x) for x in person()]
+ with patch.dict(os.environ, {"DISABLE_VOICE_EMBEDDING_CHECK": "false"}):
+ check._initialize_spaces(vp, {"embedding": ref, "backend_fingerprint": "abc"})
+ (space,) = VoiceEmbeddingSpace.query.all()
+ assert vp.legacy_space_id() == space.id and vp.current_space_id() == space.id
+ # Idempotent: a second start adds nothing.
+ check._initialize_spaces(vp, {"embedding": ref, "backend_fingerprint": "abc"})
+ assert VoiceEmbeddingSpace.query.count() == 1
+ VoiceEmbeddingSpace.query.delete()
+ db.session.commit()
+ assert not before or True