Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
15.8 KB
Raw Blame
"""
Shared speaker identification service.

Provides LLM-based speaker identification from transcript context,
used by both the web UI (recordings.py) and REST API (api_v1.py).
"""

import os
import re
import json
from flask import current_app


def identify_speakers_from_transcript(transcription_data, user_id, candidate_names=None):
    """
    Identify speakers in a transcription using an LLM.

    Args:
        transcription_data: List of transcript segments (already parsed JSON).
        user_id: Current user's ID (for token tracking).
        candidate_names: Optional list of the user's saved speaker-profile names.
            When provided (contextual auto-labelling for connectors that return
            no voice embeddings), the model may only assign these exact names or
            an empty string, and the prompt switches to a prefix-cache-friendly
            transcript-first layout with an admin-editable guidance suffix.

    Returns:
        dict mapping original speaker labels to identified names.
        Values are empty string "" for unidentified speakers.

    Raises:
        ValueError: If LLM API key is not configured.
        Exception: On LLM call failure.
    """
    from src.services.llm import call_llm_completion
    from src.utils import safe_json_loads
    from src.models import SystemSetting

    # Extract unique speakers in order of appearance
    seen_speakers = set()
    unique_speakers = []
    for segment in transcription_data:
        speaker = segment.get('speaker')
        if speaker and speaker not in seen_speakers:
            seen_speakers.add(speaker)
            unique_speakers.append(speaker)

    if not unique_speakers:
        return {}

    # Normalize all labels to SPEAKER_XX format for the LLM
    speaker_to_label = {}
    for idx, speaker in enumerate(unique_speakers):
        speaker_to_label[speaker] = f'SPEAKER_{str(idx).zfill(2)}'

    # Create temporary transcript with normalized labels
    formatted_lines = []
    for segment in transcription_data:
        original_speaker = segment.get('speaker')
        label = speaker_to_label.get(original_speaker, 'Unknown Speaker')
        sentence = segment.get('sentence', '')
        formatted_lines.append(f"[{label}]: {sentence}")
    formatted_transcription = "\n".join(formatted_lines)

    speaker_labels = list(speaker_to_label.values())

    current_app.logger.info(f"[Auto-Identify] Formatted transcript (first 500 chars): {formatted_transcription[:500]}")
    current_app.logger.info(f"[Auto-Identify] Speaker labels: {speaker_labels}")

    # Apply configurable transcript length limit
    transcript_limit = SystemSetting.get_setting('transcript_length_limit', 30000)
    if transcript_limit == -1:
        transcript_text = formatted_transcription
    else:
        transcript_text = formatted_transcription[:transcript_limit]

    # Optional candidate constraint: when the caller supplies the user's saved
    # speaker profiles (contextual auto-labelling for connectors that return no
    # voice embeddings), the model may only assign those exact names. The prompt
    # is laid out transcript-first with the editable guidance + candidate list as
    # the trailing suffix, so it stays prefix-cache friendly and only the suffix
    # is admin-editable (see DEFAULT_CONTEXTUAL_SPEAKER_PROMPT).
    candidates = [str(name).strip() for name in (candidate_names or []) if str(name).strip()]

    if candidates:
        from src.config.prompts import DEFAULT_CONTEXTUAL_SPEAKER_PROMPT
        try:
            from src.tasks.processing import (
                PREFIX_CACHE_OPTIMIZED_PROMPTS, _SHARED_LLM_SYSTEM_MSG, _shared_user_prefix,
            )
        except Exception:
            PREFIX_CACHE_OPTIMIZED_PROMPTS = False
            _SHARED_LLM_SYSTEM_MSG = None
            _shared_user_prefix = None

        guidance = SystemSetting.get_setting(
            'admin_default_contextual_speaker_prompt', DEFAULT_CONTEXTUAL_SPEAKER_PROMPT
        ) or DEFAULT_CONTEXTUAL_SPEAKER_PROMPT

        # Everything after the transcript. The concrete candidate list, the
        # speaker labels, and the strict JSON contract are code-controlled so an
        # admin edit to `guidance` cannot break the output format or the
        # exact-name constraint.
        suffix = (
            f"{guidance}\n\n"
            f"Known speaker profiles you may assign: {', '.join(candidates)}\n"
            f"Speakers to identify: {', '.join(speaker_labels)}\n\n"
            "Respond with a single JSON object whose keys are the speaker labels and whose "
            "values are the assigned profile name (copied exactly from the list above) or an "
            'empty string "" when no profile clearly matches.\n\n'
            "Example format:\n"
            "{\n"
            '  "SPEAKER_01": "Jane Smith",\n'
            '  "SPEAKER_03": ""\n'
            "}\n\n"
            "JSON Response:\n"
        )
        if PREFIX_CACHE_OPTIMIZED_PROMPTS and _shared_user_prefix:
            # Reuse the shared transcript-first prefix so the layout matches the
            # title/summary calls when prefix caching is enabled.
            system_msg = _SHARED_LLM_SYSTEM_MSG
            prompt = _shared_user_prefix(transcript_text) + suffix
        else:
            system_msg = (
                "You assign speakers in a transcript to a fixed list of known people, "
                "using only contextual clues in the dialogue. Your response must be a "
                "single, valid JSON object and must only use names from the provided list."
            )
            prompt = f'Transcript:\n"""\n{transcript_text}\n"""\n\n' + suffix
    else:
        prompt = f"""Analyze the following conversation transcript and identify the names of the speakers based on the context and content of their dialogue.

The speakers that need to be identified are: {', '.join(speaker_labels)}

Look for clues in the conversation such as:
- Names mentioned by other speakers when addressing someone
- Self-introductions or references to their own name
- Context clues about roles, relationships, or positions
- Any direct mentions of names in the dialogue

Here is the complete conversation transcript:

{transcript_text}

Based on the conversation above, identify the most likely real names for the speakers. Pay close attention to how speakers address each other and any names that are mentioned in the dialogue.

Respond with a single JSON object where keys are the speaker labels (e.g., "SPEAKER_01") and values are the identified full names. If a name cannot be determined from the conversation context, use an empty string "".

Example format:
{{
  "SPEAKER_01": "Jane Smith",
  "SPEAKER_03": "Bob Johnson",
  "SPEAKER_05": ""
}}

JSON Response:
"""
        system_msg = (
            "You are an expert in analyzing conversation transcripts to identify speakers "
            "based on contextual clues in the dialogue. Analyze the conversation carefully "
            "to find names mentioned when speakers address each other or introduce themselves. "
            "Your response must be a single, valid JSON object containing only the requested "
            "speaker identifications."
        )

    current_app.logger.info("[Auto-Identify] Calling LLM")

    use_schema = os.environ.get('AUTO_IDENTIFY_RESPONSE_SCHEMA', '').strip() in ('1', 'true', 'yes')

    response_content = None
    if use_schema:
        # Build JSON schema response format with constrained keys
        schema_properties = {label: {"type": "string"} for label in speaker_labels}
        schema_response_format = {
            "type": "json_schema",
            "json_schema": {
                "name": "speaker_identification",
                "strict": True,
                "schema": {
                    "type": "object",
                    "properties": schema_properties,
                    "required": speaker_labels,
                    "additionalProperties": False
                }
            }
        }
        schema_prompt = prompt + f"\n\nIMPORTANT: Your JSON response must contain exactly these keys: {', '.join(speaker_labels)}"
        try:
            current_app.logger.info("[Auto-Identify] Trying json_schema response format")
            completion = call_llm_completion(
                messages=[
                    {"role": "system", "content": system_msg},
                    {"role": "user", "content": schema_prompt}
                ],
                temperature=0.2,
                response_format=schema_response_format,
                user_id=user_id,
                operation_type='speaker_identification'
            )
            response_content = completion.choices[0].message.content
            current_app.logger.info(f"[Auto-Identify] LLM Raw Response (schema mode): {response_content}")
        except Exception as schema_err:
            current_app.logger.warning(f"[Auto-Identify] json_schema mode failed, falling back to json_object: {schema_err}")
            response_content = None

    if response_content is None:
        completion = call_llm_completion(
            messages=[
                {"role": "system", "content": system_msg},
                {"role": "user", "content": prompt}
            ],
            temperature=0.2,
            response_format={"type": "json_object"},
            user_id=user_id,
            operation_type='speaker_identification'
        )
        response_content = completion.choices[0].message.content
        current_app.logger.info(f"[Auto-Identify] LLM Raw Response: {response_content}")

    identified_map = safe_json_loads(response_content, {})
    current_app.logger.info(f"[Auto-Identify] Parsed identified_map: {identified_map}")

    # --- Sanitize identified_map ---
    identified_map = _sanitize_identified_map(identified_map, speaker_labels, candidate_names=candidates)
    current_app.logger.info(f"[Auto-Identify] Sanitized identified_map: {identified_map}")

    # Map back to original speaker labels
    final_speaker_map = {}
    for original_speaker, temp_label in speaker_to_label.items():
        if temp_label in identified_map:
            final_speaker_map[original_speaker] = identified_map[temp_label]

    current_app.logger.info(f"[Auto-Identify] Final speaker_map: {final_speaker_map}")
    return final_speaker_map


def _sanitize_identified_map(identified_map, speaker_labels, candidate_names=None):
    """
    Clean up LLM output: handle inverted maps, strip commentary,
    clear placeholders, etc.

    When candidate_names is given (contextual auto-labelling), the result is
    constrained to those exact saved names (case-insensitive match) or an empty
    string — the model cannot introduce a name that isn't a saved profile, and
    the generic name-cleanup below is skipped so punctuated saved names like
    "Smith, John" survive intact.
    """
    speaker_label_re = re.compile(r'^SPEAKER_\d{2}$')

    # Detect inverted map ({name: "SPEAKER_XX"}) and flip it
    if identified_map and all(
        speaker_label_re.match(str(v)) for v in identified_map.values() if v
    ) and not any(speaker_label_re.match(str(k)) for k in identified_map.keys()):
        current_app.logger.warning("[Auto-Identify] Detected inverted map, flipping keys/values")
        identified_map = {v: k for k, v in identified_map.items() if v}

    allowed = {
        str(name).strip().casefold(): str(name).strip()
        for name in (candidate_names or [])
        if str(name).strip()
    }

    sanitized = {}
    for speaker_label, identified_name in identified_map.items():
        # Skip entries whose key isn't a valid SPEAKER_XX label
        if not speaker_label_re.match(str(speaker_label)):
            continue
        if not identified_name or not isinstance(identified_name, str):
            sanitized[speaker_label] = ""
            continue

        name = identified_name.strip()

        # Contextual mode: accept only an exact saved-profile name (preserving
        # its original casing/punctuation), otherwise leave the speaker blank.
        if allowed:
            sanitized[speaker_label] = allowed.get(name.casefold(), "")
            continue

        # Clear generic placeholders
        if name.lower() in ["unknown", "n/a", "not available", "unclear", "unidentified", ""]:
            sanitized[speaker_label] = ""
            continue

        # Clear label-to-label entries (e.g. "SPEAKER_01": "SPEAKER_02")
        if speaker_label_re.match(name):
            sanitized[speaker_label] = ""
            continue

        # Strip parenthetical content: "John (the host)" -> "John"
        name = re.sub(r'\s*\([^)]*\)', '', name).strip()

        # Take first name segment before comma, semicolon, or slash
        name = re.split(r'[,;/]', name)[0].strip()

        # Collapse whitespace
        name = re.sub(r'\s+', ' ', name)

        # Final check: if result still matches SPEAKER_XX, clear it
        if speaker_label_re.match(name) or not name:
            sanitized[speaker_label] = ""
            continue

        sanitized[speaker_label] = name

    return sanitized


def apply_contextual_auto_labels(recording, user):
    """Opt-in contextual speaker labelling for recordings without embeddings.

    Called from the transcription pipeline only when the recording has no voice
    embeddings (so the embedding matcher cannot run) and the user has enabled
    auto speaker labelling. Asks the LLM to map the diarized speaker labels to
    the user's saved speaker profiles, constrained to those exact names, applies
    the accepted matches to the transcription, and creates representative
    snippets for the matched speakers. Returns the applied {SPEAKER_XX: name}
    map (empty dict when nothing was applied). Never raises: any failure is
    logged and swallowed so it cannot fail the transcription.
    """
    if not user.auto_speaker_labelling or not recording.transcription:
        return {}

    from src.models import Speaker
    from src.services.speaker_embedding_matcher import apply_speaker_names_to_transcription
    from src.services.speaker_snippets import create_speaker_snippets

    candidates = [
        speaker.name
        for speaker in Speaker.query.filter_by(user_id=user.id).order_by(Speaker.name.asc()).all()
        if speaker.name
    ]
    if not candidates:
        current_app.logger.info(
            "[Auto-Identify] User %s has no saved speaker profiles; skipping contextual labelling",
            user.id,
        )
        return {}

    try:
        transcription_data = json.loads(recording.transcription)
    except (json.JSONDecodeError, TypeError):
        current_app.logger.warning(
            "[Auto-Identify] Recording %s transcription is not parseable JSON; skipping", recording.id
        )
        return {}
    if not isinstance(transcription_data, list):
        return {}

    try:
        speaker_map = identify_speakers_from_transcript(
            transcription_data, user.id, candidate_names=candidates
        )
    except Exception as error:
        current_app.logger.warning(
            "[Auto-Identify] Contextual identification failed for recording %s: %s", recording.id, error
        )
        return {}

    speaker_map = {label: name for label, name in speaker_map.items() if name}
    if not speaker_map:
        current_app.logger.info(
            "[Auto-Identify] No saved speaker matched recording %s contextually", recording.id
        )
        return {}

    if not apply_speaker_names_to_transcription(recording, speaker_map):
        return {}

    try:
        snippet_map = {label: {"name": name, "isMe": False} for label, name in speaker_map.items()}
        create_speaker_snippets(recording.id, snippet_map)
    except Exception as error:
        # Snippets are a convenience; a failure here must not undo the labelling.
        current_app.logger.warning(
            "[Auto-Identify] Could not create snippets for recording %s: %s", recording.id, error
        )

    current_app.logger.info(
        "[Auto-Identify] Applied contextual speaker matches to recording %s: %s", recording.id, speaker_map
    )
    return speaker_map