Open source video conferencing app powered by LiveKit. Built with Django and React.
β¨(summary) improve speaker assignment
leo96f97ed2d051
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ec98e532..89b58ddc 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -23,6 +23,7 @@ and this project adheres to### Changed- π§βπ»(agents) use `uv` for package management+- β¨(summary) improve speaker-to-participant assignment### Fixeddiff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.pyindex d5099f1f..6858a49d 100644--- a/src/summary/summary/core/config.py+++ b/src/summary/summary/core/config.py@@ -105,6 +105,9 @@ class Settings(BaseSettings):# Speaker to user assignmentis_resolve_speaker_identities_enabled: bool = True+ resolve_speaker_identities_default_overlap_threshold: float = 0.5+ resolve_speaker_identities_enable_split_on_words: bool = True+ resolve_speaker_identities_max_word_duration: float = 1 # seconds# Webhook-related settingswebhook_max_retries: int = 2diff --git a/src/summary/summary/core/user_assign.py b/src/summary/summary/core/user_assign.pyindex 6aab94e8..84873b8d 100644--- a/src/summary/summary/core/user_assign.py+++ b/src/summary/summary/core/user_assign.py@@ -14,11 +14,11 @@ from dataclasses import dataclass, fieldfrom datetime import datetimefrom typing import Any-logger = logging.getLogger(__name__)+from summary.core.config import get_settings++settings = get_settings()-# Minimum fraction of a speaker's total duration that must overlap with a-# participant's VAD to accept the assignment.-DEFAULT_OVERLAP_THRESHOLD = 0.5+logger = logging.getLogger(__name__)@dataclass@@ -74,7 +74,7 @@ class AssignmentResult:f" ({item['speaker']})" if name_to_speaker_count[name] > 1 else "") # Add suffix only if there are multiple detected speakers per userreturn {**item, "speaker": f"{name}{suffix}"}- return item+ return {**item}def _process_segment(item: dict[str, Any], include_words: bool = False@@ -138,6 +138,78 @@ def _overlap_duration(return overlap+def _format_timelines_debug(+ participant_timelines: dict[str, list[Interval]],+ participant_names: dict[str, str],+ speaker_timelines: dict[str, list[Interval]],+) -> str:+ """Render participant and speaker timelines side-by-side for debugging.++ Each row is the slice between two consecutive interval boundaries+ (drawn from both sides). A filled cell marks an active participant+ (left block) or speaker (right block) during that slice, so vertical+ alignment makes overlap visually obvious.+ """+ participant_ids = sorted(participant_timelines.keys())+ speaker_labels = sorted(speaker_timelines.keys())++ if not participant_ids and not speaker_labels:+ return "(no timelines)"++ boundaries: set[float] = set()+ for intervals in (*participant_timelines.values(), *speaker_timelines.values()):+ for iv in intervals:+ boundaries.add(iv.start)+ boundaries.add(iv.end)+ sorted_boundaries = sorted(boundaries)+ if len(sorted_boundaries) < 2:+ return "(no intervals)"++ p_headers = [participant_names.get(pid, pid) for pid in participant_ids]+ s_headers = list(speaker_labels)+ p_widths = [max(len(h), 3) for h in p_headers]+ s_widths = [max(len(h), 3) for h in s_headers]++ def _active(intervals: list[Interval], lo: float, hi: float) -> bool:+ mid = (lo + hi) / 2+ return any(iv.start <= mid < iv.end for iv in intervals)++ def _cells(+ intervals_list: list[list[Interval]],+ widths: list[int],+ lo: float,+ hi: float,+ ) -> str:+ return " ".join(+ ("β" * w if _active(iv, lo, hi) else "Β·" * w)+ for iv, w in zip(intervals_list, widths, strict=True)+ )++ p_iv_list = [participant_timelines[pid] for pid in participant_ids]+ s_iv_list = [speaker_timelines[sl] for sl in speaker_labels]++ time_col = "[ start β end]"+ p_hdr = (+ " ".join(h.center(w) for h, w in zip(p_headers, p_widths, strict=True))+ or "(none)"+ )+ s_hdr = (+ " ".join(h.center(w) for h, w in zip(s_headers, s_widths, strict=True))+ or "(none)"+ )+ sep = " || "+ lines = [+ f"{time_col} {p_hdr}{sep}{s_hdr}",+ "-" * (len(time_col) + 2 + len(p_hdr) + len(sep) + len(s_hdr)),+ ]+ for lo, hi in zip(sorted_boundaries, sorted_boundaries[1:], strict=False):+ time_str = f"[{lo:8.2f} β {hi:8.2f}]"+ p_row = _cells(p_iv_list, p_widths, lo, hi) or " " * len(p_hdr)+ s_row = _cells(s_iv_list, s_widths, lo, hi) or " " * len(s_hdr)+ lines.append(f"{time_str} {p_row}{sep}{s_row}")+ return "\n".join(lines)++def _build_participant_timelines(metadata: dict[str, Any],recording_start_datetime: datetime,@@ -199,24 +271,50 @@ def _build_participant_timelines(return intervals, participants_info-def _build_speaker_timelines(- transcription: Any,-) -> dict[str, list[Interval]]:+def _build_speaker_timelines(transcription: Any) -> dict[str, list[Interval]]:"""Build interval timelines from WhisperX transcription segments."""intervals: dict[str, list[Interval]] = {}-segments = transcription.segments if hasattr(transcription, "segments") else []+ max_word_duration = settings.resolve_speaker_identities_max_word_duration+for segment in segments:speaker = segment.get("speaker")if speaker is None:continue- intervals.setdefault(speaker, []).append(- Interval(segment["start"], segment["end"])- )++ words = [+ w+ for w in segment.get("words", [])+ if w.get("start") is not None and w.get("end") is not None+ ]+ if not words:+ intervals.setdefault(speaker, []).append(+ Interval(segment["start"], segment["end"])+ )+ continue++ start_time: float | None = segment["start"]+ for word in words:+ if start_time is None:+ start_time = word["start"]+ if not settings.resolve_speaker_identities_enable_split_on_words:+ continue+ if word["end"] - word["start"] > max_word_duration:+ end_time = word["start"] + max_word_duration+ if end_time > start_time:+ intervals.setdefault(speaker, []).append(+ Interval(start_time, end_time)+ )+ start_time = None++ if start_time is not None:+ last = words[-1]+ end_time = min(last["end"], last["start"] + max_word_duration)+ if end_time > start_time:+ intervals.setdefault(speaker, []).append(Interval(start_time, end_time))for speaker, speaker_intervals in intervals.items():intervals[speaker] = _merge_intervals(speaker_intervals)-return intervals@@ -225,7 +323,7 @@ def resolve_speaker_identities(transcription: Any,recording_start_datetime: datetime,recording_end_datetime: datetime,- overlap_threshold: float = DEFAULT_OVERLAP_THRESHOLD,+ overlap_threshold: float = settings.resolve_speaker_identities_default_overlap_threshold, # noqa: E501) -> AssignmentResult:"""Match WhisperX speaker labels to participants.@@ -247,9 +345,14 @@ def resolve_speaker_identities(speaker_timelines = _build_speaker_timelines(transcription)logger.debug(- "Assignment inputs: %d participants, %d speakers",+ "Assignment inputs: %d participants, %d speakers\n%s\n%s\n%s",len(participant_timelines),len(speaker_timelines),+ participant_timelines,+ speaker_timelines,+ _format_timelines_debug(+ participant_timelines, participant_names, speaker_timelines+ ),)result = AssignmentResult()diff --git a/src/summary/tests/unit/test_user_assign.py b/src/summary/tests/unit/test_user_assign.pyindex 9fd36c84..20d54838 100644--- a/src/summary/tests/unit/test_user_assign.py+++ b/src/summary/tests/unit/test_user_assign.py@@ -4,10 +4,12 @@ import mathfrom dataclasses import dataclass, fieldfrom datetime import datetime+from summary.core import user_assignfrom summary.core.user_assign import (AssignmentResult,Interval,SpeakerAssignment,+ _build_speaker_timelines,_merge_intervals,_overlap_duration,_total_duration,@@ -73,12 +75,22 @@ DIARIZATION_SINGLE_SPEAKER = FakeTranscription("end": 3.545,"text": " The stale smell.","speaker": "SPEAKER_00",+ "words": [+ {"word": "The", "start": 1.363, "end": 1.8},+ {"word": "stale", "start": 1.8, "end": 2.7},+ {"word": "smell.", "start": 2.7, "end": 3.545},+ ],},{"start": 4.466,"end": 6.247,"text": "It takes heat.","speaker": "SPEAKER_00",+ "words": [+ {"word": "It", "start": 4.466, "end": 4.7},+ {"word": "takes", "start": 4.7, "end": 5.5},+ {"word": "heat.", "start": 5.5, "end": 6.247},+ ],},],)@@ -165,6 +177,174 @@ class TestTotalDuration:assert math.isclose(_total_duration([]), 0.0)+class TestBuildSpeakerTimelines:+ """Tests for _build_speaker_timelines."""++ def test_segment_without_words_falls_back_to_segment_bounds(self):+ """Segments missing a `words` key use the segment start/end as one interval."""+ transcription = FakeTranscription(+ segments=[{"start": 1.5, "end": 3.5, "speaker": "SPEAKER_00"}],+ )+ result = _build_speaker_timelines(transcription)+ assert result == {"SPEAKER_00": [Interval(1.5, 3.5)]}++ def test_segment_with_only_none_word_timestamps_falls_back(self):+ """If every word has None start/end, fall back to segment bounds."""+ transcription = FakeTranscription(+ segments=[+ {+ "start": 1.0,+ "end": 4.0,+ "speaker": "SPEAKER_00",+ "words": [+ {"word": "hi", "start": None, "end": None},+ {"word": "there", "start": None, "end": None},+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ assert result == {"SPEAKER_00": [Interval(1.0, 4.0)]}++ def test_short_words_only_uses_segment_start_and_last_word_end(self):+ """With no overly long words, the interval runs segment start to end."""+ transcription = FakeTranscription(+ segments=[+ {+ "start": 1.0,+ "end": 5.0,+ "speaker": "SPEAKER_00",+ "words": [+ {"word": "a", "start": 1.0, "end": 1.3},+ {"word": "b", "start": 1.4, "end": 1.7},+ {"word": "c", "start": 1.8, "end": 2.1},+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ # Tail: min(2.1, 1.8 + 1.0) = 2.1+ assert result == {"SPEAKER_00": [Interval(1.0, 2.1)]}++ def test_long_word_caps_interval_at_max_duration(self):+ """A word longer than the max-word-duration cap truncates the segment."""+ max_word_duration = (+ user_assign.settings.resolve_speaker_identities_max_word_duration+ )+ transcription = FakeTranscription(+ segments=[+ {+ "start": 0.0,+ "end": max_word_duration + 7,+ "speaker": "SPEAKER_00",+ "words": [+ {+ "word": "pause",+ "start": 0.0,+ "end": max_word_duration + 7,+ },+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ assert result == {"SPEAKER_00": [Interval(0.0, max_word_duration)]}++ def test_long_word_in_middle_splits_segment(self):+ """Short words around a long word produce two intervals (before-cap + after)."""+ transcription = FakeTranscription(+ segments=[+ {+ "start": 0.0,+ "end": 20.0,+ "speaker": "SPEAKER_00",+ "words": [+ {"word": "a", "start": 0.0, "end": 0.5},+ {"word": "long", "start": 1.0, "end": 15.0},+ {"word": "z", "start": 18.0, "end": 18.4},+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ # First emit: (0.0, 1.0 + 1.0). Then start_time resets, picks up at "z" (18.0).+ # Tail: min(18.4, 18.0 + 1.0) = 18.4. So second interval is (18.0, 18.4).+ assert result == {+ "SPEAKER_00": [Interval(0.0, 2.0), Interval(18.0, 18.4)],+ }++ def test_tail_word_is_capped_at_max_duration(self):+ """The trailing word's end is capped at word.start + max_word_duration."""+ transcription = FakeTranscription(+ segments=[+ {+ "start": 0.0,+ "end": 50.0,+ "speaker": "SPEAKER_00",+ "words": [+ {"word": "a", "start": 0.0, "end": 0.4},+ # Last word ends inside the cap, so the cap doesn't apply.+ {"word": "b", "start": 1.0, "end": 1.5},+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ # Tail: min(1.5, 1.0 + 1.0) = 1.5+ assert result == {"SPEAKER_00": [Interval(0.0, 1.5)]}++ def test_split_on_words_disabled_keeps_segment_as_one_interval(self, monkeypatch):+ """With splitting disabled, long words don't split the interval."""+ monkeypatch.setattr(+ user_assign,+ "settings",+ user_assign.settings.model_copy(+ update={"resolve_speaker_identities_enable_split_on_words": False},+ ),+ )+ transcription = FakeTranscription(+ segments=[+ {+ "start": 0.0,+ "end": 20.0,+ "speaker": "SPEAKER_00",+ "words": [+ {"word": "a", "start": 0.0, "end": 0.5},+ {"word": "long", "start": 1.0, "end": 15.0},+ {"word": "z", "start": 18.0, "end": 18.4},+ ],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ # No mid-segment split; tail caps at min(18.4, 18.0 + 1.0) = 18.4.+ assert result == {"SPEAKER_00": [Interval(0.0, 18.4)]}++ def test_multiple_speakers_keep_separate_timelines(self):+ """Segments from different speakers populate independent timeline entries."""+ transcription = FakeTranscription(+ segments=[+ {+ "start": 0.0,+ "end": 1.0,+ "speaker": "SPEAKER_00",+ "words": [{"word": "hi", "start": 0.0, "end": 0.5}],+ },+ {+ "start": 2.0,+ "end": 3.0,+ "speaker": "SPEAKER_01",+ "words": [{"word": "yo", "start": 2.0, "end": 2.5}],+ },+ ],+ )+ result = _build_speaker_timelines(transcription)+ assert result == {+ "SPEAKER_00": [Interval(0.0, 0.5)],+ "SPEAKER_01": [Interval(2.0, 2.5)],+ }++class TestResolveSpeakerIdentities:"""Tests for resolve_speaker_identities."""