Skip to content

armgit / meet

Public Imported from GitHub
0

Open source video conferencing app powered by LiveKit. Built with Django and React.

✨(summary) improve speaker assignment

leo96f97ed2d051
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec98e532..89b58ddc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,7 @@ and this project adheres to
### Changed
- πŸ§‘β€πŸ’»(agents) use `uv` for package management
+- ✨(summary) improve speaker-to-participant assignment
### Fixed
diff --git a/src/summary/summary/core/config.py b/src/summary/summary/core/config.py
index d5099f1f..6858a49d 100644
--- a/src/summary/summary/core/config.py
+++ b/src/summary/summary/core/config.py
@@ -105,6 +105,9 @@ class Settings(BaseSettings):
# Speaker to user assignment
is_resolve_speaker_identities_enabled: bool = True
+ resolve_speaker_identities_default_overlap_threshold: float = 0.5
+ resolve_speaker_identities_enable_split_on_words: bool = True
+ resolve_speaker_identities_max_word_duration: float = 1 # seconds
# Webhook-related settings
webhook_max_retries: int = 2
diff --git a/src/summary/summary/core/user_assign.py b/src/summary/summary/core/user_assign.py
index 6aab94e8..84873b8d 100644
--- a/src/summary/summary/core/user_assign.py
+++ b/src/summary/summary/core/user_assign.py
@@ -14,11 +14,11 @@ from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
-logger = logging.getLogger(__name__)
+from summary.core.config import get_settings
+
+settings = get_settings()
-# Minimum fraction of a speaker's total duration that must overlap with a
-# participant's VAD to accept the assignment.
-DEFAULT_OVERLAP_THRESHOLD = 0.5
+logger = logging.getLogger(__name__)
@dataclass
@@ -74,7 +74,7 @@ class AssignmentResult:
f" ({item['speaker']})" if name_to_speaker_count[name] > 1 else ""
) # Add suffix only if there are multiple detected speakers per user
return {**item, "speaker": f"{name}{suffix}"}
- return item
+ return {**item}
def _process_segment(
item: dict[str, Any], include_words: bool = False
@@ -138,6 +138,78 @@ def _overlap_duration(
return overlap
+def _format_timelines_debug(
+ participant_timelines: dict[str, list[Interval]],
+ participant_names: dict[str, str],
+ speaker_timelines: dict[str, list[Interval]],
+) -> str:
+ """Render participant and speaker timelines side-by-side for debugging.
+
+ Each row is the slice between two consecutive interval boundaries
+ (drawn from both sides). A filled cell marks an active participant
+ (left block) or speaker (right block) during that slice, so vertical
+ alignment makes overlap visually obvious.
+ """
+ participant_ids = sorted(participant_timelines.keys())
+ speaker_labels = sorted(speaker_timelines.keys())
+
+ if not participant_ids and not speaker_labels:
+ return "(no timelines)"
+
+ boundaries: set[float] = set()
+ for intervals in (*participant_timelines.values(), *speaker_timelines.values()):
+ for iv in intervals:
+ boundaries.add(iv.start)
+ boundaries.add(iv.end)
+ sorted_boundaries = sorted(boundaries)
+ if len(sorted_boundaries) < 2:
+ return "(no intervals)"
+
+ p_headers = [participant_names.get(pid, pid) for pid in participant_ids]
+ s_headers = list(speaker_labels)
+ p_widths = [max(len(h), 3) for h in p_headers]
+ s_widths = [max(len(h), 3) for h in s_headers]
+
+ def _active(intervals: list[Interval], lo: float, hi: float) -> bool:
+ mid = (lo + hi) / 2
+ return any(iv.start <= mid < iv.end for iv in intervals)
+
+ def _cells(
+ intervals_list: list[list[Interval]],
+ widths: list[int],
+ lo: float,
+ hi: float,
+ ) -> str:
+ return " ".join(
+ ("β–ˆ" * w if _active(iv, lo, hi) else "Β·" * w)
+ for iv, w in zip(intervals_list, widths, strict=True)
+ )
+
+ p_iv_list = [participant_timelines[pid] for pid in participant_ids]
+ s_iv_list = [speaker_timelines[sl] for sl in speaker_labels]
+
+ time_col = "[ start β†’ end]"
+ p_hdr = (
+ " ".join(h.center(w) for h, w in zip(p_headers, p_widths, strict=True))
+ or "(none)"
+ )
+ s_hdr = (
+ " ".join(h.center(w) for h, w in zip(s_headers, s_widths, strict=True))
+ or "(none)"
+ )
+ sep = " || "
+ lines = [
+ f"{time_col} {p_hdr}{sep}{s_hdr}",
+ "-" * (len(time_col) + 2 + len(p_hdr) + len(sep) + len(s_hdr)),
+ ]
+ for lo, hi in zip(sorted_boundaries, sorted_boundaries[1:], strict=False):
+ time_str = f"[{lo:8.2f} β†’ {hi:8.2f}]"
+ p_row = _cells(p_iv_list, p_widths, lo, hi) or " " * len(p_hdr)
+ s_row = _cells(s_iv_list, s_widths, lo, hi) or " " * len(s_hdr)
+ lines.append(f"{time_str} {p_row}{sep}{s_row}")
+ return "\n".join(lines)
+
+
def _build_participant_timelines(
metadata: dict[str, Any],
recording_start_datetime: datetime,
@@ -199,24 +271,50 @@ def _build_participant_timelines(
return intervals, participants_info
-def _build_speaker_timelines(
- transcription: Any,
-) -> dict[str, list[Interval]]:
+def _build_speaker_timelines(transcription: Any) -> dict[str, list[Interval]]:
"""Build interval timelines from WhisperX transcription segments."""
intervals: dict[str, list[Interval]] = {}
-
segments = transcription.segments if hasattr(transcription, "segments") else []
+ max_word_duration = settings.resolve_speaker_identities_max_word_duration
+
for segment in segments:
speaker = segment.get("speaker")
if speaker is None:
continue
- intervals.setdefault(speaker, []).append(
- Interval(segment["start"], segment["end"])
- )
+
+ words = [
+ w
+ for w in segment.get("words", [])
+ if w.get("start") is not None and w.get("end") is not None
+ ]
+ if not words:
+ intervals.setdefault(speaker, []).append(
+ Interval(segment["start"], segment["end"])
+ )
+ continue
+
+ start_time: float | None = segment["start"]
+ for word in words:
+ if start_time is None:
+ start_time = word["start"]
+ if not settings.resolve_speaker_identities_enable_split_on_words:
+ continue
+ if word["end"] - word["start"] > max_word_duration:
+ end_time = word["start"] + max_word_duration
+ if end_time > start_time:
+ intervals.setdefault(speaker, []).append(
+ Interval(start_time, end_time)
+ )
+ start_time = None
+
+ if start_time is not None:
+ last = words[-1]
+ end_time = min(last["end"], last["start"] + max_word_duration)
+ if end_time > start_time:
+ intervals.setdefault(speaker, []).append(Interval(start_time, end_time))
for speaker, speaker_intervals in intervals.items():
intervals[speaker] = _merge_intervals(speaker_intervals)
-
return intervals
@@ -225,7 +323,7 @@ def resolve_speaker_identities(
transcription: Any,
recording_start_datetime: datetime,
recording_end_datetime: datetime,
- overlap_threshold: float = DEFAULT_OVERLAP_THRESHOLD,
+ overlap_threshold: float = settings.resolve_speaker_identities_default_overlap_threshold, # noqa: E501
) -> AssignmentResult:
"""Match WhisperX speaker labels to participants.
@@ -247,9 +345,14 @@ def resolve_speaker_identities(
speaker_timelines = _build_speaker_timelines(transcription)
logger.debug(
- "Assignment inputs: %d participants, %d speakers",
+ "Assignment inputs: %d participants, %d speakers\n%s\n%s\n%s",
len(participant_timelines),
len(speaker_timelines),
+ participant_timelines,
+ speaker_timelines,
+ _format_timelines_debug(
+ participant_timelines, participant_names, speaker_timelines
+ ),
)
result = AssignmentResult()
diff --git a/src/summary/tests/unit/test_user_assign.py b/src/summary/tests/unit/test_user_assign.py
index 9fd36c84..20d54838 100644
--- a/src/summary/tests/unit/test_user_assign.py
+++ b/src/summary/tests/unit/test_user_assign.py
@@ -4,10 +4,12 @@ import math
from dataclasses import dataclass, field
from datetime import datetime
+from summary.core import user_assign
from summary.core.user_assign import (
AssignmentResult,
Interval,
SpeakerAssignment,
+ _build_speaker_timelines,
_merge_intervals,
_overlap_duration,
_total_duration,
@@ -73,12 +75,22 @@ DIARIZATION_SINGLE_SPEAKER = FakeTranscription(
"end": 3.545,
"text": " The stale smell.",
"speaker": "SPEAKER_00",
+ "words": [
+ {"word": "The", "start": 1.363, "end": 1.8},
+ {"word": "stale", "start": 1.8, "end": 2.7},
+ {"word": "smell.", "start": 2.7, "end": 3.545},
+ ],
},
{
"start": 4.466,
"end": 6.247,
"text": "It takes heat.",
"speaker": "SPEAKER_00",
+ "words": [
+ {"word": "It", "start": 4.466, "end": 4.7},
+ {"word": "takes", "start": 4.7, "end": 5.5},
+ {"word": "heat.", "start": 5.5, "end": 6.247},
+ ],
},
],
)
@@ -165,6 +177,174 @@ class TestTotalDuration:
assert math.isclose(_total_duration([]), 0.0)
+class TestBuildSpeakerTimelines:
+ """Tests for _build_speaker_timelines."""
+
+ def test_segment_without_words_falls_back_to_segment_bounds(self):
+ """Segments missing a `words` key use the segment start/end as one interval."""
+ transcription = FakeTranscription(
+ segments=[{"start": 1.5, "end": 3.5, "speaker": "SPEAKER_00"}],
+ )
+ result = _build_speaker_timelines(transcription)
+ assert result == {"SPEAKER_00": [Interval(1.5, 3.5)]}
+
+ def test_segment_with_only_none_word_timestamps_falls_back(self):
+ """If every word has None start/end, fall back to segment bounds."""
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 1.0,
+ "end": 4.0,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {"word": "hi", "start": None, "end": None},
+ {"word": "there", "start": None, "end": None},
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ assert result == {"SPEAKER_00": [Interval(1.0, 4.0)]}
+
+ def test_short_words_only_uses_segment_start_and_last_word_end(self):
+ """With no overly long words, the interval runs segment start to end."""
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 1.0,
+ "end": 5.0,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {"word": "a", "start": 1.0, "end": 1.3},
+ {"word": "b", "start": 1.4, "end": 1.7},
+ {"word": "c", "start": 1.8, "end": 2.1},
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ # Tail: min(2.1, 1.8 + 1.0) = 2.1
+ assert result == {"SPEAKER_00": [Interval(1.0, 2.1)]}
+
+ def test_long_word_caps_interval_at_max_duration(self):
+ """A word longer than the max-word-duration cap truncates the segment."""
+ max_word_duration = (
+ user_assign.settings.resolve_speaker_identities_max_word_duration
+ )
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 0.0,
+ "end": max_word_duration + 7,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {
+ "word": "pause",
+ "start": 0.0,
+ "end": max_word_duration + 7,
+ },
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ assert result == {"SPEAKER_00": [Interval(0.0, max_word_duration)]}
+
+ def test_long_word_in_middle_splits_segment(self):
+ """Short words around a long word produce two intervals (before-cap + after)."""
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 0.0,
+ "end": 20.0,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {"word": "a", "start": 0.0, "end": 0.5},
+ {"word": "long", "start": 1.0, "end": 15.0},
+ {"word": "z", "start": 18.0, "end": 18.4},
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ # First emit: (0.0, 1.0 + 1.0). Then start_time resets, picks up at "z" (18.0).
+ # Tail: min(18.4, 18.0 + 1.0) = 18.4. So second interval is (18.0, 18.4).
+ assert result == {
+ "SPEAKER_00": [Interval(0.0, 2.0), Interval(18.0, 18.4)],
+ }
+
+ def test_tail_word_is_capped_at_max_duration(self):
+ """The trailing word's end is capped at word.start + max_word_duration."""
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 0.0,
+ "end": 50.0,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {"word": "a", "start": 0.0, "end": 0.4},
+ # Last word ends inside the cap, so the cap doesn't apply.
+ {"word": "b", "start": 1.0, "end": 1.5},
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ # Tail: min(1.5, 1.0 + 1.0) = 1.5
+ assert result == {"SPEAKER_00": [Interval(0.0, 1.5)]}
+
+ def test_split_on_words_disabled_keeps_segment_as_one_interval(self, monkeypatch):
+ """With splitting disabled, long words don't split the interval."""
+ monkeypatch.setattr(
+ user_assign,
+ "settings",
+ user_assign.settings.model_copy(
+ update={"resolve_speaker_identities_enable_split_on_words": False},
+ ),
+ )
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 0.0,
+ "end": 20.0,
+ "speaker": "SPEAKER_00",
+ "words": [
+ {"word": "a", "start": 0.0, "end": 0.5},
+ {"word": "long", "start": 1.0, "end": 15.0},
+ {"word": "z", "start": 18.0, "end": 18.4},
+ ],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ # No mid-segment split; tail caps at min(18.4, 18.0 + 1.0) = 18.4.
+ assert result == {"SPEAKER_00": [Interval(0.0, 18.4)]}
+
+ def test_multiple_speakers_keep_separate_timelines(self):
+ """Segments from different speakers populate independent timeline entries."""
+ transcription = FakeTranscription(
+ segments=[
+ {
+ "start": 0.0,
+ "end": 1.0,
+ "speaker": "SPEAKER_00",
+ "words": [{"word": "hi", "start": 0.0, "end": 0.5}],
+ },
+ {
+ "start": 2.0,
+ "end": 3.0,
+ "speaker": "SPEAKER_01",
+ "words": [{"word": "yo", "start": 2.0, "end": 2.5}],
+ },
+ ],
+ )
+ result = _build_speaker_timelines(transcription)
+ assert result == {
+ "SPEAKER_00": [Interval(0.0, 0.5)],
+ "SPEAKER_01": [Interval(2.0, 2.5)],
+ }
+
+
class TestResolveSpeakerIdentities:
"""Tests for resolve_speaker_identities."""