Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: bound realtime websocket state for long sessions (#3214)
zhifu gaoa53497b7ae8d
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.pyindex f6e40cce5..58355b3c0 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py@@ -10,6 +10,7 @@ Features:"""import asyncio+from collections import dequeimport jsonimport loggingimport os@@ -77,21 +78,33 @@ from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVADclass HybridSpeakerTracker:"""Speaker diarization: streaming ClusterBackend + final re-clustering."""- def __init__(self, spk_model, device, threshold=0.6):+ def __init__(+ self,+ spk_model,+ device,+ threshold=0.6,+ max_history_chunks=128,+ max_speakers=15,+ ):+ if max_history_chunks <= 0:+ raise ValueError("max_history_chunks must be positive")+ if max_speakers <= 0:+ raise ValueError("max_speakers must be positive")self.spk_model = spk_modelself.device = deviceself.threshold = threshold+ self.max_speakers = max_speakersself.speaker_centers = []+ self.speaker_center_updates = []from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spkfrom funasr.models.campplus.cluster_backend import ClusterBackendself.sv_chunk = sv_chunkself.postprocess = postprocessself.distribute_spk = distribute_spkself.cluster_backend = ClusterBackend(merge_thr=0.78).to(device)- self.all_chunks = []- self.all_embeddings = []- self.display_map = {}- self.next_display_id = 0+ self.all_chunks = deque(maxlen=max_history_chunks)+ self.all_embeddings = deque(maxlen=max_history_chunks)+ self.last_speaker_id = 0@torch.no_grad()def assign_streaming(self, audio_samples, seg_start_s, seg_end_s, sentence):@@ -99,31 +112,89 @@ class HybridSpeakerTracker:vad_seg = [[seg_start_s, seg_end_s, audio_samples]]chunks = self.sv_chunk(vad_seg)if not chunks:- sentence["spk"] = self.next_display_id- self.next_display_id += 1+ sentence["spk"] = self.last_speaker_idreturn- self.all_chunks.extend(chunks)speech_list = [ch[2] for ch in chunks]spk_res = self.spk_model.generate(input=speech_list, cache={}, is_final=True)- embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)- self.all_embeddings.append(embs)-- all_embs = torch.cat(self.all_embeddings, dim=0)- labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)- if not isinstance(labels, np.ndarray):- labels = np.array(labels)-- all_sorted = sorted(self.all_chunks, key=lambda x: x[0])- sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())+ embeddings = torch.cat([r["spk_embedding"] for r in spk_res], dim=0).detach().cpu()+ for chunk, embedding in zip(chunks, embeddings):+ # Speaker post-processing only needs timestamps after embedding extraction.+ # Do not retain NumPy views into the session audio buffer.+ self.all_chunks.append((float(chunk[0]), float(chunk[1])))+ self.all_embeddings.append(embedding.clone())++ sv_output = self._cluster_recent(update_centers=True)temp = [{"start": int(seg_start_s*1000), "end": int(seg_end_s*1000), "text": sentence["text"]}]self.distribute_spk(temp, sv_output)- raw_spk = temp[0].get("spk", 0)+ sentence["spk"] = temp[0].get("spk", self.last_speaker_id)+ self.last_speaker_id = sentence["spk"]- if raw_spk not in self.display_map:- self.display_map[raw_spk] = self.next_display_id- self.next_display_id += 1- sentence["spk"] = self.display_map[raw_spk]+ def _cluster_recent(self, update_centers):+ all_embeddings = torch.stack(list(self.all_embeddings), dim=0)+ labels = self.cluster_backend(all_embeddings, oracle_num=None)+ if not isinstance(labels, np.ndarray):+ labels = np.asarray(labels)++ chunks = [[start, end, None] for start, end in self.all_chunks]+ sv_output, cluster_centers = self.postprocess(+ chunks,+ None,+ labels,+ all_embeddings,+ return_spk_center=True,+ )+ stable_ids = self._map_cluster_centers(cluster_centers, update=update_centers)+ return [[start, end, stable_ids[int(spk)]] for start, end, spk in sv_output]++ def _map_cluster_centers(self, cluster_centers, update):+ centers = torch.as_tensor(cluster_centers, dtype=torch.float32).cpu()+ centers = torch.nn.functional.normalize(centers, dim=1)+ stable_ids = []+ used_ids = set()++ for center in centers:+ best_id = None+ best_similarity = float("-inf")+ created = False+ if self.speaker_centers:+ similarities = torch.stack(+ [torch.dot(center, known_center) for known_center in self.speaker_centers]+ )+ for candidate in torch.argsort(similarities, descending=True).tolist():+ if candidate not in used_ids:+ best_id = candidate+ best_similarity = float(similarities[candidate])+ break++ matched = best_id is not None and best_similarity >= self.threshold+ if not matched and update and len(self.speaker_centers) < self.max_speakers:+ best_id = len(self.speaker_centers)+ self.speaker_centers.append(center.clone())+ self.speaker_center_updates.append(1)+ matched = True+ created = True+ elif best_id is None:+ # There can be more active clusters than the configured identity cap.+ if not self.speaker_centers:+ best_id = 0+ else:+ similarities = torch.stack(+ [torch.dot(center, known_center) for known_center in self.speaker_centers]+ )+ best_id = int(torch.argmax(similarities))++ if update and matched and not created:+ count = self.speaker_center_updates[best_id]+ weight = 1.0 / min(count + 1, 20)+ updated = (1.0 - weight) * self.speaker_centers[best_id] + weight * center+ self.speaker_centers[best_id] = torch.nn.functional.normalize(updated, dim=0)+ self.speaker_center_updates[best_id] = count + 1++ stable_ids.append(best_id)+ used_ids.add(best_id)++ return stable_ids@torch.no_grad()def finalize(self, sentences, min_split_s=3.0):@@ -131,31 +202,36 @@ class HybridSpeakerTracker:if not self.all_embeddings or not sentences:return sentences- all_embs = torch.cat(self.all_embeddings, dim=0)- labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)- if not isinstance(labels, np.ndarray):- labels = np.array(labels)-- all_sorted = sorted(self.all_chunks, key=lambda x: x[0])- sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())-- for s in sentences:- s.pop("spk", None)- self.distribute_spk(sentences, sv_output)-- id_map = {}- next_id = 0- for s in sentences:- raw = s.get("spk", 0)- if raw not in id_map:- id_map[raw] = next_id- next_id += 1- s["spk"] = id_map[raw]-+ sv_output = self._cluster_recent(update_centers=False)+ history_start_ms = int(min(start for start, _ in self.all_chunks) * 1000)final_sentences = []for s in sentences:- sub = self._try_split(s, sv_output, id_map, min_split_s)- final_sentences.extend(sub)+ if s["end"] <= history_start_ms:+ final_sentences.append(s)+ continue+ current = dict(s)+ if s["start"] < history_start_ms:+ duration_ms = s["end"] - s["start"]+ boundary_ratio = (history_start_ms - s["start"]) / duration_ms+ split_index = min(+ len(s["text"]) - 1,+ max(1, round(len(s["text"]) * boundary_ratio)),+ )+ prefix_text = s["text"][:split_index].strip()+ suffix_text = s["text"][split_index:].strip()+ if not prefix_text or not suffix_text:+ final_sentences.append(s)+ continue++ prefix = dict(s)+ prefix.update(text=prefix_text, end=history_start_ms)+ final_sentences.append(prefix)+ current.update(+ text=suffix_text,+ start=history_start_ms,+ )+ self.distribute_spk([current], sv_output)+ final_sentences.extend(self._try_split(current, sv_output, {}, min_split_s))return final_sentences@@ -217,17 +293,26 @@ class HybridSpeakerTracker:def reset(self):self.speaker_centers = []- self.all_chunks = []- self.all_embeddings = []- self.display_map = {}- self.next_display_id = 0+ self.speaker_center_updates = []+ self.all_chunks.clear()+ self.all_embeddings.clear()+ self.last_speaker_id = 0class RealtimeASRSession:"""Manages a single streaming ASR session."""- def __init__(self, vllm_engine, asr_kwargs, vad, spk_tracker=None, sample_rate=16000, chunk_ms=960,- partial_window_sec=15.0):+ def __init__(+ self,+ vllm_engine,+ asr_kwargs,+ vad,+ spk_tracker=None,+ sample_rate=16000,+ chunk_ms=960,+ partial_window_sec=15.0,+ audio_lookback_sec=5.0,+ ):self.vllm_engine = vllm_engineself.asr_kwargs = asr_kwargsself.vad = vad@@ -243,10 +328,12 @@ class RealtimeASRSession:# without changing the final result (completed segments are always decoded# in full by _decode_segment / the is_final path). Set <=0 to disable.self.partial_window_samples = int(sample_rate * partial_window_sec) if partial_window_sec and partial_window_sec > 0 else 0+ self.audio_lookback_samples = max(0, int(sample_rate * audio_lookback_sec))self.first_decode_done = Falseself.audio_buffer = np.array([], dtype=np.float32)- self.vad_fed_samples = 0+ self.audio_buffer_start_sample = 0+ self.total_samples = 0self.prev_text = ""self.last_partial_text = ""self.last_partial_start_ms = 0@@ -261,11 +348,10 @@ class RealtimeASRSession:audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16)audio_float = audio_int16.astype(np.float32) / 32768.0self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])+ self.total_samples += len(audio_float)- new_audio = self.audio_buffer[self.vad_fed_samples:]- if len(new_audio) > 0:- new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)- self.vad_fed_samples = len(self.audio_buffer)+ if len(audio_float) > 0:+ new_confirmed = self.vad.feed(torch.from_numpy(audio_float).float(), is_final=False)for seg in new_confirmed:seg_text = self._decode_segment(seg)@@ -275,32 +361,51 @@ class RealtimeASRSession:self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})if self.spk_tracker:s0 = int(seg[0] * self.sample_rate / 1000)- s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))- self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1])+ s1 = min(int(seg[1] * self.sample_rate / 1000), self.total_samples)+ segment_audio = self._slice_audio(s0, s1).copy()+ self.spk_tracker.assign_streaming(segment_audio, seg[0]/1000, seg[1]/1000, self.locked_sentences[-1])logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"")+ self._compact_audio_buffer()++ def _slice_audio(self, start_sample, end_sample):+ local_start = max(0, start_sample - self.audio_buffer_start_sample)+ local_end = min(len(self.audio_buffer), end_sample - self.audio_buffer_start_sample)+ if local_end <= local_start:+ return np.array([], dtype=np.float32)+ return self.audio_buffer[local_start:local_end]++ def _compact_audio_buffer(self):+ if self.vad.current_speech_start is not None:+ keep_from = int(self.vad.current_speech_start * self.sample_rate / 1000)+ else:+ keep_from = self.total_samples - self.audio_lookback_samples+ keep_from = min(self.total_samples, max(self.audio_buffer_start_sample, keep_from))+ drop_samples = keep_from - self.audio_buffer_start_sample+ if drop_samples > 0:+ # A copy is required here; a slice would keep the discarded backing+ # array alive and recreate the long-session memory leak.+ self.audio_buffer = self.audio_buffer[drop_samples:].copy()+ self.audio_buffer_start_sample = keep_from++ def _release_audio_buffer(self):+ self.audio_buffer = np.array([], dtype=np.float32)+ self.audio_buffer_start_sample = self.total_samples+def should_decode(self):threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples- return (len(self.audio_buffer) - self.last_decode_samples) >= threshold+ return (self.total_samples - self.last_decode_samples) >= threshold@torch.no_grad()def decode(self, is_final=False):- if len(self.audio_buffer) < self.chunk_samples:+ if self.total_samples < self.chunk_samples:+ if is_final:+ self._release_audio_buffer()return self._build_response(is_final)if is_final:- remaining = self.audio_buffer[self.vad_fed_samples:]- if len(remaining) > 0:- new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)- self.vad_fed_samples = len(self.audio_buffer)- for seg in new_confirmed:- seg_text = self._decode_segment(seg)- if not seg_text.strip():- continue- self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})-if self.vad.current_speech_start is not None:- end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)+ end_ms = int(self.total_samples * 1000 / self.sample_rate)seg = [self.vad.current_speech_start, end_ms]seg_text = self._decode_segment(seg)if seg_text.strip():@@ -309,12 +414,13 @@ class RealtimeASRSession:if self.spk_tracker and self.locked_sentences:self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences)+ self._release_audio_buffer()return self._build_response(is_final)if self.vad.current_speech_start is not None:seg_audio, partial_start_ms = self.get_partial_decode_audio()else:- self.last_decode_samples = len(self.audio_buffer)+ self.last_decode_samples = self.total_samplesself.last_partial_text = ""return self._build_response(is_final)@@ -339,7 +445,7 @@ class RealtimeASRSession:if hallucinated:self.prev_text = ""- self.last_decode_samples = len(self.audio_buffer)+ self.last_decode_samples = self.total_samplesself.last_partial_text = textself.last_partial_start_ms = partial_start_msif text.strip() and not self.first_decode_done:@@ -360,20 +466,20 @@ class RealtimeASRSession:decode_start_sample = seg_start_sampleif self.partial_window_samples:- min_start = len(self.audio_buffer) - self.partial_window_samples+ min_start = self.total_samples - self.partial_window_samplesif min_start > decode_start_sample:decode_start_sample = min_start- decode_start_sample = max(0, decode_start_sample)+ decode_start_sample = max(self.audio_buffer_start_sample, decode_start_sample)start_ms = int(decode_start_sample * 1000 / self.sample_rate)- return self.audio_buffer[decode_start_sample:], start_ms+ return self._slice_audio(decode_start_sample, self.total_samples), start_ms@torch.no_grad()def _decode_segment(self, seg):"""Decode a completed VAD segment via vLLM."""start_sample = int(seg[0] * self.sample_rate / 1000)- end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))- seg_audio = self.audio_buffer[start_sample:end_sample]+ end_sample = min(int(seg[1] * self.sample_rate / 1000), self.total_samples)+ seg_audio = self._slice_audio(start_sample, end_sample)if len(seg_audio) < 1600:return ""audio_tensor = torch.from_numpy(seg_audio).float()@@ -393,7 +499,7 @@ class RealtimeASRSession:return ""def _build_response(self, is_final):- duration_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)+ duration_ms = int(self.total_samples * 1000 / self.sample_rate)sentences = list(self.locked_sentences)partial = self.last_partial_textif partial:@@ -412,7 +518,8 @@ class RealtimeASRSession:def reset(self):self.audio_buffer = np.array([], dtype=np.float32)- self.vad_fed_samples = 0+ self.audio_buffer_start_sample = 0+ self.total_samples = 0self.first_decode_done = Falseself.vad.reset()self.prev_text = ""@@ -477,6 +584,17 @@ def create_speaker_tracker(spk_model, args):return HybridSpeakerTracker(spk_model, args.device)+async def run_session_work(args, operation, *operation_args, **operation_kwargs):+ """Run blocking session work off-loop without concurrent shared-model access."""+ lock = getattr(args, "_session_work_lock", None)+ if lock is None:+ lock = asyncio.Lock()+ args._session_work_lock = lock++ async with lock:+ return await asyncio.to_thread(operation, *operation_args, **operation_kwargs)++async def handle_client(websocket, args):vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args)vad = DynamicStreamingVAD(vad_model)@@ -516,17 +634,17 @@ async def handle_client(websocket, args):await websocket.send(json.dumps({"event": "language_set", "language": lang}))logger.info(f"Language set: {lang}")elif cmd.upper() == "STOP":- if session.is_active and len(session.audio_buffer) > 0:- result = session.decode(is_final=True)+ if session.is_active and session.total_samples > 0:+ result = await run_session_work(args, session.decode, is_final=True)await websocket.send(json.dumps(result))logger.info(f"Final: {len(result['sentences'])} sentences")session.is_active = Falseawait websocket.send(json.dumps({"event": "stopped"}))elif isinstance(message, bytes) and session.is_active:- session.add_audio(message)+ await run_session_work(args, session.add_audio, message)now = time.time()if now - last_decode_time >= decode_interval and session.should_decode():- result = session.decode(is_final=False)+ result = await run_session_work(args, session.decode, is_final=False)await websocket.send(json.dumps(result))last_decode_time = nowdiff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.pyindex 62396a913..00db6bafd 100644--- a/tests/test_realtime_ws_service.py+++ b/tests/test_realtime_ws_service.py@@ -1,9 +1,13 @@+import asyncioimport importlib.utilimport sys+import threading+import timeimport typesfrom pathlib import Pathimport numpy as np+import torchREPO_ROOT = Path(__file__).resolve().parents[1]@@ -106,6 +110,7 @@ def test_partial_decode_audio_uses_recent_window():partial_window_sec=2.0,)session.audio_buffer = np.arange(sample_rate * 5, dtype=np.float32)+ session.total_samples = len(session.audio_buffer)audio, start_ms = session.get_partial_decode_audio()@@ -133,6 +138,7 @@ def test_partial_decode_audio_keeps_short_current_segment():partial_window_sec=10.0,)session.audio_buffer = np.arange(sample_rate * 5, dtype=np.float32)+ session.total_samples = len(session.audio_buffer)audio, start_ms = session.get_partial_decode_audio()assert start_ms == 1500@@ -158,8 +164,297 @@ def test_empty_partial_response_preserves_zero_speech_start():chunk_ms=960,)session.audio_buffer = np.zeros(sample_rate, dtype=np.float32)+ session.total_samples = len(session.audio_buffer)session.last_partial_text = ""response = session._build_response(is_final=False)assert response["partial_start_ms"] == 0+++class SilentVad:+ current_speech_start = None++ def feed(self, audio, is_final=False):+ return []++ def reset(self):+ pass+++class SegmentVad(SilentVad):+ def __init__(self, sample_rate, segment_end_sample):+ self.sample_rate = sample_rate+ self.segment_end_sample = segment_end_sample+ self.total_samples = 0++ def feed(self, audio, is_final=False):+ self.total_samples += len(audio)+ if self.total_samples == self.segment_end_sample:+ end_ms = int(self.total_samples * 1000 / self.sample_rate)+ return [[end_ms - 1000, end_ms]]+ return []+++class DummyTokenizer:+ def encode(self, text):+ return []++ def decode(self, token_ids, skip_special_tokens=True):+ return ""+++class DummyEngine:+ def __init__(self):+ self._engine = types.SimpleNamespace(tokenizer=DummyTokenizer())+ self.input_lengths = []++ def generate(self, inputs, **kwargs):+ self.input_lengths.extend(len(audio) for audio in inputs)+ return [{"text": "hello"}]+++def test_two_hour_session_keeps_audio_bounded_and_duration_absolute():+ module = load_service_module()+ sample_rate = 10+ lookback_seconds = 5+ session = module.RealtimeASRSession(+ vllm_engine=DummyEngine(),+ asr_kwargs={},+ vad=SilentVad(),+ sample_rate=sample_rate,+ chunk_ms=1000,+ audio_lookback_sec=lookback_seconds,+ )+ one_second = np.zeros(sample_rate, dtype=np.int16).tobytes()++ for _ in range(2 * 60 * 60):+ session.add_audio(one_second)++ assert session.total_samples == 2 * 60 * 60 * sample_rate+ assert len(session.audio_buffer) <= lookback_seconds * sample_rate+ assert session.audio_buffer_start_sample == session.total_samples - len(session.audio_buffer)+ assert session._build_response(is_final=False)["duration_ms"] == 2 * 60 * 60 * 1000+++def test_completed_segment_uses_absolute_offsets_after_audio_compaction():+ module = load_service_module()+ sample_rate = 16000+ vad = SegmentVad(sample_rate=sample_rate, segment_end_sample=11 * sample_rate)+ engine = DummyEngine()+ session = module.RealtimeASRSession(+ vllm_engine=engine,+ asr_kwargs={},+ vad=vad,+ sample_rate=sample_rate,+ chunk_ms=1000,+ audio_lookback_sec=2,+ )+ one_second = np.zeros(sample_rate, dtype=np.int16).tobytes()++ for _ in range(11):+ session.add_audio(one_second)++ assert engine.input_lengths == [sample_rate]+ assert session.locked_sentences == [{"text": "hello", "start": 10000, "end": 11000}]+ assert session._build_response(is_final=False)["duration_ms"] == 11000+ assert session.audio_buffer_start_sample > 0+++def test_final_decode_releases_audio_without_resetting_absolute_duration():+ module = load_service_module()+ sample_rate = 10+ for sample_count in (5, 100):+ session = module.RealtimeASRSession(+ vllm_engine=DummyEngine(),+ asr_kwargs={},+ vad=SilentVad(),+ sample_rate=sample_rate,+ chunk_ms=1000,+ audio_lookback_sec=5,+ )+ session.add_audio(np.zeros(sample_count, dtype=np.int16).tobytes())++ response = session.decode(is_final=True)++ assert response["duration_ms"] == sample_count * 100+ assert len(session.audio_buffer) == 0+ assert session.audio_buffer_start_sample == session.total_samples+++def test_speaker_history_and_identity_state_have_hard_limits(monkeypatch):+ utils_stub = types.ModuleType("funasr.models.campplus.utils")+ utils_stub.sv_chunk = lambda segments: segments++ def postprocess(segments, vad_segments, labels, embeddings, return_spk_center=False):+ output = [[segment[0], segment[1], int(label)] for segment, label in zip(segments, labels)]+ centers = torch.stack(+ [embeddings[labels == label].mean(0) for label in sorted(set(labels.tolist()))]+ )+ return (output, centers) if return_spk_center else output++ def distribute_spk(sentences, speaker_segments):+ for sentence in sentences:+ sentence["spk"] = int(speaker_segments[-1][2])+ return sentences++ utils_stub.postprocess = postprocess+ utils_stub.distribute_spk = distribute_spk+ cluster_stub = types.ModuleType("funasr.models.campplus.cluster_backend")++ class FakeClusterBackend:+ def __init__(self, merge_thr):+ pass++ def to(self, device):+ return self++ def __call__(self, embeddings, oracle_num=None):+ return np.zeros(len(embeddings), dtype=np.int64)++ cluster_stub.ClusterBackend = FakeClusterBackend+ monkeypatch.setitem(sys.modules, "funasr.models.campplus.utils", utils_stub)+ monkeypatch.setitem(sys.modules, "funasr.models.campplus.cluster_backend", cluster_stub)++ module = load_service_module()++ class FakeSpeakerModel:+ def generate(self, input, **kwargs):+ return [{"spk_embedding": torch.tensor([[1.0, 0.0]])} for _ in input]++ tracker = module.HybridSpeakerTracker(+ spk_model=FakeSpeakerModel(),+ device="cpu",+ max_history_chunks=3,+ max_speakers=2,+ )+ tracker.sv_chunk = lambda segments: [+ [segments[0][0], segments[0][1], segments[0][2]]+ ]+ tracker.cluster_backend = lambda embeddings, oracle_num=None: np.zeros(+ len(embeddings), dtype=np.int64+ )++ for index in range(6):+ sentence = {"text": f"segment {index}", "start": index * 1000, "end": (index + 1) * 1000}+ tracker.assign_streaming(+ np.ones(16000, dtype=np.float32),+ index,+ index + 1,+ sentence,+ )+ assert sentence["spk"] == 0++ assert len(tracker.all_chunks) == 3+ assert len(tracker.all_embeddings) == 3+ assert all(len(chunk) == 2 for chunk in tracker.all_chunks)++ first = tracker._map_cluster_centers(+ torch.tensor([[1.0, 0.0], [0.0, 1.0]]), update=True+ )+ swapped = tracker._map_cluster_centers(+ torch.tensor([[0.0, 1.0], [1.0, 0.0]]), update=True+ )+ capped = tracker._map_cluster_centers(+ torch.tensor([[-1.0, 0.0]]), update=True+ )++ assert first == [0, 1]+ assert swapped == [1, 0]+ assert capped[0] in {0, 1}+ assert len(tracker.speaker_centers) == 2+ assert tracker.all_chunks.maxlen == 3+ assert tracker.all_embeddings.maxlen == 3++ finalized = tracker.finalize(+ [{"text": "abcdef", "start": 0, "end": 6000, "spk": 1}]+ )+ assert finalized == [+ {"text": "abc", "start": 0, "end": 3000, "spk": 1},+ {"text": "def", "start": 3000, "end": 6000, "spk": 0},+ ]+++def test_handler_keeps_event_loop_responsive_during_session_work(monkeypatch):+ module = load_service_module()++ class BlockingSession:+ active_workers = 0+ max_active_workers = 0+ worker_lock = threading.Lock()++ def __init__(self, *args, **kwargs):+ self.is_active = False+ self.audio_buffer = np.zeros(1, dtype=np.float32)++ def reset(self):+ pass++ def add_audio(self, message):+ with self.worker_lock:+ type(self).active_workers += 1+ type(self).max_active_workers = max(+ type(self).max_active_workers,+ type(self).active_workers,+ )+ try:+ time.sleep(0.2)+ finally:+ with self.worker_lock:+ type(self).active_workers -= 1++ def should_decode(self):+ return False++ class FakeWebSocket:+ remote_address = ("127.0.0.1", 12345)++ def __init__(self):+ self.messages = iter(["START", b"audio"])+ self.sent = []++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self.messages)+ except StopIteration as error:+ raise StopAsyncIteration from error++ async def send(self, message):+ self.sent.append(message)++ monkeypatch.setattr(module, "load_models", lambda args: (object(), {}, object(), None))+ monkeypatch.setattr(module, "DynamicStreamingVAD", lambda model: object())+ monkeypatch.setattr(module, "create_speaker_tracker", lambda model, args: None)+ monkeypatch.setattr(module, "RealtimeASRSession", BlockingSession)++ async def exercise_handler():+ stop = False+ gaps = []++ async def ticker():+ previous = asyncio.get_running_loop().time()+ while not stop:+ await asyncio.sleep(0.005)+ current = asyncio.get_running_loop().time()+ gaps.append(current - previous)+ previous = current++ ticker_task = asyncio.create_task(ticker())+ await asyncio.sleep(0.01)+ args = types.SimpleNamespace(device="cpu", decode_interval=0.48, partial_window_sec=15.0)+ await asyncio.gather(+ module.handle_client(FakeWebSocket(), args),+ module.handle_client(FakeWebSocket(), args),+ )+ stop = True+ await ticker_task+ return gaps++ gaps = asyncio.run(exercise_handler())++ assert gaps+ assert max(gaps) < 0.08+ assert BlockingSession.max_active_workers == 1