Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(realtime): preserve long segment tails
LauraGPT25b1245e9c9a
diff --git a/funasr/bin/realtime_ws.py b/funasr/bin/realtime_ws.pyindex 7ed405ba8..39d25d7b0 100644--- a/funasr/bin/realtime_ws.py+++ b/funasr/bin/realtime_ws.py@@ -282,6 +282,22 @@ def _normalize_transcript(text):return _normalize_transcript_with_positions(text)[0]+def _is_supported_transcript_extension(candidate, reference):+ candidate, hallucinated = detect_and_fix_hallucination(candidate)+ if hallucinated:+ return False+ candidate_cmp = _normalize_transcript(candidate)+ reference_cmp = _normalize_transcript(reference)+ if len(candidate_cmp) - len(reference_cmp) < 4:+ return False+ common_prefix_len = 0+ for candidate_char, reference_char in zip(candidate_cmp, reference_cmp):+ if candidate_char != reference_char:+ break+ common_prefix_len += 1+ return common_prefix_len >= max(8, (len(reference_cmp) * 3) // 4)++def _merge_overlapping_transcripts(existing, incoming):"""Join adjacent rolling-window transcripts when their text overlap is clear."""existing_norm, existing_positions = _normalize_transcript_with_positions(existing)@@ -714,6 +730,11 @@ class RealtimeASRSession:self.segment_partial_start_ms = 0self.segment_partial_end_ms = 0self.segment_partial_stable_count = 0+ self.segment_partial_observation_count = 0+ self.segment_best_partial_text = ""+ self.segment_best_partial_start_ms = 0+ self.segment_best_partial_end_ms = 0+ self.segment_best_partial_observation_count = 0self.last_decode_samples = 0self.locked_sentences = []self.prev_seg_text = ""@@ -771,11 +792,17 @@ class RealtimeASRSession:self.audio_buffer = np.array([], dtype=np.float32)self.audio_buffer_start_sample = self.total_samples- def _reset_partial_history(self):+ def _reset_partial_history(self, preserve_best=False):self.segment_partial_text = ""self.segment_partial_start_ms = 0self.segment_partial_end_ms = 0self.segment_partial_stable_count = 0+ self.segment_partial_observation_count = 0+ if not preserve_best:+ self.segment_best_partial_text = ""+ self.segment_best_partial_start_ms = 0+ self.segment_best_partial_end_ms = 0+ self.segment_best_partial_observation_count = 0def _record_partial_text(self, text, start_ms, hallucinated=False):"""Build a confidence-bearing transcript across overlapping partial windows."""@@ -783,20 +810,33 @@ class RealtimeASRSession:self.last_partial_end_ms = end_msself.last_partial_eligible = bool(text.strip()) and not hallucinatedif not text.strip() or hallucinated:- self._reset_partial_history()+ self._reset_partial_history(preserve_best=hallucinated)returnstart_ms = int(start_ms)incoming_norm = _normalize_transcript(text)+ best_norm = _normalize_transcript(self.segment_best_partial_text)+ if not self.segment_best_partial_text or start_ms != self.segment_best_partial_start_ms:+ self.segment_best_partial_text = text+ self.segment_best_partial_start_ms = start_ms+ self.segment_best_partial_end_ms = end_ms+ self.segment_best_partial_observation_count = 1+ else:+ self.segment_best_partial_observation_count += 1+ if len(incoming_norm) > len(best_norm):+ self.segment_best_partial_text = text+ self.segment_best_partial_end_ms = end_msif not self.segment_partial_text:self.segment_partial_text = textself.segment_partial_start_ms = start_msself.segment_partial_end_ms = end_msself.segment_partial_stable_count = 1+ self.segment_partial_observation_count = 1returnhistory_norm = _normalize_transcript(self.segment_partial_text)if start_ms == self.segment_partial_start_ms:+ self.segment_partial_observation_count += 1if incoming_norm == history_norm:self.segment_partial_text = textself.segment_partial_end_ms = end_ms@@ -812,6 +852,7 @@ class RealtimeASRSession:self.segment_partial_start_ms = start_msself.segment_partial_end_ms = end_msself.segment_partial_stable_count = 1+ self.segment_partial_observation_count = 1returnif start_ms > self.segment_partial_end_ms:@@ -819,6 +860,7 @@ class RealtimeASRSession:self.segment_partial_start_ms = start_msself.segment_partial_end_ms = end_msself.segment_partial_stable_count = 1+ self.segment_partial_observation_count = 1returnmerged = _merge_overlapping_transcripts(self.segment_partial_text, text)@@ -826,11 +868,13 @@ class RealtimeASRSession:self.segment_partial_text = mergedself.segment_partial_end_ms = end_msself.segment_partial_stable_count = 1+ self.segment_partial_observation_count += 1else:self.segment_partial_text = textself.segment_partial_start_ms = start_msself.segment_partial_end_ms = end_msself.segment_partial_stable_count = 1+ self.segment_partial_observation_count = 1def should_decode(self):threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples@@ -1028,6 +1072,64 @@ class RealtimeASRSession:partial_text = self._partial_fallback_candidate(seg, require_stable=not final_hallucinated)+ long_segment_reason = ""+ if not partial_text:+ segment_duration_ms = max(0, int(seg[1]) - int(seg[0]))+ decode_chunk_ms = int(self.chunk_samples * 1000 / self.sample_rate)+ recent_partial = self.last_partial_text.strip()+ recent_start_ms = self.last_partial_start_ms+ recent_end_ms = self.last_partial_end_ms+ recent_observations = self.segment_partial_observation_count+ recent_eligible = self.last_partial_eligible+ if len(_normalize_transcript(self.segment_best_partial_text)) > len(+ _normalize_transcript(recent_partial)+ ):+ recent_partial = self.segment_best_partial_text.strip()+ recent_start_ms = self.segment_best_partial_start_ms+ recent_end_ms = self.segment_best_partial_end_ms+ recent_observations = self.segment_best_partial_observation_count+ recent_eligible = True+ tail_gap_ms = int(seg[1]) - int(recent_end_ms)+ recent_partial, recent_hallucinated = detect_and_fix_hallucination(+ recent_partial+ )+ if (+ segment_duration_ms >= 8000+ and recent_eligible+ and recent_observations >= 2+ and int(recent_start_ms) == int(seg[0])+ and 0 <= tail_gap_ms <= max(100, decode_chunk_ms * 2)+ and recent_partial+ and not recent_hallucinated+ ):+ final_cmp = _normalize_transcript(final_text)+ recent_cmp = _normalize_transcript(recent_partial)+ common_prefix_len = 0+ for final_char, partial_char in zip(final_cmp, recent_cmp):+ if final_char != partial_char:+ break+ common_prefix_len += 1+ diverged = common_prefix_len < min(len(final_cmp), len(recent_cmp))+ catastrophically_short = (+ diverged+ and len(final_cmp) >= 2+ and len(recent_cmp) >= 24+ and len(recent_cmp) >= len(final_cmp) * 3+ and common_prefix_len >= max(1, len(final_cmp) // 2)+ )+ tail_regressed = (+ diverged+ and len(final_cmp) >= 16+ and len(recent_cmp) - len(final_cmp) >= 4+ and common_prefix_len >= max(12, (len(final_cmp) * 3) // 4)+ )+ if catastrophically_short or tail_regressed:+ partial_text = recent_partial+ long_segment_reason = (+ "catastrophically short"+ if catastrophically_short+ else "tail-regressed"+ )if not partial_text:return final_textpartial_text, partial_hallucinated = detect_and_fix_hallucination(partial_text)@@ -1044,12 +1146,15 @@ class RealtimeASRSession:use_partial = ((final_hallucinated and not partial_hallucinated)or truncated_prefix+ or bool(long_segment_reason))if use_partial and partial_cmp:- reason = "hallucinated" if final_hallucinated else "truncated"+ reason = long_segment_reason or (+ "hallucinated" if final_hallucinated else "truncated"+ )logger.warning("Completed segment [%d-%dms] decode was %s; "- "keeping the latest aligned partial (%d -> %d chars)",+ "keeping an aligned observed partial (%d -> %d chars)",int(seg[0]), int(seg[1]), reason, len(final_text), len(partial_text),)return partial_text@@ -1109,9 +1214,40 @@ class RealtimeASRSession:except Exception as e:logger.error(f"Segment decode error: {e}")+ decoded_text = texttext = self._reconcile_completed_segment_text(- text, seg, decode_succeeded=decode_succeeded+ decoded_text, seg, decode_succeeded=decode_succeeded)+ retry_end_ms = min(int(self.last_partial_end_ms), int(seg[1]))+ if (+ decode_succeeded+ and text != decoded_text+ and int(seg[1]) - int(seg[0]) >= 8000+ and int(seg[0]) < retry_end_ms < int(seg[1])+ ):+ retry_end_sample = min(+ int(retry_end_ms * self.sample_rate / 1000), self.total_samples+ )+ retry_audio = self._slice_audio(start_sample, retry_end_sample)+ if len(retry_audio) >= 1600:+ try:+ retry_results = self.vllm_engine.generate(+ inputs=[torch.from_numpy(retry_audio).float()],+ hotwords=self.asr_kwargs.get("hotwords"),+ language=self.asr_kwargs.get("language"),+ max_new_tokens=512,+ )+ retry_text = retry_results[0]["text"] if retry_results else ""+ retry_text = _clean_asr_text(retry_text)+ if _is_supported_transcript_extension(retry_text, text):+ logger.warning(+ "Completed segment [%d-%dms] accepted an aligned "+ "boundary retry extension (%d -> %d chars)",+ int(seg[0]), int(seg[1]), len(text), len(retry_text),+ )+ text = retry_text+ except Exception as error:+ logger.warning("Segment boundary retry failed: %s", error)text = _postprocess_result_text(text, self.asr_kwargs)self.prev_seg_text = textself._clear_completed_partial(seg)diff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.pyindex 22fbf0fb8..27223a36b 100644--- a/tests/test_realtime_ws_service.py+++ b/tests/test_realtime_ws_service.py@@ -492,6 +492,215 @@ def test_completed_segment_keeps_aligned_partial_when_final_is_truncated_prefix(assert session.last_partial_text == ""+def make_reported_long_segment_reconciliation_session(module, partial, partial_end_ms):+ session = module.RealtimeASRSession(+ FixedTextEngine(""),+ {},+ ControllableVad(),+ sample_rate=16000,+ chunk_ms=960,+ )+ session.last_partial_text = partial+ session.last_partial_start_ms = 2920+ session.last_partial_end_ms = partial_end_ms+ session.last_partial_eligible = True+ session.segment_partial_observation_count = 2+ session.segment_best_partial_text = partial+ session.segment_best_partial_start_ms = 2920+ session.segment_best_partial_end_ms = partial_end_ms+ session.segment_best_partial_observation_count = 2+ return session+++def test_long_segment_recovers_when_final_regresses_after_shared_prefix():+ module = load_service_module()+ partial = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,OK OK,好嘞,胡总各位长官。"+ )+ session = make_reported_long_segment_reconciliation_session(+ module, partial, partial_end_ms=13500+ )++ text = session._reconcile_completed_segment_text(+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜",+ [2920, 14420],+ decode_succeeded=True,+ )++ assert text == partial+++def test_long_segment_recovers_from_catastrophically_short_final():+ module = load_service_module()+ partial = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,O K O K,好嘞,胡总各位长官。"+ )+ session = make_reported_long_segment_reconciliation_session(+ module, partial, partial_end_ms=13500+ )++ text = session._reconcile_completed_segment_text(+ "哇,嘿",+ [2920, 14000],+ decode_succeeded=True,+ )++ assert text == partial+++def test_long_segment_keeps_distinct_short_final_without_prefix_alignment():+ module = load_service_module()+ partial = "会议取消,稍后另行通知,请各位等待后续的完整安排和确认消息。"+ session = make_reported_long_segment_reconciliation_session(+ module, partial, partial_end_ms=13500+ )++ text = session._reconcile_completed_segment_text(+ "项目批准",+ [2920, 14000],+ decode_succeeded=True,+ )++ assert text == "项目批准"+++def test_long_segment_does_not_trust_a_single_regressed_partial_observation():+ module = load_service_module()+ partial = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,OK OK,好嘞,胡总各位长官。"+ )+ session = make_reported_long_segment_reconciliation_session(+ module, partial, partial_end_ms=13500+ )+ session.segment_partial_observation_count = 1+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"++ text = session._reconcile_completed_segment_text(+ final,+ [2920, 14420],+ decode_succeeded=True,+ )++ assert text == final+++def test_long_segment_can_recover_the_best_earlier_partial():+ module = load_service_module()+ best_partial = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"+ "OK OK OK。"+ )+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"+ session = make_reported_long_segment_reconciliation_session(+ module, final, partial_end_ms=13500+ )+ session.segment_partial_observation_count = 10+ session.segment_best_partial_text = best_partial+ session.segment_best_partial_start_ms = 2920+ session.segment_best_partial_end_ms = 12500+ session.segment_best_partial_observation_count = 10++ text = session._reconcile_completed_segment_text(+ final,+ [2920, 14420],+ decode_succeeded=True,+ )++ assert text == best_partial+++def test_long_segment_keeps_clean_history_after_a_hallucinated_partial():+ module = load_service_module()+ best_partial = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,OK OK,好嘞,胡总各位长官。"+ )+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"+ session = make_reported_long_segment_reconciliation_session(+ module, best_partial, partial_end_ms=13500+ )+ session.segment_best_partial_observation_count = 8++ session.last_partial_text = final+ session.last_partial_start_ms = 2920+ session.last_partial_end_ms = 13500+ session.last_partial_eligible = False+ session._reset_partial_history(preserve_best=True)++ text = session._reconcile_completed_segment_text(+ final,+ [2920, 14420],+ decode_succeeded=True,+ )++ assert text == best_partial+++def test_retry_extension_requires_a_longer_aligned_clean_transcript():+ module = load_service_module()+ fallback = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"+ "OK OK OK。"+ )+ extension = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,OK OK,好嘞,胡总各位长官。"+ )++ assert module._is_supported_transcript_extension(extension, fallback) is True+ assert module._is_supported_transcript_extension("完全不同的较长结果文本", fallback) is False+ assert module._is_supported_transcript_extension("哇,不好意思。", fallback) is False+++def test_completed_segment_retries_only_after_a_supported_regression():+ module = load_service_module()+ fallback = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"+ "OK OK OK。"+ )+ extension = (+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"+ "哎,OK OK,好嘞,胡总各位长官。"+ )+ engine = ScriptedTextEngine(+ [+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜",+ extension,+ ]+ )+ session = make_reported_long_segment_reconciliation_session(+ module, fallback, partial_end_ms=13500+ )+ session.vllm_engine = engine+ session.total_samples = int(14.42 * session.sample_rate)+ session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)++ text = session._decode_segment([2920, 14420])++ assert text == extension+ assert len(engine.generate_kwargs) == 2+++def test_completed_segment_does_not_retry_a_normal_final():+ module = load_service_module()+ final = "感谢各位今天参加会议,后续安排将在明天正式通知。"+ engine = ScriptedTextEngine([final])+ session = make_reported_long_segment_reconciliation_session(+ module, "感谢各位今天参加会以。", partial_end_ms=13500+ )+ session.vllm_engine = engine+ session.total_samples = int(14.42 * session.sample_rate)+ session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)++ text = session._decode_segment([2920, 14420])++ assert text == final+ assert len(engine.generate_kwargs) == 1++def test_completed_segment_keeps_clean_aligned_partial_when_final_hallucinates():module = load_service_module()partial = "不好意思,上午杭高架交通比较混乱一点,我们不晓得我们有点耽误。"