Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(realtime): preserve long segment tails

LauraGPT25b1245e9c9a
diff --git a/funasr/bin/realtime_ws.py b/funasr/bin/realtime_ws.py
index 7ed405ba8..39d25d7b0 100644
--- a/funasr/bin/realtime_ws.py
+++ b/funasr/bin/realtime_ws.py
@@ -282,6 +282,22 @@ def _normalize_transcript(text):
return _normalize_transcript_with_positions(text)[0]
+def _is_supported_transcript_extension(candidate, reference):
+ candidate, hallucinated = detect_and_fix_hallucination(candidate)
+ if hallucinated:
+ return False
+ candidate_cmp = _normalize_transcript(candidate)
+ reference_cmp = _normalize_transcript(reference)
+ if len(candidate_cmp) - len(reference_cmp) < 4:
+ return False
+ common_prefix_len = 0
+ for candidate_char, reference_char in zip(candidate_cmp, reference_cmp):
+ if candidate_char != reference_char:
+ break
+ common_prefix_len += 1
+ return common_prefix_len >= max(8, (len(reference_cmp) * 3) // 4)
+
+
def _merge_overlapping_transcripts(existing, incoming):
"""Join adjacent rolling-window transcripts when their text overlap is clear."""
existing_norm, existing_positions = _normalize_transcript_with_positions(existing)
@@ -714,6 +730,11 @@ class RealtimeASRSession:
self.segment_partial_start_ms = 0
self.segment_partial_end_ms = 0
self.segment_partial_stable_count = 0
+ self.segment_partial_observation_count = 0
+ self.segment_best_partial_text = ""
+ self.segment_best_partial_start_ms = 0
+ self.segment_best_partial_end_ms = 0
+ self.segment_best_partial_observation_count = 0
self.last_decode_samples = 0
self.locked_sentences = []
self.prev_seg_text = ""
@@ -771,11 +792,17 @@ class RealtimeASRSession:
self.audio_buffer = np.array([], dtype=np.float32)
self.audio_buffer_start_sample = self.total_samples
- def _reset_partial_history(self):
+ def _reset_partial_history(self, preserve_best=False):
self.segment_partial_text = ""
self.segment_partial_start_ms = 0
self.segment_partial_end_ms = 0
self.segment_partial_stable_count = 0
+ self.segment_partial_observation_count = 0
+ if not preserve_best:
+ self.segment_best_partial_text = ""
+ self.segment_best_partial_start_ms = 0
+ self.segment_best_partial_end_ms = 0
+ self.segment_best_partial_observation_count = 0
def _record_partial_text(self, text, start_ms, hallucinated=False):
"""Build a confidence-bearing transcript across overlapping partial windows."""
@@ -783,20 +810,33 @@ class RealtimeASRSession:
self.last_partial_end_ms = end_ms
self.last_partial_eligible = bool(text.strip()) and not hallucinated
if not text.strip() or hallucinated:
- self._reset_partial_history()
+ self._reset_partial_history(preserve_best=hallucinated)
return
start_ms = int(start_ms)
incoming_norm = _normalize_transcript(text)
+ best_norm = _normalize_transcript(self.segment_best_partial_text)
+ if not self.segment_best_partial_text or start_ms != self.segment_best_partial_start_ms:
+ self.segment_best_partial_text = text
+ self.segment_best_partial_start_ms = start_ms
+ self.segment_best_partial_end_ms = end_ms
+ self.segment_best_partial_observation_count = 1
+ else:
+ self.segment_best_partial_observation_count += 1
+ if len(incoming_norm) > len(best_norm):
+ self.segment_best_partial_text = text
+ self.segment_best_partial_end_ms = end_ms
if not self.segment_partial_text:
self.segment_partial_text = text
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
+ self.segment_partial_observation_count = 1
return
history_norm = _normalize_transcript(self.segment_partial_text)
if start_ms == self.segment_partial_start_ms:
+ self.segment_partial_observation_count += 1
if incoming_norm == history_norm:
self.segment_partial_text = text
self.segment_partial_end_ms = end_ms
@@ -812,6 +852,7 @@ class RealtimeASRSession:
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
+ self.segment_partial_observation_count = 1
return
if start_ms > self.segment_partial_end_ms:
@@ -819,6 +860,7 @@ class RealtimeASRSession:
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
+ self.segment_partial_observation_count = 1
return
merged = _merge_overlapping_transcripts(self.segment_partial_text, text)
@@ -826,11 +868,13 @@ class RealtimeASRSession:
self.segment_partial_text = merged
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
+ self.segment_partial_observation_count += 1
else:
self.segment_partial_text = text
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
+ self.segment_partial_observation_count = 1
def should_decode(self):
threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples
@@ -1028,6 +1072,64 @@ class RealtimeASRSession:
partial_text = self._partial_fallback_candidate(
seg, require_stable=not final_hallucinated
)
+ long_segment_reason = ""
+ if not partial_text:
+ segment_duration_ms = max(0, int(seg[1]) - int(seg[0]))
+ decode_chunk_ms = int(self.chunk_samples * 1000 / self.sample_rate)
+ recent_partial = self.last_partial_text.strip()
+ recent_start_ms = self.last_partial_start_ms
+ recent_end_ms = self.last_partial_end_ms
+ recent_observations = self.segment_partial_observation_count
+ recent_eligible = self.last_partial_eligible
+ if len(_normalize_transcript(self.segment_best_partial_text)) > len(
+ _normalize_transcript(recent_partial)
+ ):
+ recent_partial = self.segment_best_partial_text.strip()
+ recent_start_ms = self.segment_best_partial_start_ms
+ recent_end_ms = self.segment_best_partial_end_ms
+ recent_observations = self.segment_best_partial_observation_count
+ recent_eligible = True
+ tail_gap_ms = int(seg[1]) - int(recent_end_ms)
+ recent_partial, recent_hallucinated = detect_and_fix_hallucination(
+ recent_partial
+ )
+ if (
+ segment_duration_ms >= 8000
+ and recent_eligible
+ and recent_observations >= 2
+ and int(recent_start_ms) == int(seg[0])
+ and 0 <= tail_gap_ms <= max(100, decode_chunk_ms * 2)
+ and recent_partial
+ and not recent_hallucinated
+ ):
+ final_cmp = _normalize_transcript(final_text)
+ recent_cmp = _normalize_transcript(recent_partial)
+ common_prefix_len = 0
+ for final_char, partial_char in zip(final_cmp, recent_cmp):
+ if final_char != partial_char:
+ break
+ common_prefix_len += 1
+ diverged = common_prefix_len < min(len(final_cmp), len(recent_cmp))
+ catastrophically_short = (
+ diverged
+ and len(final_cmp) >= 2
+ and len(recent_cmp) >= 24
+ and len(recent_cmp) >= len(final_cmp) * 3
+ and common_prefix_len >= max(1, len(final_cmp) // 2)
+ )
+ tail_regressed = (
+ diverged
+ and len(final_cmp) >= 16
+ and len(recent_cmp) - len(final_cmp) >= 4
+ and common_prefix_len >= max(12, (len(final_cmp) * 3) // 4)
+ )
+ if catastrophically_short or tail_regressed:
+ partial_text = recent_partial
+ long_segment_reason = (
+ "catastrophically short"
+ if catastrophically_short
+ else "tail-regressed"
+ )
if not partial_text:
return final_text
partial_text, partial_hallucinated = detect_and_fix_hallucination(partial_text)
@@ -1044,12 +1146,15 @@ class RealtimeASRSession:
use_partial = (
(final_hallucinated and not partial_hallucinated)
or truncated_prefix
+ or bool(long_segment_reason)
)
if use_partial and partial_cmp:
- reason = "hallucinated" if final_hallucinated else "truncated"
+ reason = long_segment_reason or (
+ "hallucinated" if final_hallucinated else "truncated"
+ )
logger.warning(
"Completed segment [%d-%dms] decode was %s; "
- "keeping the latest aligned partial (%d -> %d chars)",
+ "keeping an aligned observed partial (%d -> %d chars)",
int(seg[0]), int(seg[1]), reason, len(final_text), len(partial_text),
)
return partial_text
@@ -1109,9 +1214,40 @@ class RealtimeASRSession:
except Exception as e:
logger.error(f"Segment decode error: {e}")
+ decoded_text = text
text = self._reconcile_completed_segment_text(
- text, seg, decode_succeeded=decode_succeeded
+ decoded_text, seg, decode_succeeded=decode_succeeded
)
+ retry_end_ms = min(int(self.last_partial_end_ms), int(seg[1]))
+ if (
+ decode_succeeded
+ and text != decoded_text
+ and int(seg[1]) - int(seg[0]) >= 8000
+ and int(seg[0]) < retry_end_ms < int(seg[1])
+ ):
+ retry_end_sample = min(
+ int(retry_end_ms * self.sample_rate / 1000), self.total_samples
+ )
+ retry_audio = self._slice_audio(start_sample, retry_end_sample)
+ if len(retry_audio) >= 1600:
+ try:
+ retry_results = self.vllm_engine.generate(
+ inputs=[torch.from_numpy(retry_audio).float()],
+ hotwords=self.asr_kwargs.get("hotwords"),
+ language=self.asr_kwargs.get("language"),
+ max_new_tokens=512,
+ )
+ retry_text = retry_results[0]["text"] if retry_results else ""
+ retry_text = _clean_asr_text(retry_text)
+ if _is_supported_transcript_extension(retry_text, text):
+ logger.warning(
+ "Completed segment [%d-%dms] accepted an aligned "
+ "boundary retry extension (%d -> %d chars)",
+ int(seg[0]), int(seg[1]), len(text), len(retry_text),
+ )
+ text = retry_text
+ except Exception as error:
+ logger.warning("Segment boundary retry failed: %s", error)
text = _postprocess_result_text(text, self.asr_kwargs)
self.prev_seg_text = text
self._clear_completed_partial(seg)
diff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.py
index 22fbf0fb8..27223a36b 100644
--- a/tests/test_realtime_ws_service.py
+++ b/tests/test_realtime_ws_service.py
@@ -492,6 +492,215 @@ def test_completed_segment_keeps_aligned_partial_when_final_is_truncated_prefix(
assert session.last_partial_text == ""
+def make_reported_long_segment_reconciliation_session(module, partial, partial_end_ms):
+ session = module.RealtimeASRSession(
+ FixedTextEngine(""),
+ {},
+ ControllableVad(),
+ sample_rate=16000,
+ chunk_ms=960,
+ )
+ session.last_partial_text = partial
+ session.last_partial_start_ms = 2920
+ session.last_partial_end_ms = partial_end_ms
+ session.last_partial_eligible = True
+ session.segment_partial_observation_count = 2
+ session.segment_best_partial_text = partial
+ session.segment_best_partial_start_ms = 2920
+ session.segment_best_partial_end_ms = partial_end_ms
+ session.segment_best_partial_observation_count = 2
+ return session
+
+
+def test_long_segment_recovers_when_final_regresses_after_shared_prefix():
+ module = load_service_module()
+ partial = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,OK OK,好嘞,胡总各位长官。"
+ )
+ session = make_reported_long_segment_reconciliation_session(
+ module, partial, partial_end_ms=13500
+ )
+
+ text = session._reconcile_completed_segment_text(
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜",
+ [2920, 14420],
+ decode_succeeded=True,
+ )
+
+ assert text == partial
+
+
+def test_long_segment_recovers_from_catastrophically_short_final():
+ module = load_service_module()
+ partial = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,O K O K,好嘞,胡总各位长官。"
+ )
+ session = make_reported_long_segment_reconciliation_session(
+ module, partial, partial_end_ms=13500
+ )
+
+ text = session._reconcile_completed_segment_text(
+ "哇,嘿",
+ [2920, 14000],
+ decode_succeeded=True,
+ )
+
+ assert text == partial
+
+
+def test_long_segment_keeps_distinct_short_final_without_prefix_alignment():
+ module = load_service_module()
+ partial = "会议取消,稍后另行通知,请各位等待后续的完整安排和确认消息。"
+ session = make_reported_long_segment_reconciliation_session(
+ module, partial, partial_end_ms=13500
+ )
+
+ text = session._reconcile_completed_segment_text(
+ "项目批准",
+ [2920, 14000],
+ decode_succeeded=True,
+ )
+
+ assert text == "项目批准"
+
+
+def test_long_segment_does_not_trust_a_single_regressed_partial_observation():
+ module = load_service_module()
+ partial = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,OK OK,好嘞,胡总各位长官。"
+ )
+ session = make_reported_long_segment_reconciliation_session(
+ module, partial, partial_end_ms=13500
+ )
+ session.segment_partial_observation_count = 1
+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"
+
+ text = session._reconcile_completed_segment_text(
+ final,
+ [2920, 14420],
+ decode_succeeded=True,
+ )
+
+ assert text == final
+
+
+def test_long_segment_can_recover_the_best_earlier_partial():
+ module = load_service_module()
+ best_partial = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"
+ "OK OK OK。"
+ )
+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"
+ session = make_reported_long_segment_reconciliation_session(
+ module, final, partial_end_ms=13500
+ )
+ session.segment_partial_observation_count = 10
+ session.segment_best_partial_text = best_partial
+ session.segment_best_partial_start_ms = 2920
+ session.segment_best_partial_end_ms = 12500
+ session.segment_best_partial_observation_count = 10
+
+ text = session._reconcile_completed_segment_text(
+ final,
+ [2920, 14420],
+ decode_succeeded=True,
+ )
+
+ assert text == best_partial
+
+
+def test_long_segment_keeps_clean_history_after_a_hallucinated_partial():
+ module = load_service_module()
+ best_partial = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,OK OK,好嘞,胡总各位长官。"
+ )
+ final = "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜"
+ session = make_reported_long_segment_reconciliation_session(
+ module, best_partial, partial_end_ms=13500
+ )
+ session.segment_best_partial_observation_count = 8
+
+ session.last_partial_text = final
+ session.last_partial_start_ms = 2920
+ session.last_partial_end_ms = 13500
+ session.last_partial_eligible = False
+ session._reset_partial_history(preserve_best=True)
+
+ text = session._reconcile_completed_segment_text(
+ final,
+ [2920, 14420],
+ decode_succeeded=True,
+ )
+
+ assert text == best_partial
+
+
+def test_retry_extension_requires_a_longer_aligned_clean_transcript():
+ module = load_service_module()
+ fallback = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"
+ "OK OK OK。"
+ )
+ extension = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,OK OK,好嘞,胡总各位长官。"
+ )
+
+ assert module._is_supported_transcript_extension(extension, fallback) is True
+ assert module._is_supported_transcript_extension("完全不同的较长结果文本", fallback) is False
+ assert module._is_supported_transcript_extension("哇,不好意思。", fallback) is False
+
+
+def test_completed_segment_retries_only_after_a_supported_regression():
+ module = load_service_module()
+ fallback = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎,"
+ "OK OK OK。"
+ )
+ extension = (
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点。"
+ "哎,OK OK,好嘞,胡总各位长官。"
+ )
+ engine = ScriptedTextEngine(
+ [
+ "哇,不好意思,上午高雄交通比较混乱一点,我们不晓得我们有点哎。好,拜",
+ extension,
+ ]
+ )
+ session = make_reported_long_segment_reconciliation_session(
+ module, fallback, partial_end_ms=13500
+ )
+ session.vllm_engine = engine
+ session.total_samples = int(14.42 * session.sample_rate)
+ session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)
+
+ text = session._decode_segment([2920, 14420])
+
+ assert text == extension
+ assert len(engine.generate_kwargs) == 2
+
+
+def test_completed_segment_does_not_retry_a_normal_final():
+ module = load_service_module()
+ final = "感谢各位今天参加会议,后续安排将在明天正式通知。"
+ engine = ScriptedTextEngine([final])
+ session = make_reported_long_segment_reconciliation_session(
+ module, "感谢各位今天参加会以。", partial_end_ms=13500
+ )
+ session.vllm_engine = engine
+ session.total_samples = int(14.42 * session.sample_rate)
+ session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)
+
+ text = session._decode_segment([2920, 14420])
+
+ assert text == final
+ assert len(engine.generate_kwargs) == 1
+
+
def test_completed_segment_keeps_clean_aligned_partial_when_final_hallucinates():
module = load_service_module()
partial = "不好意思,上午杭高架交通比较混乱一点,我们不晓得我们有点耽误。"