Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(kws): preserve streaming frames across packet and EOS boundaries
zhifu gaod83a92475da9
diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.ymlindex 0102edf11..55b2f4cb0 100644--- a/.github/workflows/test-kws-output.yml+++ b/.github/workflows/test-kws-output.yml@@ -7,9 +7,16 @@ on:- "funasr/models/fsmn_kws_mt/**"- "funasr/models/sanm_kws/**"- "funasr/models/sanm_kws_streaming/**"+ - "funasr/frontends/wav_frontend.py"+ - "funasr/auto/auto_model.py"- "funasr/utils/datadir_writer.py"- "funasr/utils/kws_utils.py"- "tests/test_kws_optional_output.py"+ - "tests/test_kws_streaming_continuity.py"+ - "tests/test_kws_utils.py"+ - "tests/test_fsmn_vad_streaming_buffers.py"+ - "tests/test_dynamic_streaming_vad.py"+ - "tests/test_fsmn_vad_dynamic_silence.py"- ".github/workflows/test-kws-output.yml"push:branches: [main]@@ -18,9 +25,16 @@ on:- "funasr/models/fsmn_kws_mt/**"- "funasr/models/sanm_kws/**"- "funasr/models/sanm_kws_streaming/**"+ - "funasr/frontends/wav_frontend.py"+ - "funasr/auto/auto_model.py"- "funasr/utils/datadir_writer.py"- "funasr/utils/kws_utils.py"- "tests/test_kws_optional_output.py"+ - "tests/test_kws_streaming_continuity.py"+ - "tests/test_kws_utils.py"+ - "tests/test_fsmn_vad_streaming_buffers.py"+ - "tests/test_dynamic_streaming_vad.py"+ - "tests/test_fsmn_vad_dynamic_silence.py"- ".github/workflows/test-kws-output.yml"permissions:@@ -30,6 +44,9 @@ jobs:kws-output:runs-on: ubuntu-latesttimeout-minutes: 15+ env:+ OMP_NUM_THREADS: "1"+ MKL_NUM_THREADS: "1"steps:- uses: actions/checkout@v4- uses: actions/setup-python@v5@@ -38,7 +55,14 @@ jobs:cache: pip- name: Install CPU test dependenciesrun: |- python -m pip install torch --index-url https://download.pytorch.org/whl/cpu- python -m pip install "numpy<2" "kaldiio>=2.17.0" librosa "rapidfuzz>=3.0.0" six- - name: Check KWS results and optional file output without model weights- run: python -m unittest discover -s tests -p test_kws_optional_output.py -v+ python -m pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu+ python -m pip install "numpy<2" "kaldiio>=2.17.0" librosa "rapidfuzz>=3.0.0" six pytest omegaconf hydra-core pyyaml tqdm+ - name: Check KWS continuity and related frontend contracts without model weights+ run: |+ python -m pytest -q \+ tests/test_kws_optional_output.py \+ tests/test_kws_streaming_continuity.py \+ tests/test_kws_utils.py \+ tests/test_fsmn_vad_streaming_buffers.py \+ tests/test_dynamic_streaming_vad.py \+ tests/test_fsmn_vad_dynamic_silence.pydiff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.pyindex 3dfee293e..45f34095e 100644--- a/funasr/frontends/wav_frontend.py+++ b/funasr/frontends/wav_frontend.py@@ -549,7 +549,7 @@ class WavFrontendOnline(nn.Module):feats[i][0, :].unsqueeze(dim=0).repeat((self.lfr_m - 1) // 2, 1))# need the number of the input frames + self.lfr_splice_cache[0].shape[0] is greater than self.lfr_m- if feats_lengths[0] + cache["lfr_splice_cache"][0].shape[0] >= self.lfr_m:+ if is_final or feats_lengths[0] + cache["lfr_splice_cache"][0].shape[0] >= self.lfr_m:lfr_splice_cache_tensor = torch.stack(cache["lfr_splice_cache"]) # B T Dfeats = torch.cat((lfr_splice_cache_tensor, feats), dim=1)feats_lengths += lfr_splice_cache_tensor[0].shape[0]@@ -591,7 +591,7 @@ class WavFrontendOnline(nn.Module):)return torch.empty(0), feats_lengthselse:- if is_final:+ if is_final and cache["lfr_splice_cache"]:cache["waveforms"] = (waveformsif cache["reserve_waveforms"].numel() == 0diff --git a/funasr/models/sanm_kws_streaming/model.py b/funasr/models/sanm_kws_streaming/model.pyindex 051be0c21..12ff51b01 100644--- a/funasr/models/sanm_kws_streaming/model.py+++ b/funasr/models/sanm_kws_streaming/model.py@@ -198,6 +198,9 @@ class SanmKWSStreaming(SanmKWS):cache["decoder"] = cache_decodercache["frontend"] = {}cache["prev_samples"] = torch.empty(0)+ cache["pending_features"] = torch.empty((batch_size, 0, feats_dims))+ cache["frontend_started"] = False+ cache["encoder_started"] = Falsereturn cache@@ -224,40 +227,51 @@ class SanmKWSStreaming(SanmKWS):speech = speech.to(device=kwargs["device"])speech_lengths = speech_lengths.to(device=kwargs["device"])- # Encoderis_final = kwargs.get("is_final", False)- encoder_out, encoder_out_lens = self.encode_chunk(- speech, speech_lengths, cache=cache, is_final=is_final+ no_pending_context = (+ is_final+ and speech.shape[1] == 0+ and cache["encoder"]["chunk_size"][2] == 0+ and cache["encoder"]["encoder_out"] is not None)- if isinstance(encoder_out, tuple):- encoder_out = encoder_out[0]-- chunk_size = cache["encoder"]["chunk_size"]- real_start_pos = chunk_size[0]-- if encoder_out_lens[0] > chunk_size[0] + chunk_size[1] + chunk_size[2]:- assert False, print("impossible case 1 !")- if encoder_out_lens[0] == chunk_size[0] + chunk_size[1] + chunk_size[2]:- real_end_pos = chunk_size[0] + chunk_size[1]- elif encoder_out_lens[0] > chunk_size[0] + chunk_size[1]:- real_end_pos = chunk_size[0] + chunk_size[1]- elif encoder_out_lens[0] > chunk_size[0]:- real_end_pos = encoder_out_lens[0]+ if no_pending_context:+ # Zero-right-context EOS only decodes output already committed.+ encoder_out_accum = cache["encoder"]["encoder_out"]else:- assert False, print("impossible case 2 !")+ encoder_out, encoder_out_lens = self.encode_chunk(+ speech, speech_lengths, cache=cache, is_final=is_final+ )+ if isinstance(encoder_out, tuple):+ encoder_out = encoder_out[0]++ chunk_size = cache["encoder"]["chunk_size"]+ real_start_pos = chunk_size[0]++ if encoder_out_lens[0] > chunk_size[0] + chunk_size[1] + chunk_size[2]:+ assert False, print("impossible case 1 !")+ if is_final and encoder_out_lens[0] >= real_start_pos:+ real_end_pos = encoder_out_lens[0]+ elif encoder_out_lens[0] == chunk_size[0] + chunk_size[1] + chunk_size[2]:+ real_end_pos = chunk_size[0] + chunk_size[1]+ elif encoder_out_lens[0] > chunk_size[0] + chunk_size[1]:+ real_end_pos = chunk_size[0] + chunk_size[1]+ elif encoder_out_lens[0] > chunk_size[0]:+ real_end_pos = encoder_out_lens[0]+ else:+ assert False, print("impossible case 2 !")- encoder_out_accum = cache["encoder"]["encoder_out"]- if encoder_out_accum is not None:- encoder_out_accum = torch.cat((encoder_out_accum, encoder_out[:, real_start_pos:real_end_pos, :]), dim=1)- else:- encoder_out_accum = encoder_out[:, real_start_pos:real_end_pos, :]- cache["encoder"]["encoder_out"] = encoder_out_accum+ encoder_out_accum = cache["encoder"]["encoder_out"]+ if encoder_out_accum is not None:+ encoder_out_accum = torch.cat((encoder_out_accum, encoder_out[:, real_start_pos:real_end_pos, :]), dim=1)+ else:+ encoder_out_accum = encoder_out[:, real_start_pos:real_end_pos, :]+ cache["encoder"]["encoder_out"] = encoder_out_accum- if cache["encoder"]["encoder_out_lens"] is not None:- cache["encoder"]["encoder_out_lens"][0] += real_end_pos - real_start_pos- else:- cache["encoder"]["encoder_out_lens"] = encoder_out_lens- cache["encoder"]["encoder_out_lens"][0] = real_end_pos - real_start_pos+ if cache["encoder"]["encoder_out_lens"] is not None:+ cache["encoder"]["encoder_out_lens"][0] += real_end_pos - real_start_pos+ else:+ cache["encoder"]["encoder_out_lens"] = encoder_out_lens+ cache["encoder"]["encoder_out_lens"][0] = real_end_pos - real_start_posif is_final:if kwargs.get("output_dir") is not None:@@ -285,6 +299,44 @@ class SanmKWSStreaming(SanmKWS):else:return None+ def _consume_streaming_features(+ self, speech, speech_lengths, key, tokenizer, frontend, cache, **kwargs+ ):+ """Commit canonical middle windows, then all remaining valid final frames."""+ pending = cache["pending_features"]+ if speech.numel():+ pending = torch.cat((pending, speech[:, : int(speech_lengths[0]), :]), dim=1)+ left, middle, right = cache["encoder"]["chunk_size"]+ needed = middle + (0 if cache["encoder_started"] else right)+ cache["encoder"]["tail_chunk"] = False+ while pending.shape[1] >= needed:+ if not cache["encoder_started"]:+ # Only left padding is synthetic; all real frames get preprocessing.+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :left, :]+ block = pending[:, :needed, :].clone()+ self.generate_chunk(+ block, torch.tensor([needed]), key=key, tokenizer=tokenizer,+ frontend=frontend, cache=cache, **{**kwargs, "is_final": False},+ )+ cache["encoder_started"] = True+ if left + right == 0:+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :0, :]+ pending = pending[:, needed:, :]+ needed = middle+ cache["pending_features"] = pending.clone()+ if not kwargs.get("is_final", False):+ return []+ if not cache["encoder_started"]:+ if pending.shape[1] == 0:+ return []+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :left, :]+ # An empty new-feature tensor flushes preprocessed overlap without embedding it twice.+ return self.generate_chunk(+ pending.clone(), torch.tensor([pending.shape[1]]), key=key,+ tokenizer=tokenizer, frontend=frontend, cache=cache,+ **{**kwargs, "is_final": True},+ )+def inference(self,data_in,@@ -295,17 +347,27 @@ class SanmKWSStreaming(SanmKWS):cache: dict = None,**kwargs,):- """Run inference on input data.-- Args:- data_in: Input data (audio samples, file paths, or text).- data_lengths: Lengths of each input sample in the batch.- key: Sample identifiers.- tokenizer: Tokenizer instance for text encoding/decoding.- frontend: Audio frontend for feature extraction.- cache: State cache dict for streaming inference.- **kwargs: Additional keyword arguments.- """+ """Consume audio packets and decode once at the end of an utterance.++ Array/tensor packets must share the same caller-owned cache. Nonfinal+ calls return an empty result list while retaining unconsumed audio and+ features. Finalization flushes valid frames, returns detection results,+ and resets cache fields in place. File/URL inputs finalize automatically.++ Args:+ data_in: Mono audio samples, a file path, or a URL.+ data_lengths: Optional input lengths.+ key: Utterance identifiers; batch size must be one.+ tokenizer: Keyword tokenizer with token_list and seg_dict.+ frontend: Streaming audio frontend.+ cache: Caller-owned state, reused until is_final=True.+ **kwargs: Includes device, keywords, is_final, and chunk_size+ [left, middle, right] in frontend feature frames.++ Returns:+ Tuple of a result list and timing metadata. Results are emitted+ only at utterance end; this is not a per-packet wake-event API.+ """if cache is None:cache = {}keywords = kwargs.get("keywords")@@ -319,8 +381,11 @@ class SanmKWSStreaming(SanmKWS):meta_data = {}chunk_size = kwargs["chunk_size"]- chunk_stride_samples = int(chunk_size[1] * 960) # 600ms- first_chunk_padding_samples = int(chunk_size[2] * 960) # 600ms+ if len(chunk_size) != 3 or chunk_size[0] < 0 or chunk_size[1] <= 0 or chunk_size[2] < 0:+ raise ValueError("chunk_size must be [left >= 0, middle > 0, right >= 0]")+ frame_stride_samples = int(frontend.fs * frontend.frame_shift * frontend.lfr_n / 1000)+ if frame_stride_samples <= 0:+ raise ValueError("frontend frame stride must be positive")if len(cache) == 0:self.init_cache(cache, **kwargs)@@ -335,158 +400,54 @@ class SanmKWSStreaming(SanmKWS):tokenizer=tokenizer,cache=cfg,)- _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True-+ is_final = cfg["is_final"]time2 = time.perf_counter()meta_data["load_data"] = f"{time2 - time1:0.3f}"assert len(audio_sample_list) == 1, "batch_size must be set 1"-- audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0]))-- if len(audio_sample) < first_chunk_padding_samples:- print("key: {}, audio is too short for inference {}".format(key, len(audio_sample)))-- audio_sample_pre = audio_sample[0 : first_chunk_padding_samples]- feat_pre, feat_pre_lengths = extract_fbank(- [audio_sample_pre],- data_type=kwargs.get("data_type", "sound"),- frontend=frontend,- cache=cache["frontend"],- is_final=False,- )-- audio_sample = audio_sample[first_chunk_padding_samples:]- audio_chunks = int(len(audio_sample) // chunk_stride_samples)-- for i in range(audio_chunks):- if i == 0:- cache["encoder"]["feats"][:, chunk_size[2] :, :] = feat_pre-- kwargs["is_final"] = False- audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples]-- if kwargs["is_final"] and len(audio_sample_i) < 960:- cache["encoder"]["tail_chunk"] = True- speech = cache["encoder"]["feats"]- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(- speech.device- )- else:- # extract fbank feats- speech, speech_lengths = extract_fbank(- [audio_sample_i],- data_type=kwargs.get("data_type", "sound"),- frontend=frontend,- cache=cache["frontend"],- is_final=kwargs["is_final"],- )- time3 = time.perf_counter()- meta_data["extract_feat"] = f"{time3 - time2:0.3f}"- meta_data["batch_data_time"] = (- speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000- )-- results_chunk_i = self.generate_chunk(- speech,- speech_lengths,- key=key,- tokenizer=tokenizer,- cache=cache,- frontend=frontend,- **kwargs,- )-- # results_chunk_i must be None when is_final=False- assert results_chunk_i is None-- # process tail samples- tail_audio_sample = audio_sample[ audio_chunks * chunk_stride_samples: ]- if len(tail_audio_sample) < 960:- kwargs["is_final"] = _is_final- cache["encoder"]["tail_chunk"] = True- speech = cache["encoder"]["feats"]- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(- speech.device- )- results_chunk_tail = self.generate_chunk(- speech,- speech_lengths,- key=key,- tokenizer=tokenizer,- cache=cache,- frontend=frontend,- **kwargs,- )- elif len(tail_audio_sample) <= first_chunk_padding_samples:- kwargs["is_final"] = _is_final- # extract fbank feats- # cache["encoder"]["tail_chunk"] = True # cannot be true+ incoming = audio_sample_list[0]+ if incoming.numel():+ meta_data["batch_data_time"] = incoming.numel() / frontend.fs+ audio = torch.cat((cache["prev_samples"], incoming))++ # Frontend calls and encoder windows do not depend on caller packet sizes.+ offset = 0+ while True:+ frames = chunk_size[1] + (0 if cache["frontend_started"] else chunk_size[2])+ sample_count = frames * frame_stride_samples+ if len(audio) - offset < sample_count:+ breakspeech, speech_lengths = extract_fbank(- [ tail_audio_sample ],+ [audio[offset : offset + sample_count]],data_type=kwargs.get("data_type", "sound"),frontend=frontend,cache=cache["frontend"],- is_final=kwargs["is_final"],+ is_final=False,)- results_chunk_tail = self.generate_chunk(- speech,- speech_lengths,- key=key,- tokenizer=tokenizer,- cache=cache,- frontend=frontend,- **kwargs,+ cache["frontend_started"] = True+ self._consume_streaming_features(+ speech, speech_lengths, key, tokenizer, frontend, cache,+ **{**kwargs, "is_final": False},)- elif len(tail_audio_sample) > first_chunk_padding_samples and \- len(tail_audio_sample) < chunk_stride_samples:- kwargs["is_final"] = False- # extract fbank feats+ offset += sample_count++ cache["prev_samples"] = audio[offset:].clone()+ results = []+ if is_final:speech, speech_lengths = extract_fbank(- [ tail_audio_sample ],+ [cache["prev_samples"]],data_type=kwargs.get("data_type", "sound"),frontend=frontend,cache=cache["frontend"],- is_final=kwargs["is_final"],+ is_final=True,)- results_chunk = self.generate_chunk(- speech,- speech_lengths,- key=key,- tokenizer=tokenizer,- cache=cache,- frontend=frontend,- **kwargs,+ results = self._consume_streaming_features(+ speech, speech_lengths, key, tokenizer, frontend, cache,+ **{**kwargs, "is_final": True},)- # results_chunk must be None when is_final=False- assert results_chunk is None-- # push tail chunk- kwargs["is_final"] = _is_final- cache["encoder"]["tail_chunk"] = True- speech = cache["encoder"]["feats"]- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(- speech.device- )- results_chunk_tail = self.generate_chunk(- speech,- speech_lengths,- key=key,- tokenizer=tokenizer,- cache=cache,- frontend=frontend,- **kwargs,- )-- result = results_chunk_tail-- if _is_final:self.init_cache(cache, **kwargs)- if kwargs.get("output_dir"):- if not hasattr(self, "writer"):- self.writer = DatadirWriter(kwargs.get("output_dir"))-- return result, meta_data+ meta_data["extract_feat"] = f"{time.perf_counter() - time2:0.3f}"+ return results, meta_datadef export(self, **kwargs):"""Export.diff --git a/tests/test_kws_streaming_continuity.py b/tests/test_kws_streaming_continuity.pynew file mode 100644index 000000000..d019f99dc--- /dev/null+++ b/tests/test_kws_streaming_continuity.py@@ -0,0 +1,454 @@+"""KWS public streaming contracts with production overlap and no model weights."""++import tempfile+import unittest+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import patch++import soundfile as sf+import torch++from funasr.auto.auto_model import AutoModel+from funasr.frontends.wav_frontend import WavFrontendOnline+from funasr.models.sanm_kws_streaming.model import SanmKWSStreaming+from funasr.models.scama.encoder import SANMEncoderChunkOpt+from funasr.models.transformer.embedding import StreamSinusoidalPositionEncoder+from funasr.utils.load_utils import extract_fbank+++class MarkerFrontend:+ """One feature per complete 960-sample block; never invent short-input frames."""++ fs, frame_shift, lfr_n, n_mels, lfr_m = 16000, 10, 6, 1, 1++ def __call__(self, audio, lengths, cache, is_final=False, **kwargs):+ samples = torch.cat((cache.get("samples", audio[:, :0]), audio), dim=1)+ count = samples.shape[1] // 960+ features = samples[:, : count * 960 : 960, None].clone()+ cache["samples"] = samples[:, count * 960 :].clone()+ return features, torch.tensor([count])+++class IdentityChunkEncoder(SANMEncoderChunkOpt):+ """Keep the real forward_chunk and overlap cache, replacing acoustic layers."""++ def __init__(self):+ torch.nn.Module.__init__(self)+ self.embed = None+ self.encoders0 = torch.nn.ModuleList()+ self.encoders = torch.nn.ModuleList()+ self.normalize_before = False++ def output_size(self):+ return 1+++class RecordingDecoder:+ def __init__(self, ctc, **kwargs):+ self.state = ctc++ def decode(self, features):+ self.state.frames.append(features.detach().clone())+ return self.state.detected, "wake", 0.9+++class FourDimFrontend(MarkerFrontend):+ n_mels = 4++ def __call__(self, *args, **kwargs):+ features, lengths = super().__call__(*args, **kwargs)+ return features.repeat(1, 1, 4), lengths+++class ContextLayer(torch.nn.Module):+ def forward_chunk(self, features, state, chunk_size, look_back):+ return features + 0.01 * features.mean(dim=1, keepdim=True), state+++class PositionedEncoder(IdentityChunkEncoder):+ def __init__(self, contextual=False):+ super().__init__()+ self.embed = StreamSinusoidalPositionEncoder()+ if contextual:+ self.encoders0.append(ContextLayer())+ self.positions = []++ def output_size(self):+ return 4++ def forward_chunk(self, features, lengths, cache, **kwargs):+ before = cache["start_idx"]+ result = super().forward_chunk(features, lengths, cache, **kwargs)+ self.positions.append((before, cache["start_idx"]))+ return result+++class LightweightKws(SanmKWSStreaming):+ def __init__(self, detected=True):+ torch.nn.Module.__init__(self)+ self.anchor = torch.nn.Parameter(torch.zeros(1))+ self.encoder = IdentityChunkEncoder()+ self.specaug = self.normalize = None+ self.ctc = SimpleNamespace(frames=[], detected=detected)+ self.eval()+++class KwsStreamingContinuityTest(unittest.TestCase):+ def setUp(self):+ p = patch("funasr.utils.kws_utils.KwsCtcPrefixDecoder", RecordingDecoder)+ p.start()+ self.addCleanup(p.stop)++ def setup_model(self, frontend=None, detected=True, chunk_size=None):+ model = LightweightKws(detected)+ frontend = frontend or MarkerFrontend()+ config = {+ "device": "cpu",+ "batch_size": 1,+ "disable_pbar": True,+ "keywords": "wake",+ "chunk_size": chunk_size or [4, 8, 4],+ "encoder_conf": {"output_size": 1},+ "frontend_conf": {"n_mels": frontend.n_mels, "lfr_m": frontend.lfr_m},+ "frontend": frontend,+ "tokenizer": SimpleNamespace(token_list=["wake"], seg_dict={}),+ }+ wrapper = AutoModel.__new__(AutoModel)+ wrapper.model = model+ wrapper.kwargs = config+ wrapper.vad_model = wrapper.punc_model = None+ wrapper._store_base_configs()+ self.addCleanup(+ lambda: model.writer.close() if hasattr(model, "writer") else None+ )+ return model, wrapper, config++ @staticmethod+ def markers(frames):+ return torch.arange(1, frames + 1, dtype=torch.float32).repeat_interleave(960)++ def stream(self, wrapper, audio, pieces, empty_flush=False, **kwargs):+ cache = {}+ offset, index = 0, 0+ while offset < len(audio):+ end = min(len(audio), offset + pieces[index % len(pieces)])+ final = end == len(audio) and not empty_flush+ result = wrapper.generate(+ input=audio[offset:end],+ cache=cache,+ key="sample",+ is_final=final,+ **kwargs+ )+ if not final:+ self.assertEqual(result, [])+ self.assertFalse(cache["encoder"]["tail_chunk"])+ offset, index = end, index + 1+ if empty_flush or len(audio) == 0:+ result = wrapper.generate(+ input=audio[:0], cache=cache, key="sample", is_final=True, **kwargs+ )+ self.assertEqual(cache["prev_samples"].numel(), 0)+ self.assertIsNone(cache["encoder"]["encoder_out"])+ return result++ def test_public_nonfinal_contract_and_unflushed_cache(self):+ model, wrapper, config = self.setup_model()+ cache = {}+ results, metadata = model.inference(+ [self.markers(12)], key=["sample"], cache=cache, is_final=False, **config+ )+ self.assertEqual(results, [])+ self.assertIsInstance(metadata, dict)+ self.assertFalse(cache["encoder"]["tail_chunk"])+ self.assertEqual(model.ctc.frames, [])+ self.assertEqual(+ wrapper.generate(+ input=self.markers(12), cache={}, key="sample", is_final=False+ ),+ [],+ )++ def test_complete_inputs_keep_each_valid_frame_once(self):+ for frames in (1, 3, 4, 5, 7, 8, 9, 12, 13, 16, 20, 21, 32, 37):+ with self.subTest(frames=frames):+ model, wrapper, _ = self.setup_model()+ self.assertEqual(+ self.stream(wrapper, self.markers(frames), [100000]),+ [{"key": "sample", "text": "detected wake 0.9"}],+ )+ self.assertEqual(len(model.ctc.frames), 1)+ torch.testing.assert_close(+ model.ctc.frames[0][:, 0],+ torch.arange(1, frames + 1, dtype=torch.float32),+ )++ def test_partitions_and_empty_flush_preserve_marker_frames(self):+ for pieces in ([959], [960], [3840], [7680], [1, 959, 3001, 7000]):+ for empty_flush in (False, True):+ with self.subTest(pieces=pieces, empty_flush=empty_flush):+ model, wrapper, _ = self.setup_model()+ self.stream(wrapper, self.markers(37), pieces, empty_flush)+ self.assertEqual(len(model.ctc.frames), 1)+ torch.testing.assert_close(+ model.ctc.frames[0][:, 0],+ torch.arange(1, 38, dtype=torch.float32),+ )++ def test_empty_and_insufficient_audio_do_not_fabricate_detection(self):+ for length in (0, 1, 399, 959):+ with self.subTest(length=length):+ model, wrapper, _ = self.setup_model()+ self.assertEqual(self.stream(wrapper, torch.ones(length), [100]), [])+ self.assertEqual(model.ctc.frames, [])++ def test_final_output_once_and_second_utterance_starts_clean(self):+ with tempfile.TemporaryDirectory() as directory:+ model, wrapper, _ = self.setup_model(detected=False)+ cache = {}+ for key in ("first", "second"):+ self.assertEqual(+ wrapper.generate(+ input=self.markers(12),+ cache=cache,+ key=key,+ is_final=False,+ output_dir=directory,+ ),+ [],+ )+ if key == "first":+ self.assertFalse((Path(directory) / "detect").exists())+ result = wrapper.generate(+ input=self.markers(5),+ cache=cache,+ key=key,+ is_final=True,+ output_dir=directory,+ )+ self.assertEqual(result, [{"key": key, "text": "rejected"}])+ self.assertIsNone(cache["encoder"]["encoder_out"])+ self.assertEqual(+ (Path(directory) / "detect").read_text(),+ "first rejected\nsecond rejected\n",+ )+ self.assertEqual(len(model.ctc.frames), 2)+ torch.testing.assert_close(model.ctc.frames[0], model.ctc.frames[1])++ def test_file_input_finalizes_even_when_flag_is_false(self):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "markers.wav"+ sf.write(path, self.markers(21).numpy(), 16000, subtype="FLOAT")+ model, wrapper, _ = self.setup_model()+ self.assertEqual(+ wrapper.generate(+ input=str(path), cache={}, key="sample", is_final=False+ ),+ [{"key": "sample", "text": "detected wake 0.9"}],+ )+ torch.testing.assert_close(+ model.ctc.frames[0][:, 0], torch.arange(1, 22, dtype=torch.float32)+ )++ def test_real_frontend_frames_match_complete_audio_and_partitions(self):+ def frontend():+ return WavFrontendOnline(n_mels=8, lfr_m=7, lfr_n=6, dither=0.0)++ for length in (+ 0,+ 399,+ 400,+ 559,+ 560,+ 879,+ 960,+ 3839,+ 3840,+ 3841,+ 11520,+ 12479,+ 12480,+ 15360,+ 15361,+ 19199,+ 40000,+ ):+ with self.subTest(length=length):+ time = torch.arange(length, dtype=torch.float32) / 16000+ audio = 0.1 * torch.sin(2 * torch.pi * (300 * time + 100 * time**2))+ expected, lengths = extract_fbank(+ [audio], frontend=frontend(), cache={}, is_final=True+ )+ count = int(lengths[0]) if lengths.numel() else 0+ fbank_count = max(0, 1 + (length - 400) // 160)+ self.assertEqual(count, (fbank_count + 5) // 6)+ for pieces in ([100000], [1, 959, 3001, 7000]):+ model, wrapper, _ = self.setup_model(frontend())+ result = self.stream(wrapper, audio, pieces, empty_flush=True)+ if not count:+ self.assertEqual(result, [])+ self.assertEqual(model.ctc.frames, [])+ else:+ self.assertEqual(len(model.ctc.frames), 1)+ torch.testing.assert_close(+ model.ctc.frames[0],+ expected[0, :count],+ rtol=1e-5,+ atol=1e-5,+ )++ def test_real_position_encoding_covers_prefix_and_never_reembeds_overlap(self):+ for pieces in ([100000], [1, 959, 3001, 7000]):+ with self.subTest(pieces=pieces):+ model, wrapper, _ = self.setup_model(FourDimFrontend())+ model.encoder = PositionedEncoder()+ self.stream(wrapper, self.markers(37), pieces, empty_flush=True)+ raw = torch.arange(1, 38, dtype=torch.float32)[None, :, None].repeat(+ 1, 1, 4+ )+ expected = StreamSinusoidalPositionEncoder()(raw * 2, {"start_idx": 0})+ torch.testing.assert_close(model.ctc.frames[0], expected[0])+ self.assertEqual(+ sum(end - start for start, end in model.encoder.positions), 37+ )++ def test_context_sensitive_windows_are_independent_of_packet_and_final_boundaries(+ self,+ ):+ for frames in (12, 13, 19, 20, 21, 28, 37):+ reference = None+ for pieces, empty_flush in (+ ([100000], False),+ ([100000], True),+ ([959, 3841], False),+ ):+ with self.subTest(+ frames=frames, pieces=pieces, empty_flush=empty_flush+ ):+ model, wrapper, _ = self.setup_model(FourDimFrontend())+ model.encoder = PositionedEncoder(contextual=True)+ self.stream(wrapper, self.markers(frames), pieces, empty_flush)+ observed = model.ctc.frames[0]+ if reference is None:+ reference = observed+ else:+ torch.testing.assert_close(observed, reference)++ def test_asymmetric_and_zero_overlap_chunk_shapes(self):+ for chunk_size in ([0, 8, 4], [2, 8, 4], [4, 8, 0], [0, 8, 0]):+ with self.subTest(chunk_size=chunk_size):+ model, wrapper, _ = self.setup_model(chunk_size=chunk_size)+ self.stream(wrapper, self.markers(24), [959, 3841], empty_flush=True)+ torch.testing.assert_close(+ model.ctc.frames[0][:, 0], torch.arange(1, 25, dtype=torch.float32)+ )++ def test_real_sanm_convolution_handles_exact_stride_endings(self):+ for chunk_size in ([0, 8, 0], [4, 8, 0], [4, 8, 4]):+ reference = None+ for pieces in ([100000], [959, 3841]):+ with self.subTest(chunk_size=chunk_size, pieces=pieces):+ model, wrapper, _ = self.setup_model(+ FourDimFrontend(), chunk_size=chunk_size+ )+ torch.manual_seed(23)+ model.encoder = SANMEncoderChunkOpt(+ input_size=4,+ output_size=4,+ attention_heads=2,+ linear_units=8,+ num_blocks=1,+ kernel_size=3,+ input_layer="pe_online",+ dropout_rate=0.0,+ positional_dropout_rate=0.0,+ attention_dropout_rate=0.0,+ ).eval()+ self.stream(wrapper, self.markers(24), pieces, empty_flush=True)+ observed = model.ctc.frames[0]+ self.assertEqual(observed.shape, (24, 4))+ if reference is None:+ reference = observed+ else:+ torch.testing.assert_close(observed, reference)++ def test_short_final_frontend_waveforms_align_to_real_samples(self):+ for lfr_m, lfr_n in ((7, 6), (5, 1)):+ for length in (0, 399, 400, 559, 560, 879, 960, 12479):+ for pieces in ([100000], [1, 399, 401]):+ with self.subTest(lfr=(lfr_m, lfr_n), length=length, pieces=pieces):+ frontend = WavFrontendOnline(+ n_mels=8, lfr_m=lfr_m, lfr_n=lfr_n, dither=0.0+ )+ audio = torch.linspace(-0.1, 0.1, length)+ cache, offset, emitted, index = {}, 0, 0, 0+ while offset < length:+ end = min(length, offset + pieces[index % len(pieces)])+ features, _ = frontend(+ audio[None, offset:end],+ [end - offset],+ cache=cache,+ is_final=False,+ return_waveform=True,+ )+ if features.ndim == 3 and features.shape[1]:+ count = features.shape[1]+ start = emitted * lfr_n * 160+ stop = start + (count - 1) * lfr_n * 160 + 400+ torch.testing.assert_close(+ cache["aligned_waveforms"], audio[None, start:stop]+ )+ emitted += count+ offset, index = end, index + 1+ features, _ = frontend(+ audio[None, :0],+ [0],+ cache=cache,+ is_final=True,+ return_waveform=True,+ )+ if features.ndim == 3 and features.shape[1]:+ count = features.shape[1]+ start = emitted * lfr_n * 160+ stop = start + (count - 1) * lfr_n * 160 + 400+ self.assertLessEqual(stop, length)+ torch.testing.assert_close(+ cache["aligned_waveforms"], audio[None, start:stop]+ )+ emitted += count+ fbank_count = max(0, 1 + (length - 400) // 160)+ self.assertEqual(emitted, (fbank_count + lfr_n - 1) // lfr_n)++ def test_short_nonempty_eos_aligns_final_waveform(self):+ for lfr_m, lfr_n in ((7, 6), (5, 1)):+ for length in (399, 400, 559, 560, 879, 960, 12479):+ with self.subTest(lfr=(lfr_m, lfr_n), length=length):+ frontend = WavFrontendOnline(+ n_mels=8, lfr_m=lfr_m, lfr_n=lfr_n, dither=0.0+ )+ audio = torch.linspace(-0.1, 0.1, length)+ cache = {}+ features, _ = frontend(+ audio[None],+ [length],+ cache=cache,+ is_final=True,+ return_waveform=True,+ )+ fbank_count = max(0, 1 + (length - 400) // 160)+ count = (fbank_count + lfr_n - 1) // lfr_n+ if not count:+ self.assertEqual(features.numel(), 0)+ self.assertEqual(cache["aligned_waveforms"].numel(), 0)+ else:+ self.assertEqual(features.shape[1], count)+ stop = (count - 1) * lfr_n * 160 + 400+ self.assertLessEqual(stop, length)+ torch.testing.assert_close(+ cache["aligned_waveforms"], audio[None, :stop]+ )+++if __name__ == "__main__":+ unittest.main()