Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(kws): preserve streaming frames across packet and EOS boundaries

zhifu gaod83a92475da9
diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.yml
index 0102edf11..55b2f4cb0 100644
--- a/.github/workflows/test-kws-output.yml
+++ b/.github/workflows/test-kws-output.yml
@@ -7,9 +7,16 @@ on:
- "funasr/models/fsmn_kws_mt/**"
- "funasr/models/sanm_kws/**"
- "funasr/models/sanm_kws_streaming/**"
+ - "funasr/frontends/wav_frontend.py"
+ - "funasr/auto/auto_model.py"
- "funasr/utils/datadir_writer.py"
- "funasr/utils/kws_utils.py"
- "tests/test_kws_optional_output.py"
+ - "tests/test_kws_streaming_continuity.py"
+ - "tests/test_kws_utils.py"
+ - "tests/test_fsmn_vad_streaming_buffers.py"
+ - "tests/test_dynamic_streaming_vad.py"
+ - "tests/test_fsmn_vad_dynamic_silence.py"
- ".github/workflows/test-kws-output.yml"
push:
branches: [main]
@@ -18,9 +25,16 @@ on:
- "funasr/models/fsmn_kws_mt/**"
- "funasr/models/sanm_kws/**"
- "funasr/models/sanm_kws_streaming/**"
+ - "funasr/frontends/wav_frontend.py"
+ - "funasr/auto/auto_model.py"
- "funasr/utils/datadir_writer.py"
- "funasr/utils/kws_utils.py"
- "tests/test_kws_optional_output.py"
+ - "tests/test_kws_streaming_continuity.py"
+ - "tests/test_kws_utils.py"
+ - "tests/test_fsmn_vad_streaming_buffers.py"
+ - "tests/test_dynamic_streaming_vad.py"
+ - "tests/test_fsmn_vad_dynamic_silence.py"
- ".github/workflows/test-kws-output.yml"
permissions:
@@ -30,6 +44,9 @@ jobs:
kws-output:
runs-on: ubuntu-latest
timeout-minutes: 15
+ env:
+ OMP_NUM_THREADS: "1"
+ MKL_NUM_THREADS: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
@@ -38,7 +55,14 @@ jobs:
cache: pip
- name: Install CPU test dependencies
run: |
- python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- python -m pip install "numpy<2" "kaldiio>=2.17.0" librosa "rapidfuzz>=3.0.0" six
- - name: Check KWS results and optional file output without model weights
- run: python -m unittest discover -s tests -p test_kws_optional_output.py -v
+ python -m pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu
+ python -m pip install "numpy<2" "kaldiio>=2.17.0" librosa "rapidfuzz>=3.0.0" six pytest omegaconf hydra-core pyyaml tqdm
+ - name: Check KWS continuity and related frontend contracts without model weights
+ run: |
+ python -m pytest -q \
+ tests/test_kws_optional_output.py \
+ tests/test_kws_streaming_continuity.py \
+ tests/test_kws_utils.py \
+ tests/test_fsmn_vad_streaming_buffers.py \
+ tests/test_dynamic_streaming_vad.py \
+ tests/test_fsmn_vad_dynamic_silence.py
diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py
index 3dfee293e..45f34095e 100644
--- a/funasr/frontends/wav_frontend.py
+++ b/funasr/frontends/wav_frontend.py
@@ -549,7 +549,7 @@ class WavFrontendOnline(nn.Module):
feats[i][0, :].unsqueeze(dim=0).repeat((self.lfr_m - 1) // 2, 1)
)
# need the number of the input frames + self.lfr_splice_cache[0].shape[0] is greater than self.lfr_m
- if feats_lengths[0] + cache["lfr_splice_cache"][0].shape[0] >= self.lfr_m:
+ if is_final or feats_lengths[0] + cache["lfr_splice_cache"][0].shape[0] >= self.lfr_m:
lfr_splice_cache_tensor = torch.stack(cache["lfr_splice_cache"]) # B T D
feats = torch.cat((lfr_splice_cache_tensor, feats), dim=1)
feats_lengths += lfr_splice_cache_tensor[0].shape[0]
@@ -591,7 +591,7 @@ class WavFrontendOnline(nn.Module):
)
return torch.empty(0), feats_lengths
else:
- if is_final:
+ if is_final and cache["lfr_splice_cache"]:
cache["waveforms"] = (
waveforms
if cache["reserve_waveforms"].numel() == 0
diff --git a/funasr/models/sanm_kws_streaming/model.py b/funasr/models/sanm_kws_streaming/model.py
index 051be0c21..12ff51b01 100644
--- a/funasr/models/sanm_kws_streaming/model.py
+++ b/funasr/models/sanm_kws_streaming/model.py
@@ -198,6 +198,9 @@ class SanmKWSStreaming(SanmKWS):
cache["decoder"] = cache_decoder
cache["frontend"] = {}
cache["prev_samples"] = torch.empty(0)
+ cache["pending_features"] = torch.empty((batch_size, 0, feats_dims))
+ cache["frontend_started"] = False
+ cache["encoder_started"] = False
return cache
@@ -224,40 +227,51 @@ class SanmKWSStreaming(SanmKWS):
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
- # Encoder
is_final = kwargs.get("is_final", False)
- encoder_out, encoder_out_lens = self.encode_chunk(
- speech, speech_lengths, cache=cache, is_final=is_final
+ no_pending_context = (
+ is_final
+ and speech.shape[1] == 0
+ and cache["encoder"]["chunk_size"][2] == 0
+ and cache["encoder"]["encoder_out"] is not None
)
- if isinstance(encoder_out, tuple):
- encoder_out = encoder_out[0]
-
- chunk_size = cache["encoder"]["chunk_size"]
- real_start_pos = chunk_size[0]
-
- if encoder_out_lens[0] > chunk_size[0] + chunk_size[1] + chunk_size[2]:
- assert False, print("impossible case 1 !")
- if encoder_out_lens[0] == chunk_size[0] + chunk_size[1] + chunk_size[2]:
- real_end_pos = chunk_size[0] + chunk_size[1]
- elif encoder_out_lens[0] > chunk_size[0] + chunk_size[1]:
- real_end_pos = chunk_size[0] + chunk_size[1]
- elif encoder_out_lens[0] > chunk_size[0]:
- real_end_pos = encoder_out_lens[0]
+ if no_pending_context:
+ # Zero-right-context EOS only decodes output already committed.
+ encoder_out_accum = cache["encoder"]["encoder_out"]
else:
- assert False, print("impossible case 2 !")
+ encoder_out, encoder_out_lens = self.encode_chunk(
+ speech, speech_lengths, cache=cache, is_final=is_final
+ )
+ if isinstance(encoder_out, tuple):
+ encoder_out = encoder_out[0]
+
+ chunk_size = cache["encoder"]["chunk_size"]
+ real_start_pos = chunk_size[0]
+
+ if encoder_out_lens[0] > chunk_size[0] + chunk_size[1] + chunk_size[2]:
+ assert False, print("impossible case 1 !")
+ if is_final and encoder_out_lens[0] >= real_start_pos:
+ real_end_pos = encoder_out_lens[0]
+ elif encoder_out_lens[0] == chunk_size[0] + chunk_size[1] + chunk_size[2]:
+ real_end_pos = chunk_size[0] + chunk_size[1]
+ elif encoder_out_lens[0] > chunk_size[0] + chunk_size[1]:
+ real_end_pos = chunk_size[0] + chunk_size[1]
+ elif encoder_out_lens[0] > chunk_size[0]:
+ real_end_pos = encoder_out_lens[0]
+ else:
+ assert False, print("impossible case 2 !")
- encoder_out_accum = cache["encoder"]["encoder_out"]
- if encoder_out_accum is not None:
- encoder_out_accum = torch.cat((encoder_out_accum, encoder_out[:, real_start_pos:real_end_pos, :]), dim=1)
- else:
- encoder_out_accum = encoder_out[:, real_start_pos:real_end_pos, :]
- cache["encoder"]["encoder_out"] = encoder_out_accum
+ encoder_out_accum = cache["encoder"]["encoder_out"]
+ if encoder_out_accum is not None:
+ encoder_out_accum = torch.cat((encoder_out_accum, encoder_out[:, real_start_pos:real_end_pos, :]), dim=1)
+ else:
+ encoder_out_accum = encoder_out[:, real_start_pos:real_end_pos, :]
+ cache["encoder"]["encoder_out"] = encoder_out_accum
- if cache["encoder"]["encoder_out_lens"] is not None:
- cache["encoder"]["encoder_out_lens"][0] += real_end_pos - real_start_pos
- else:
- cache["encoder"]["encoder_out_lens"] = encoder_out_lens
- cache["encoder"]["encoder_out_lens"][0] = real_end_pos - real_start_pos
+ if cache["encoder"]["encoder_out_lens"] is not None:
+ cache["encoder"]["encoder_out_lens"][0] += real_end_pos - real_start_pos
+ else:
+ cache["encoder"]["encoder_out_lens"] = encoder_out_lens
+ cache["encoder"]["encoder_out_lens"][0] = real_end_pos - real_start_pos
if is_final:
if kwargs.get("output_dir") is not None:
@@ -285,6 +299,44 @@ class SanmKWSStreaming(SanmKWS):
else:
return None
+ def _consume_streaming_features(
+ self, speech, speech_lengths, key, tokenizer, frontend, cache, **kwargs
+ ):
+ """Commit canonical middle windows, then all remaining valid final frames."""
+ pending = cache["pending_features"]
+ if speech.numel():
+ pending = torch.cat((pending, speech[:, : int(speech_lengths[0]), :]), dim=1)
+ left, middle, right = cache["encoder"]["chunk_size"]
+ needed = middle + (0 if cache["encoder_started"] else right)
+ cache["encoder"]["tail_chunk"] = False
+ while pending.shape[1] >= needed:
+ if not cache["encoder_started"]:
+ # Only left padding is synthetic; all real frames get preprocessing.
+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :left, :]
+ block = pending[:, :needed, :].clone()
+ self.generate_chunk(
+ block, torch.tensor([needed]), key=key, tokenizer=tokenizer,
+ frontend=frontend, cache=cache, **{**kwargs, "is_final": False},
+ )
+ cache["encoder_started"] = True
+ if left + right == 0:
+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :0, :]
+ pending = pending[:, needed:, :]
+ needed = middle
+ cache["pending_features"] = pending.clone()
+ if not kwargs.get("is_final", False):
+ return []
+ if not cache["encoder_started"]:
+ if pending.shape[1] == 0:
+ return []
+ cache["encoder"]["feats"] = cache["encoder"]["feats"][:, :left, :]
+ # An empty new-feature tensor flushes preprocessed overlap without embedding it twice.
+ return self.generate_chunk(
+ pending.clone(), torch.tensor([pending.shape[1]]), key=key,
+ tokenizer=tokenizer, frontend=frontend, cache=cache,
+ **{**kwargs, "is_final": True},
+ )
+
def inference(
self,
data_in,
@@ -295,17 +347,27 @@ class SanmKWSStreaming(SanmKWS):
cache: dict = None,
**kwargs,
):
- """Run inference on input data.
-
- Args:
- data_in: Input data (audio samples, file paths, or text).
- data_lengths: Lengths of each input sample in the batch.
- key: Sample identifiers.
- tokenizer: Tokenizer instance for text encoding/decoding.
- frontend: Audio frontend for feature extraction.
- cache: State cache dict for streaming inference.
- **kwargs: Additional keyword arguments.
- """
+ """Consume audio packets and decode once at the end of an utterance.
+
+ Array/tensor packets must share the same caller-owned cache. Nonfinal
+ calls return an empty result list while retaining unconsumed audio and
+ features. Finalization flushes valid frames, returns detection results,
+ and resets cache fields in place. File/URL inputs finalize automatically.
+
+ Args:
+ data_in: Mono audio samples, a file path, or a URL.
+ data_lengths: Optional input lengths.
+ key: Utterance identifiers; batch size must be one.
+ tokenizer: Keyword tokenizer with token_list and seg_dict.
+ frontend: Streaming audio frontend.
+ cache: Caller-owned state, reused until is_final=True.
+ **kwargs: Includes device, keywords, is_final, and chunk_size
+ [left, middle, right] in frontend feature frames.
+
+ Returns:
+ Tuple of a result list and timing metadata. Results are emitted
+ only at utterance end; this is not a per-packet wake-event API.
+ """
if cache is None:
cache = {}
keywords = kwargs.get("keywords")
@@ -319,8 +381,11 @@ class SanmKWSStreaming(SanmKWS):
meta_data = {}
chunk_size = kwargs["chunk_size"]
- chunk_stride_samples = int(chunk_size[1] * 960) # 600ms
- first_chunk_padding_samples = int(chunk_size[2] * 960) # 600ms
+ if len(chunk_size) != 3 or chunk_size[0] < 0 or chunk_size[1] <= 0 or chunk_size[2] < 0:
+ raise ValueError("chunk_size must be [left >= 0, middle > 0, right >= 0]")
+ frame_stride_samples = int(frontend.fs * frontend.frame_shift * frontend.lfr_n / 1000)
+ if frame_stride_samples <= 0:
+ raise ValueError("frontend frame stride must be positive")
if len(cache) == 0:
self.init_cache(cache, **kwargs)
@@ -335,158 +400,54 @@ class SanmKWSStreaming(SanmKWS):
tokenizer=tokenizer,
cache=cfg,
)
- _is_final = cfg["is_final"] # if data_in is a file or url, set is_final=True
-
+ is_final = cfg["is_final"]
time2 = time.perf_counter()
meta_data["load_data"] = f"{time2 - time1:0.3f}"
assert len(audio_sample_list) == 1, "batch_size must be set 1"
-
- audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0]))
-
- if len(audio_sample) < first_chunk_padding_samples:
- print("key: {}, audio is too short for inference {}".format(key, len(audio_sample)))
-
- audio_sample_pre = audio_sample[0 : first_chunk_padding_samples]
- feat_pre, feat_pre_lengths = extract_fbank(
- [audio_sample_pre],
- data_type=kwargs.get("data_type", "sound"),
- frontend=frontend,
- cache=cache["frontend"],
- is_final=False,
- )
-
- audio_sample = audio_sample[first_chunk_padding_samples:]
- audio_chunks = int(len(audio_sample) // chunk_stride_samples)
-
- for i in range(audio_chunks):
- if i == 0:
- cache["encoder"]["feats"][:, chunk_size[2] :, :] = feat_pre
-
- kwargs["is_final"] = False
- audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples]
-
- if kwargs["is_final"] and len(audio_sample_i) < 960:
- cache["encoder"]["tail_chunk"] = True
- speech = cache["encoder"]["feats"]
- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(
- speech.device
- )
- else:
- # extract fbank feats
- speech, speech_lengths = extract_fbank(
- [audio_sample_i],
- data_type=kwargs.get("data_type", "sound"),
- frontend=frontend,
- cache=cache["frontend"],
- is_final=kwargs["is_final"],
- )
- time3 = time.perf_counter()
- meta_data["extract_feat"] = f"{time3 - time2:0.3f}"
- meta_data["batch_data_time"] = (
- speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000
- )
-
- results_chunk_i = self.generate_chunk(
- speech,
- speech_lengths,
- key=key,
- tokenizer=tokenizer,
- cache=cache,
- frontend=frontend,
- **kwargs,
- )
-
- # results_chunk_i must be None when is_final=False
- assert results_chunk_i is None
-
- # process tail samples
- tail_audio_sample = audio_sample[ audio_chunks * chunk_stride_samples: ]
- if len(tail_audio_sample) < 960:
- kwargs["is_final"] = _is_final
- cache["encoder"]["tail_chunk"] = True
- speech = cache["encoder"]["feats"]
- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(
- speech.device
- )
- results_chunk_tail = self.generate_chunk(
- speech,
- speech_lengths,
- key=key,
- tokenizer=tokenizer,
- cache=cache,
- frontend=frontend,
- **kwargs,
- )
- elif len(tail_audio_sample) <= first_chunk_padding_samples:
- kwargs["is_final"] = _is_final
- # extract fbank feats
- # cache["encoder"]["tail_chunk"] = True # cannot be true
+ incoming = audio_sample_list[0]
+ if incoming.numel():
+ meta_data["batch_data_time"] = incoming.numel() / frontend.fs
+ audio = torch.cat((cache["prev_samples"], incoming))
+
+ # Frontend calls and encoder windows do not depend on caller packet sizes.
+ offset = 0
+ while True:
+ frames = chunk_size[1] + (0 if cache["frontend_started"] else chunk_size[2])
+ sample_count = frames * frame_stride_samples
+ if len(audio) - offset < sample_count:
+ break
speech, speech_lengths = extract_fbank(
- [ tail_audio_sample ],
+ [audio[offset : offset + sample_count]],
data_type=kwargs.get("data_type", "sound"),
frontend=frontend,
cache=cache["frontend"],
- is_final=kwargs["is_final"],
+ is_final=False,
)
- results_chunk_tail = self.generate_chunk(
- speech,
- speech_lengths,
- key=key,
- tokenizer=tokenizer,
- cache=cache,
- frontend=frontend,
- **kwargs,
+ cache["frontend_started"] = True
+ self._consume_streaming_features(
+ speech, speech_lengths, key, tokenizer, frontend, cache,
+ **{**kwargs, "is_final": False},
)
- elif len(tail_audio_sample) > first_chunk_padding_samples and \
- len(tail_audio_sample) < chunk_stride_samples:
- kwargs["is_final"] = False
- # extract fbank feats
+ offset += sample_count
+
+ cache["prev_samples"] = audio[offset:].clone()
+ results = []
+ if is_final:
speech, speech_lengths = extract_fbank(
- [ tail_audio_sample ],
+ [cache["prev_samples"]],
data_type=kwargs.get("data_type", "sound"),
frontend=frontend,
cache=cache["frontend"],
- is_final=kwargs["is_final"],
+ is_final=True,
)
- results_chunk = self.generate_chunk(
- speech,
- speech_lengths,
- key=key,
- tokenizer=tokenizer,
- cache=cache,
- frontend=frontend,
- **kwargs,
+ results = self._consume_streaming_features(
+ speech, speech_lengths, key, tokenizer, frontend, cache,
+ **{**kwargs, "is_final": True},
)
- # results_chunk must be None when is_final=False
- assert results_chunk is None
-
- # push tail chunk
- kwargs["is_final"] = _is_final
- cache["encoder"]["tail_chunk"] = True
- speech = cache["encoder"]["feats"]
- speech_lengths = torch.tensor([speech.shape[1]], dtype=torch.int64).to(
- speech.device
- )
- results_chunk_tail = self.generate_chunk(
- speech,
- speech_lengths,
- key=key,
- tokenizer=tokenizer,
- cache=cache,
- frontend=frontend,
- **kwargs,
- )
-
- result = results_chunk_tail
-
- if _is_final:
self.init_cache(cache, **kwargs)
- if kwargs.get("output_dir"):
- if not hasattr(self, "writer"):
- self.writer = DatadirWriter(kwargs.get("output_dir"))
-
- return result, meta_data
+ meta_data["extract_feat"] = f"{time.perf_counter() - time2:0.3f}"
+ return results, meta_data
def export(self, **kwargs):
"""Export.
diff --git a/tests/test_kws_streaming_continuity.py b/tests/test_kws_streaming_continuity.py
new file mode 100644
index 000000000..d019f99dc
--- /dev/null
+++ b/tests/test_kws_streaming_continuity.py
@@ -0,0 +1,454 @@
+"""KWS public streaming contracts with production overlap and no model weights."""
+
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import soundfile as sf
+import torch
+
+from funasr.auto.auto_model import AutoModel
+from funasr.frontends.wav_frontend import WavFrontendOnline
+from funasr.models.sanm_kws_streaming.model import SanmKWSStreaming
+from funasr.models.scama.encoder import SANMEncoderChunkOpt
+from funasr.models.transformer.embedding import StreamSinusoidalPositionEncoder
+from funasr.utils.load_utils import extract_fbank
+
+
+class MarkerFrontend:
+ """One feature per complete 960-sample block; never invent short-input frames."""
+
+ fs, frame_shift, lfr_n, n_mels, lfr_m = 16000, 10, 6, 1, 1
+
+ def __call__(self, audio, lengths, cache, is_final=False, **kwargs):
+ samples = torch.cat((cache.get("samples", audio[:, :0]), audio), dim=1)
+ count = samples.shape[1] // 960
+ features = samples[:, : count * 960 : 960, None].clone()
+ cache["samples"] = samples[:, count * 960 :].clone()
+ return features, torch.tensor([count])
+
+
+class IdentityChunkEncoder(SANMEncoderChunkOpt):
+ """Keep the real forward_chunk and overlap cache, replacing acoustic layers."""
+
+ def __init__(self):
+ torch.nn.Module.__init__(self)
+ self.embed = None
+ self.encoders0 = torch.nn.ModuleList()
+ self.encoders = torch.nn.ModuleList()
+ self.normalize_before = False
+
+ def output_size(self):
+ return 1
+
+
+class RecordingDecoder:
+ def __init__(self, ctc, **kwargs):
+ self.state = ctc
+
+ def decode(self, features):
+ self.state.frames.append(features.detach().clone())
+ return self.state.detected, "wake", 0.9
+
+
+class FourDimFrontend(MarkerFrontend):
+ n_mels = 4
+
+ def __call__(self, *args, **kwargs):
+ features, lengths = super().__call__(*args, **kwargs)
+ return features.repeat(1, 1, 4), lengths
+
+
+class ContextLayer(torch.nn.Module):
+ def forward_chunk(self, features, state, chunk_size, look_back):
+ return features + 0.01 * features.mean(dim=1, keepdim=True), state
+
+
+class PositionedEncoder(IdentityChunkEncoder):
+ def __init__(self, contextual=False):
+ super().__init__()
+ self.embed = StreamSinusoidalPositionEncoder()
+ if contextual:
+ self.encoders0.append(ContextLayer())
+ self.positions = []
+
+ def output_size(self):
+ return 4
+
+ def forward_chunk(self, features, lengths, cache, **kwargs):
+ before = cache["start_idx"]
+ result = super().forward_chunk(features, lengths, cache, **kwargs)
+ self.positions.append((before, cache["start_idx"]))
+ return result
+
+
+class LightweightKws(SanmKWSStreaming):
+ def __init__(self, detected=True):
+ torch.nn.Module.__init__(self)
+ self.anchor = torch.nn.Parameter(torch.zeros(1))
+ self.encoder = IdentityChunkEncoder()
+ self.specaug = self.normalize = None
+ self.ctc = SimpleNamespace(frames=[], detected=detected)
+ self.eval()
+
+
+class KwsStreamingContinuityTest(unittest.TestCase):
+ def setUp(self):
+ p = patch("funasr.utils.kws_utils.KwsCtcPrefixDecoder", RecordingDecoder)
+ p.start()
+ self.addCleanup(p.stop)
+
+ def setup_model(self, frontend=None, detected=True, chunk_size=None):
+ model = LightweightKws(detected)
+ frontend = frontend or MarkerFrontend()
+ config = {
+ "device": "cpu",
+ "batch_size": 1,
+ "disable_pbar": True,
+ "keywords": "wake",
+ "chunk_size": chunk_size or [4, 8, 4],
+ "encoder_conf": {"output_size": 1},
+ "frontend_conf": {"n_mels": frontend.n_mels, "lfr_m": frontend.lfr_m},
+ "frontend": frontend,
+ "tokenizer": SimpleNamespace(token_list=["wake"], seg_dict={}),
+ }
+ wrapper = AutoModel.__new__(AutoModel)
+ wrapper.model = model
+ wrapper.kwargs = config
+ wrapper.vad_model = wrapper.punc_model = None
+ wrapper._store_base_configs()
+ self.addCleanup(
+ lambda: model.writer.close() if hasattr(model, "writer") else None
+ )
+ return model, wrapper, config
+
+ @staticmethod
+ def markers(frames):
+ return torch.arange(1, frames + 1, dtype=torch.float32).repeat_interleave(960)
+
+ def stream(self, wrapper, audio, pieces, empty_flush=False, **kwargs):
+ cache = {}
+ offset, index = 0, 0
+ while offset < len(audio):
+ end = min(len(audio), offset + pieces[index % len(pieces)])
+ final = end == len(audio) and not empty_flush
+ result = wrapper.generate(
+ input=audio[offset:end],
+ cache=cache,
+ key="sample",
+ is_final=final,
+ **kwargs
+ )
+ if not final:
+ self.assertEqual(result, [])
+ self.assertFalse(cache["encoder"]["tail_chunk"])
+ offset, index = end, index + 1
+ if empty_flush or len(audio) == 0:
+ result = wrapper.generate(
+ input=audio[:0], cache=cache, key="sample", is_final=True, **kwargs
+ )
+ self.assertEqual(cache["prev_samples"].numel(), 0)
+ self.assertIsNone(cache["encoder"]["encoder_out"])
+ return result
+
+ def test_public_nonfinal_contract_and_unflushed_cache(self):
+ model, wrapper, config = self.setup_model()
+ cache = {}
+ results, metadata = model.inference(
+ [self.markers(12)], key=["sample"], cache=cache, is_final=False, **config
+ )
+ self.assertEqual(results, [])
+ self.assertIsInstance(metadata, dict)
+ self.assertFalse(cache["encoder"]["tail_chunk"])
+ self.assertEqual(model.ctc.frames, [])
+ self.assertEqual(
+ wrapper.generate(
+ input=self.markers(12), cache={}, key="sample", is_final=False
+ ),
+ [],
+ )
+
+ def test_complete_inputs_keep_each_valid_frame_once(self):
+ for frames in (1, 3, 4, 5, 7, 8, 9, 12, 13, 16, 20, 21, 32, 37):
+ with self.subTest(frames=frames):
+ model, wrapper, _ = self.setup_model()
+ self.assertEqual(
+ self.stream(wrapper, self.markers(frames), [100000]),
+ [{"key": "sample", "text": "detected wake 0.9"}],
+ )
+ self.assertEqual(len(model.ctc.frames), 1)
+ torch.testing.assert_close(
+ model.ctc.frames[0][:, 0],
+ torch.arange(1, frames + 1, dtype=torch.float32),
+ )
+
+ def test_partitions_and_empty_flush_preserve_marker_frames(self):
+ for pieces in ([959], [960], [3840], [7680], [1, 959, 3001, 7000]):
+ for empty_flush in (False, True):
+ with self.subTest(pieces=pieces, empty_flush=empty_flush):
+ model, wrapper, _ = self.setup_model()
+ self.stream(wrapper, self.markers(37), pieces, empty_flush)
+ self.assertEqual(len(model.ctc.frames), 1)
+ torch.testing.assert_close(
+ model.ctc.frames[0][:, 0],
+ torch.arange(1, 38, dtype=torch.float32),
+ )
+
+ def test_empty_and_insufficient_audio_do_not_fabricate_detection(self):
+ for length in (0, 1, 399, 959):
+ with self.subTest(length=length):
+ model, wrapper, _ = self.setup_model()
+ self.assertEqual(self.stream(wrapper, torch.ones(length), [100]), [])
+ self.assertEqual(model.ctc.frames, [])
+
+ def test_final_output_once_and_second_utterance_starts_clean(self):
+ with tempfile.TemporaryDirectory() as directory:
+ model, wrapper, _ = self.setup_model(detected=False)
+ cache = {}
+ for key in ("first", "second"):
+ self.assertEqual(
+ wrapper.generate(
+ input=self.markers(12),
+ cache=cache,
+ key=key,
+ is_final=False,
+ output_dir=directory,
+ ),
+ [],
+ )
+ if key == "first":
+ self.assertFalse((Path(directory) / "detect").exists())
+ result = wrapper.generate(
+ input=self.markers(5),
+ cache=cache,
+ key=key,
+ is_final=True,
+ output_dir=directory,
+ )
+ self.assertEqual(result, [{"key": key, "text": "rejected"}])
+ self.assertIsNone(cache["encoder"]["encoder_out"])
+ self.assertEqual(
+ (Path(directory) / "detect").read_text(),
+ "first rejected\nsecond rejected\n",
+ )
+ self.assertEqual(len(model.ctc.frames), 2)
+ torch.testing.assert_close(model.ctc.frames[0], model.ctc.frames[1])
+
+ def test_file_input_finalizes_even_when_flag_is_false(self):
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "markers.wav"
+ sf.write(path, self.markers(21).numpy(), 16000, subtype="FLOAT")
+ model, wrapper, _ = self.setup_model()
+ self.assertEqual(
+ wrapper.generate(
+ input=str(path), cache={}, key="sample", is_final=False
+ ),
+ [{"key": "sample", "text": "detected wake 0.9"}],
+ )
+ torch.testing.assert_close(
+ model.ctc.frames[0][:, 0], torch.arange(1, 22, dtype=torch.float32)
+ )
+
+ def test_real_frontend_frames_match_complete_audio_and_partitions(self):
+ def frontend():
+ return WavFrontendOnline(n_mels=8, lfr_m=7, lfr_n=6, dither=0.0)
+
+ for length in (
+ 0,
+ 399,
+ 400,
+ 559,
+ 560,
+ 879,
+ 960,
+ 3839,
+ 3840,
+ 3841,
+ 11520,
+ 12479,
+ 12480,
+ 15360,
+ 15361,
+ 19199,
+ 40000,
+ ):
+ with self.subTest(length=length):
+ time = torch.arange(length, dtype=torch.float32) / 16000
+ audio = 0.1 * torch.sin(2 * torch.pi * (300 * time + 100 * time**2))
+ expected, lengths = extract_fbank(
+ [audio], frontend=frontend(), cache={}, is_final=True
+ )
+ count = int(lengths[0]) if lengths.numel() else 0
+ fbank_count = max(0, 1 + (length - 400) // 160)
+ self.assertEqual(count, (fbank_count + 5) // 6)
+ for pieces in ([100000], [1, 959, 3001, 7000]):
+ model, wrapper, _ = self.setup_model(frontend())
+ result = self.stream(wrapper, audio, pieces, empty_flush=True)
+ if not count:
+ self.assertEqual(result, [])
+ self.assertEqual(model.ctc.frames, [])
+ else:
+ self.assertEqual(len(model.ctc.frames), 1)
+ torch.testing.assert_close(
+ model.ctc.frames[0],
+ expected[0, :count],
+ rtol=1e-5,
+ atol=1e-5,
+ )
+
+ def test_real_position_encoding_covers_prefix_and_never_reembeds_overlap(self):
+ for pieces in ([100000], [1, 959, 3001, 7000]):
+ with self.subTest(pieces=pieces):
+ model, wrapper, _ = self.setup_model(FourDimFrontend())
+ model.encoder = PositionedEncoder()
+ self.stream(wrapper, self.markers(37), pieces, empty_flush=True)
+ raw = torch.arange(1, 38, dtype=torch.float32)[None, :, None].repeat(
+ 1, 1, 4
+ )
+ expected = StreamSinusoidalPositionEncoder()(raw * 2, {"start_idx": 0})
+ torch.testing.assert_close(model.ctc.frames[0], expected[0])
+ self.assertEqual(
+ sum(end - start for start, end in model.encoder.positions), 37
+ )
+
+ def test_context_sensitive_windows_are_independent_of_packet_and_final_boundaries(
+ self,
+ ):
+ for frames in (12, 13, 19, 20, 21, 28, 37):
+ reference = None
+ for pieces, empty_flush in (
+ ([100000], False),
+ ([100000], True),
+ ([959, 3841], False),
+ ):
+ with self.subTest(
+ frames=frames, pieces=pieces, empty_flush=empty_flush
+ ):
+ model, wrapper, _ = self.setup_model(FourDimFrontend())
+ model.encoder = PositionedEncoder(contextual=True)
+ self.stream(wrapper, self.markers(frames), pieces, empty_flush)
+ observed = model.ctc.frames[0]
+ if reference is None:
+ reference = observed
+ else:
+ torch.testing.assert_close(observed, reference)
+
+ def test_asymmetric_and_zero_overlap_chunk_shapes(self):
+ for chunk_size in ([0, 8, 4], [2, 8, 4], [4, 8, 0], [0, 8, 0]):
+ with self.subTest(chunk_size=chunk_size):
+ model, wrapper, _ = self.setup_model(chunk_size=chunk_size)
+ self.stream(wrapper, self.markers(24), [959, 3841], empty_flush=True)
+ torch.testing.assert_close(
+ model.ctc.frames[0][:, 0], torch.arange(1, 25, dtype=torch.float32)
+ )
+
+ def test_real_sanm_convolution_handles_exact_stride_endings(self):
+ for chunk_size in ([0, 8, 0], [4, 8, 0], [4, 8, 4]):
+ reference = None
+ for pieces in ([100000], [959, 3841]):
+ with self.subTest(chunk_size=chunk_size, pieces=pieces):
+ model, wrapper, _ = self.setup_model(
+ FourDimFrontend(), chunk_size=chunk_size
+ )
+ torch.manual_seed(23)
+ model.encoder = SANMEncoderChunkOpt(
+ input_size=4,
+ output_size=4,
+ attention_heads=2,
+ linear_units=8,
+ num_blocks=1,
+ kernel_size=3,
+ input_layer="pe_online",
+ dropout_rate=0.0,
+ positional_dropout_rate=0.0,
+ attention_dropout_rate=0.0,
+ ).eval()
+ self.stream(wrapper, self.markers(24), pieces, empty_flush=True)
+ observed = model.ctc.frames[0]
+ self.assertEqual(observed.shape, (24, 4))
+ if reference is None:
+ reference = observed
+ else:
+ torch.testing.assert_close(observed, reference)
+
+ def test_short_final_frontend_waveforms_align_to_real_samples(self):
+ for lfr_m, lfr_n in ((7, 6), (5, 1)):
+ for length in (0, 399, 400, 559, 560, 879, 960, 12479):
+ for pieces in ([100000], [1, 399, 401]):
+ with self.subTest(lfr=(lfr_m, lfr_n), length=length, pieces=pieces):
+ frontend = WavFrontendOnline(
+ n_mels=8, lfr_m=lfr_m, lfr_n=lfr_n, dither=0.0
+ )
+ audio = torch.linspace(-0.1, 0.1, length)
+ cache, offset, emitted, index = {}, 0, 0, 0
+ while offset < length:
+ end = min(length, offset + pieces[index % len(pieces)])
+ features, _ = frontend(
+ audio[None, offset:end],
+ [end - offset],
+ cache=cache,
+ is_final=False,
+ return_waveform=True,
+ )
+ if features.ndim == 3 and features.shape[1]:
+ count = features.shape[1]
+ start = emitted * lfr_n * 160
+ stop = start + (count - 1) * lfr_n * 160 + 400
+ torch.testing.assert_close(
+ cache["aligned_waveforms"], audio[None, start:stop]
+ )
+ emitted += count
+ offset, index = end, index + 1
+ features, _ = frontend(
+ audio[None, :0],
+ [0],
+ cache=cache,
+ is_final=True,
+ return_waveform=True,
+ )
+ if features.ndim == 3 and features.shape[1]:
+ count = features.shape[1]
+ start = emitted * lfr_n * 160
+ stop = start + (count - 1) * lfr_n * 160 + 400
+ self.assertLessEqual(stop, length)
+ torch.testing.assert_close(
+ cache["aligned_waveforms"], audio[None, start:stop]
+ )
+ emitted += count
+ fbank_count = max(0, 1 + (length - 400) // 160)
+ self.assertEqual(emitted, (fbank_count + lfr_n - 1) // lfr_n)
+
+ def test_short_nonempty_eos_aligns_final_waveform(self):
+ for lfr_m, lfr_n in ((7, 6), (5, 1)):
+ for length in (399, 400, 559, 560, 879, 960, 12479):
+ with self.subTest(lfr=(lfr_m, lfr_n), length=length):
+ frontend = WavFrontendOnline(
+ n_mels=8, lfr_m=lfr_m, lfr_n=lfr_n, dither=0.0
+ )
+ audio = torch.linspace(-0.1, 0.1, length)
+ cache = {}
+ features, _ = frontend(
+ audio[None],
+ [length],
+ cache=cache,
+ is_final=True,
+ return_waveform=True,
+ )
+ fbank_count = max(0, 1 + (length - 400) // 160)
+ count = (fbank_count + lfr_n - 1) // lfr_n
+ if not count:
+ self.assertEqual(features.numel(), 0)
+ self.assertEqual(cache["aligned_waveforms"].numel(), 0)
+ else:
+ self.assertEqual(features.shape[1], count)
+ stop = (count - 1) * lfr_n * 160 + 400
+ self.assertLessEqual(stop, length)
+ torch.testing.assert_close(
+ cache["aligned_waveforms"], audio[None, :stop]
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()