Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(paraformer): keep one keyed result per utterance when no tokens are predicted
Mohammad Hijjawi16c3cc2741af
diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.pyindex a8f23e926..9531afcd3 100644--- a/funasr/models/contextual_paraformer/model.py+++ b/funasr/models/contextual_paraformer/model.py@@ -457,7 +457,11 @@ class ContextualParaformer(Paraformer):)pre_token_length = pre_token_length.round().long()if torch.max(pre_token_length) < 1:- return []+ # no tokens for any utterance: keep one empty result per key+ b = encoder_out.size(0)+ if tokenizer is None:+ return [{"key": key[i], "token_int": []} for i in range(b)], meta_data+ return [{"key": key[i], "text": ""} for i in range(b)], meta_datadecoder_outs = self.cal_decoder_with_predictor(encoder_out,diff --git a/funasr/models/paraformer/model.py b/funasr/models/paraformer/model.pyindex 86808be0d..c20343aea 100644--- a/funasr/models/paraformer/model.py+++ b/funasr/models/paraformer/model.py@@ -616,19 +616,24 @@ class Paraformer(torch.nn.Module):)pre_token_length = pre_token_length.round().long()+ b = encoder_out.size(0)+ if isinstance(key[0], (list, tuple)):+ key = key[0]+ if len(key) < b:+ key = key * bif torch.max(pre_token_length) < 1:- return []+ # no tokens for any utterance: keep one empty result per key+ if tokenizer is None:+ return [{"key": key[i], "token_int": []} for i in range(b)], meta_data+ if pred_timestamp:+ return [{"key": key[i], "text": "", "timestamp": []} for i in range(b)], meta_data+ return [{"key": key[i], "text": ""} for i in range(b)], meta_datadecoder_outs = self.cal_decoder_with_predictor(encoder_out, encoder_out_lens, pre_acoustic_embeds, pre_token_length)decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1]results = []- b, n, d = decoder_out.size()- if isinstance(key[0], (list, tuple)):- key = key[0]- if len(key) < b:- key = key * bfor i in range(b):x = encoder_out[i, : encoder_out_lens[i], :]am_scores = decoder_out[i, : pre_token_length[i], :]diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.pyindex 6a5787530..c1c3ba55a 100644--- a/funasr/models/seaco_paraformer/model.py+++ b/funasr/models/seaco_paraformer/model.py@@ -483,7 +483,13 @@ class SeacoParaformer(BiCifParaformer, Paraformer):pre_acoustic_embeds, pre_token_length = predictor_outs[0], predictor_outs[1]pre_token_length = pre_token_length.round().long()if torch.max(pre_token_length) < 1:- return ([],)+ # no tokens for any utterance: keep one empty result per key+ b = encoder_out.size(0)+ if tokenizer is None:+ return [{"key": key[i], "token_int": []} for i in range(b)], meta_data+ if self.predictor_name == "CifPredictorV3":+ return [{"key": key[i], "text": "", "timestamp": []} for i in range(b)], meta_data+ return [{"key": key[i], "text": ""} for i in range(b)], meta_datadecoder_out = self._seaco_decode_with_ASF(encoder_out,diff --git a/tests/test_paraformer_empty_token_batch.py b/tests/test_paraformer_empty_token_batch.pynew file mode 100644index 000000000..7771224e3--- /dev/null+++ b/tests/test_paraformer_empty_token_batch.py@@ -0,0 +1,146 @@+import unittest++import numpy as np+import torch++from funasr.auto.auto_model import AutoModel+from funasr.frontends.wav_frontend import WavFrontend+from funasr.models.contextual_paraformer.model import ContextualParaformer+from funasr.models.paraformer.model import Paraformer+from funasr.models.seaco_paraformer.model import SeacoParaformer+++class _DummyTokenizer:+ def ids2tokens(self, ids):+ return ["x" for _ in ids]++ def tokens2text(self, tokens):+ return "".join(tokens)+++class _ZeroTokenMixin:+ """Encoder/predictor stand-ins where every utterance predicts zero tokens."""++ def _setup(self):+ torch.nn.Module.__init__(self)+ self.dummy = torch.nn.Parameter(torch.zeros(1))+ self.beam_search = None+ self.ctc = None+ self.sos = 1+ self.eos = 2+ self.blank_id = 0++ def encode(self, speech, speech_lengths):+ return torch.zeros(speech.size(0), 4, 2), torch.full((speech.size(0),), 4)++ def calc_predictor(self, encoder_out, encoder_out_lens):+ batch = encoder_out.size(0)+ return (+ torch.zeros(batch, 1, 2),+ torch.full((batch,), 0.4),+ torch.zeros(batch, 4),+ torch.zeros(batch, 4),+ )++ def generate_hotwords_list(self, *args, **kwargs):+ return None++ def cal_decoder_with_predictor(self, *args, **kwargs):+ raise AssertionError("decoder must not run when no token is predicted")++ _seaco_decode_with_ASF = cal_decoder_with_predictor+++class _ZeroTokenParaformer(_ZeroTokenMixin, Paraformer):+ def __init__(self):+ self._setup()+++class _ZeroTokenContextualParaformer(_ZeroTokenMixin, ContextualParaformer):+ def __init__(self):+ self._setup()+++class _ZeroTokenSeacoParaformer(_ZeroTokenMixin, SeacoParaformer):+ def __init__(self, predictor_name="CifPredictorV3"):+ self._setup()+ self.predictor_name = predictor_name+++def _frontend():+ return WavFrontend(cmvn_file=None, dither=0.0)+++def _inference(model, keys, **kwargs):+ return model.inference(+ [np.zeros(1600, dtype=np.float32) for _ in keys],+ key=list(keys),+ frontend=_frontend(),+ device="cpu",+ **kwargs,+ )+++class TestParaformerEmptyTokenBatch(unittest.TestCase):+ def test_paraformer_returns_one_keyed_result_per_utterance(self):+ results, meta_data = _inference(+ _ZeroTokenParaformer(), ["a", "b"], tokenizer=_DummyTokenizer()+ )+ self.assertEqual(results, [{"key": "a", "text": ""}, {"key": "b", "text": ""}])+ self.assertIn("batch_data_time", meta_data)++ def test_paraformer_keeps_timestamp_field(self):+ results, _ = _inference(+ _ZeroTokenParaformer(), ["a", "b"], tokenizer=_DummyTokenizer(), output_timestamp=True+ )+ self.assertEqual(+ results,+ [{"key": "a", "text": "", "timestamp": []}, {"key": "b", "text": "", "timestamp": []}],+ )++ def test_paraformer_without_tokenizer_returns_empty_token_ids(self):+ results, _ = _inference(_ZeroTokenParaformer(), ["a", "b"])+ self.assertEqual(results, [{"key": "a", "token_int": []}, {"key": "b", "token_int": []}])++ def test_contextual_paraformer_returns_one_keyed_result_per_utterance(self):+ results, _ = _inference(+ _ZeroTokenContextualParaformer(), ["a", "b"], tokenizer=_DummyTokenizer()+ )+ self.assertEqual(results, [{"key": "a", "text": ""}, {"key": "b", "text": ""}])++ def test_seaco_paraformer_returns_one_keyed_result_per_utterance(self):+ results, _ = _inference(+ _ZeroTokenSeacoParaformer(), ["a", "b"], tokenizer=_DummyTokenizer()+ )+ self.assertEqual(+ results,+ [{"key": "a", "text": "", "timestamp": []}, {"key": "b", "text": "", "timestamp": []}],+ )++ results, _ = _inference(+ _ZeroTokenSeacoParaformer("CifPredictorV2"), ["a"], tokenizer=_DummyTokenizer()+ )+ self.assertEqual(results, [{"key": "a", "text": ""}])++ def test_auto_model_keeps_every_utterance_of_the_batch(self):+ am = AutoModel.__new__(AutoModel)+ am.model = _ZeroTokenParaformer()+ am.kwargs = {+ "batch_size": 2,+ "disable_pbar": True,+ "device": "cpu",+ "frontend": _frontend(),+ "tokenizer": _DummyTokenizer(),+ }++ res = AutoModel.inference(+ am,+ [np.zeros(1600, dtype=np.float32), np.zeros(1600, dtype=np.float32)],+ key=["a", "b"],+ )++ self.assertEqual(res, [{"key": "a", "text": ""}, {"key": "b", "text": ""}])+++if __name__ == "__main__":+ unittest.main()