Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(paraformer): handle trailing zero-fire samples in mixed CIF batches

BillVan194539e7511aaf4c
diff --git a/funasr/models/paraformer/cif_predictor.py b/funasr/models/paraformer/cif_predictor.py
index 927ae2195..70e6e2c8d 100644
--- a/funasr/models/paraformer/cif_predictor.py
+++ b/funasr/models/paraformer/cif_predictor.py
@@ -655,6 +655,8 @@ def cif_v1_export(hidden, alphas, threshold: float):
batch_idxs = torch.cumsum(batch_len, dim=0)
shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
shift_batch_idxs[0] = 0
+ # Empty samples have no frame to reset and may point past the last frame.
+ shift_batch_idxs = shift_batch_idxs[batch_len > 0]
shift_frames[shift_batch_idxs] = 0
remains = fires - torch.floor(fires)
@@ -884,6 +886,8 @@ def cif_v1(hidden, alphas, threshold):
batch_idxs = torch.cumsum(batch_len, dim=0)
shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
shift_batch_idxs[0] = 0
+ # Empty samples have no frame to reset and may point past the last frame.
+ shift_batch_idxs = shift_batch_idxs[batch_len > 0]
shift_frames[shift_batch_idxs] = 0
remains = fires - torch.floor(fires)
diff --git a/tests/test_cif_mixed_zero_fires.py b/tests/test_cif_mixed_zero_fires.py
new file mode 100644
index 000000000..6d012d505
--- /dev/null
+++ b/tests/test_cif_mixed_zero_fires.py
@@ -0,0 +1,204 @@
+"""CPU CIF regressions using synthetic tensors; no model weights are needed."""
+
+import importlib.util
+import itertools
+from pathlib import Path
+import tempfile
+import unittest
+
+import numpy as np
+import torch
+
+from funasr.models.paraformer.cif_predictor import (
+ CifPredictorV2,
+ CifPredictorV2Export,
+ cif_v1,
+ cif_v1_export,
+)
+
+FUNCTIONS = (cif_v1, cif_v1_export)
+# All nonzero totals stay away from integer and rounding boundaries.
+# -1 supplies exact zeros; 0 supplies positive mass below the firing threshold.
+ALPHAS = {
+ -1: [0.0] * 5,
+ 0: [0.11, 0.13, 0.09, 0.08, 0.12],
+ 1: [0.23, 0.34, 0.12, 0.33, 0.18],
+ 2: [0.62, 0.52, 0.43, 0.38, 0.35],
+ 3: [0.71, 0.67, 0.69, 0.61, 0.52],
+}
+PATTERNS = (
+ (1, 0),
+ (0, 1),
+ (1, 0, 1),
+ (1, 0, 0),
+ (0, 0, 1),
+ (0, 1, 0),
+ (0, 0),
+ (-1, -1),
+ (1, 1),
+ (2, 0, 3, 0),
+ (0, 2, 0, 0, 3, 0, 0),
+ (1,),
+ (0,),
+ (3, 2, 1),
+)
+
+
+def _inputs(pattern):
+ alphas = torch.tensor([ALPHAS[n] for n in pattern], dtype=torch.float32)
+ hidden = torch.arange(alphas.numel() * 3, dtype=torch.float32)
+ hidden = hidden.reshape(len(pattern), 5, 3) / 17 - 2
+ return hidden, alphas
+
+
+def _sequential_frames(hidden, alphas):
+ """Integrate one sample at a time, splitting each firing frame at mass 1."""
+ samples = []
+ for sample, weights in zip(hidden, alphas):
+ mass = weights.new_zeros(())
+ frame = torch.zeros_like(sample[0])
+ emitted = []
+ for value, weight in zip(sample, weights):
+ if (mass + weight).item() >= 1.0:
+ needed = 1.0 - mass
+ emitted.append(frame + needed * value)
+ mass = weight - needed
+ frame = mass * value
+ else:
+ mass = mass + weight
+ frame = frame + weight * value
+ samples.append(torch.stack(emitted) if emitted else sample[:0])
+ return samples
+
+
+def _expected(hidden, alphas, function):
+ samples = _sequential_frames(hidden, alphas)
+ # Preserve the existing round (eager) versus floor (export) padding contract.
+ lengths = alphas.sum(-1)
+ width = int((lengths.round() if function is cif_v1 else lengths.floor()).max())
+ padded = [torch.nn.functional.pad(sample, (0, 0, 0, width - len(sample))) for sample in samples]
+ return torch.stack(padded), [len(sample) for sample in samples]
+
+
+class _ExportCif(torch.nn.Module):
+ def forward(self, hidden, alphas):
+ return cif_v1_export(hidden, alphas, 1.0)
+
+
+class TestCifMixedZeroFires(unittest.TestCase):
+ def test_minimal_trailing_empty(self):
+ hidden = torch.tensor([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]])
+ alphas = torch.tensor([[0.6, 0.6], [0.1, 0.1]])
+ for function in FUNCTIONS:
+ with self.subTest(function=function.__name__):
+ frames, fires = function(hidden, alphas, 1.0)
+ torch.testing.assert_close(frames, torch.tensor([[[1.8, 2.8]], [[0.0, 0.0]]]))
+ self.assertEqual((fires >= 1).sum(1).tolist(), [1, 0])
+
+ def test_predictor_callers(self):
+ model = CifPredictorV2(idim=2, l_order=0, r_order=0, tail_threshold=0.0).eval()
+ hidden = torch.arange(12, dtype=torch.float32).reshape(3, 2, 2)
+ mask = torch.tensor([[[1.0, 1.0]], [[0.0, 0.0]], [[0.0, 0.0]]])
+ with torch.no_grad():
+ # Real predictor layers produce alpha=0.6 before the padding mask.
+ model.cif_output.weight.zero_()
+ model.cif_output.bias.fill_(torch.logit(torch.tensor(0.6)))
+ for predictor in (model, CifPredictorV2Export(model).eval()):
+ with self.subTest(predictor=type(predictor).__name__):
+ frames, _, _, fires = predictor(hidden, mask=mask)
+ expected = torch.tensor([[[0.8, 1.8]], [[0.0, 0.0]], [[0.0, 0.0]]])
+ torch.testing.assert_close(frames, expected)
+ self.assertEqual((fires >= 1).sum(1).tolist(), [1, 0, 0])
+
+ def test_frames_counts_and_padding(self):
+ for pattern, function in itertools.product(PATTERNS, FUNCTIONS):
+ with self.subTest(pattern=pattern, function=function.__name__):
+ hidden, alphas = _inputs(pattern)
+ expected, counts = _expected(hidden, alphas, function)
+ self.assertEqual(counts, [max(n, 0) for n in pattern])
+ frames, fires = function(hidden, alphas, 1.0)
+ torch.testing.assert_close(frames, expected)
+ self.assertEqual((fires >= 1).sum(1).tolist(), counts)
+ self.assertEqual(frames.dtype, hidden.dtype)
+ self.assertEqual(frames.device, hidden.device)
+
+ def test_batch_permutations(self):
+ hidden, alphas = _inputs((0, 2, 0, 3))
+ for function in FUNCTIONS:
+ expected, _ = _expected(hidden, alphas, function)
+ _, expected_fires = function(hidden, alphas, 1.0)
+ for permutation in itertools.permutations(range(4)):
+ with self.subTest(function=function.__name__, permutation=permutation):
+ order = list(permutation)
+ frames, fires = function(hidden[order], alphas[order], 1.0)
+ torch.testing.assert_close(frames, expected[order])
+ torch.testing.assert_close(fires, expected_fires[order])
+
+ def test_gradients_away_from_boundaries(self):
+ for pattern, function in itertools.product(((2, 0, 3, 0), (3, 2, 1)), FUNCTIONS):
+ with self.subTest(pattern=pattern, function=function.__name__):
+ hidden, alphas = _inputs(pattern)
+ hidden.requires_grad_()
+ alphas.requires_grad_()
+ expected, _ = _expected(hidden, alphas, function)
+ frames, _ = function(hidden, alphas, 1.0)
+ weights = torch.linspace(-0.7, 1.3, frames.numel()).reshape_as(frames)
+ gradients = torch.autograd.grad((frames * weights).sum(), (hidden, alphas))
+ reference = torch.autograd.grad((expected * weights).sum(), (hidden, alphas))
+ for actual, wanted in zip(gradients, reference):
+ self.assertTrue(torch.isfinite(actual).all())
+ torch.testing.assert_close(actual, wanted, atol=2e-5, rtol=2e-5)
+
+ @unittest.skipUnless(
+ importlib.util.find_spec("onnx") and importlib.util.find_spec("onnxruntime"),
+ "ONNX validation requires onnx and onnxruntime",
+ )
+ def test_onnx_dynamic_batches(self):
+ import onnx
+ import onnxruntime as ort
+
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "cif.onnx"
+ # Keep the scripted function's data-dependent empty-batch branch.
+ torch.onnx.export(
+ _ExportCif(),
+ _inputs((2, 1)),
+ str(path),
+ opset_version=14,
+ dynamo=False,
+ input_names=["hidden", "alphas"],
+ output_names=["frames", "fires"],
+ dynamic_axes={
+ "hidden": {0: "batch", 1: "time"},
+ "alphas": {0: "batch", 1: "time"},
+ "frames": {0: "batch", 1: "tokens"},
+ "fires": {0: "batch", 1: "time"},
+ },
+ )
+ onnx.checker.check_model(str(path))
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = 1
+ options.inter_op_num_threads = 1
+ session = ort.InferenceSession(
+ str(path), sess_options=options, providers=["CPUExecutionProvider"]
+ )
+ for pattern in PATTERNS:
+ with self.subTest(pattern=pattern):
+ hidden, alphas = _inputs(pattern)
+ # Also exercise a time dimension different from the export example.
+ hidden = torch.nn.functional.pad(hidden, (0, 0, 0, 2))
+ alphas = torch.nn.functional.pad(alphas, (0, 2))
+ expected, counts = _expected(hidden, alphas, cif_v1_export)
+ frames, fires = session.run(
+ None, {"hidden": hidden.numpy(), "alphas": alphas.numpy()}
+ )
+ self.assertEqual(frames.shape, tuple(expected.shape))
+ self.assertEqual(fires.shape, tuple(alphas.shape))
+ np.testing.assert_allclose(frames, expected.numpy(), atol=2e-5, rtol=2e-5)
+ self.assertEqual((fires >= 1).sum(1).tolist(), counts)
+ _, torch_fires = cif_v1_export(hidden, alphas, 1.0)
+ np.testing.assert_allclose(fires, torch_fires.numpy(), atol=2e-6, rtol=2e-6)
+
+
+if __name__ == "__main__":
+ unittest.main()