Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(metrics): size the duplicated WER cost matrices for long inputs (#3785)
baolong lin24217c8ff0b8
diff --git a/examples/aishell/e_paraformer/utils/compute_wer.py b/examples/aishell/e_paraformer/utils/compute_wer.pyindex 3d00010ca..b117f3216 100755--- a/examples/aishell/e_paraformer/utils/compute_wer.py+++ b/examples/aishell/e_paraformer/utils/compute_wer.py@@ -100,7 +100,11 @@ def compute_wer_by_line(hyp, ref):len_hyp = len(hyp)len_ref = len(ref)- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit+ # accumulator overflows as soon as either input reaches 32768 tokens and+ # the first row/column initialisation tries to store that index. int32+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)diff --git a/examples/aishell/paraformer/utils/compute_wer.py b/examples/aishell/paraformer/utils/compute_wer.pyindex 3d00010ca..b117f3216 100755--- a/examples/aishell/paraformer/utils/compute_wer.py+++ b/examples/aishell/paraformer/utils/compute_wer.py@@ -100,7 +100,11 @@ def compute_wer_by_line(hyp, ref):len_hyp = len(hyp)len_ref = len(ref)- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit+ # accumulator overflows as soon as either input reaches 32768 tokens and+ # the first row/column initialisation tries to store that index. int32+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)diff --git a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py b/runtime/python/libtorch/funasr_torch/utils/compute_wer.pyindex 46ec08e5c..e86399526 100755--- a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py+++ b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py@@ -96,7 +96,11 @@ def compute_wer_by_line(hyp, ref):len_hyp = len(hyp)len_ref = len(ref)- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit+ # accumulator overflows as soon as either input reaches 32768 tokens and+ # the first row/column initialisation tries to store that index. int32+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)diff --git a/runtime/python/utils/compute_wer.py b/runtime/python/utils/compute_wer.pyindex 46ec08e5c..e86399526 100755--- a/runtime/python/utils/compute_wer.py+++ b/runtime/python/utils/compute_wer.py@@ -96,7 +96,11 @@ def compute_wer_by_line(hyp, ref):len_hyp = len(hyp)len_ref = len(ref)- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit+ # accumulator overflows as soon as either input reaches 32768 tokens and+ # the first row/column initialisation tries to store that index. int32+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)diff --git a/tests/test_metrics_wer.py b/tests/test_metrics_wer.pyindex 7ac98a62f..9147f2fa3 100644--- a/tests/test_metrics_wer.py+++ b/tests/test_metrics_wer.py@@ -1,5 +1,56 @@+import importlib.util+from pathlib import Path++import pytest+from funasr.metrics.wer import compute_wer, compute_wer_by_line, print_cer_detail+# Standalone copies of compute_wer_by_line kept for the runtime bundles and the+# Aishell examples. They duplicate the helper instead of importing it, so they+# need the same boundary coverage as the library version.+DUPLICATE_WER_HELPERS = [+ "runtime/python/utils/compute_wer.py",+ "runtime/python/libtorch/funasr_torch/utils/compute_wer.py",+ "examples/aishell/paraformer/utils/compute_wer.py",+ "examples/aishell/e_paraformer/utils/compute_wer.py",+]+++def _load_helper(relative_path):+ path = Path(__file__).resolve().parent.parent / relative_path+ spec = importlib.util.spec_from_file_location(+ f"dup_{relative_path.replace('/', '_').removesuffix('.py')}", path+ )+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++[email protected]("relative_path", DUPLICATE_WER_HELPERS)+def test_duplicate_wer_helpers_handle_inputs_at_the_int16_boundary(relative_path):+ helper = _load_helper(relative_path)++ out_item = helper.compute_wer_by_line(["a"] * 32768, ["a"])++ assert out_item["nwords"] == 1+ assert out_item["cor"] == 1+ assert out_item["wrong"] == 32767+ assert out_item["ins"] == 32767++[email protected]("relative_path", DUPLICATE_WER_HELPERS)+def test_duplicate_wer_helpers_agree_with_the_library(relative_path):+ helper = _load_helper(relative_path)++ for hyp, ref in [+ (["a", "b", "c"], ["a", "x", "c"]),+ (["uh"], []),+ (["the", "cat", "sat"], ["the", "cat", "sat", "down"]),+ ]:+ assert {+ k: int(v) for k, v in helper.compute_wer_by_line(hyp, ref).items()+ } == {k: int(v) for k, v in compute_wer_by_line(hyp, ref).items()}+def test_cer_detail_for_empty_reference_does_not_divide_by_zero():out_item = compute_wer_by_line(["uh"], [])