Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(metrics): size the duplicated WER cost matrices for long inputs (#3785)

baolong lin24217c8ff0b8
diff --git a/examples/aishell/e_paraformer/utils/compute_wer.py b/examples/aishell/e_paraformer/utils/compute_wer.py
index 3d00010ca..b117f3216 100755
--- a/examples/aishell/e_paraformer/utils/compute_wer.py
+++ b/examples/aishell/e_paraformer/utils/compute_wer.py
@@ -100,7 +100,11 @@ def compute_wer_by_line(hyp, ref):
len_hyp = len(hyp)
len_ref = len(ref)
- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)
+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit
+ # accumulator overflows as soon as either input reaches 32768 tokens and
+ # the first row/column initialisation tries to store that index. int32
+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.
+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)
ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)
diff --git a/examples/aishell/paraformer/utils/compute_wer.py b/examples/aishell/paraformer/utils/compute_wer.py
index 3d00010ca..b117f3216 100755
--- a/examples/aishell/paraformer/utils/compute_wer.py
+++ b/examples/aishell/paraformer/utils/compute_wer.py
@@ -100,7 +100,11 @@ def compute_wer_by_line(hyp, ref):
len_hyp = len(hyp)
len_ref = len(ref)
- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)
+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit
+ # accumulator overflows as soon as either input reaches 32768 tokens and
+ # the first row/column initialisation tries to store that index. int32
+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.
+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)
ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)
diff --git a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
index 46ec08e5c..e86399526 100755
--- a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
+++ b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
@@ -96,7 +96,11 @@ def compute_wer_by_line(hyp, ref):
len_hyp = len(hyp)
len_ref = len(ref)
- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)
+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit
+ # accumulator overflows as soon as either input reaches 32768 tokens and
+ # the first row/column initialisation tries to store that index. int32
+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.
+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)
ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)
diff --git a/runtime/python/utils/compute_wer.py b/runtime/python/utils/compute_wer.py
index 46ec08e5c..e86399526 100755
--- a/runtime/python/utils/compute_wer.py
+++ b/runtime/python/utils/compute_wer.py
@@ -96,7 +96,11 @@ def compute_wer_by_line(hyp, ref):
len_hyp = len(hyp)
len_ref = len(ref)
- cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16)
+ # An edit distance can be as large as max(len_hyp, len_ref), so a 16-bit
+ # accumulator overflows as soon as either input reaches 32768 tokens and
+ # the first row/column initialisation tries to store that index. int32
+ # covers any input this O(len_hyp * len_ref) loop can realistically reach.
+ cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int32)
ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8)
diff --git a/tests/test_metrics_wer.py b/tests/test_metrics_wer.py
index 7ac98a62f..9147f2fa3 100644
--- a/tests/test_metrics_wer.py
+++ b/tests/test_metrics_wer.py
@@ -1,5 +1,56 @@
+import importlib.util
+from pathlib import Path
+
+import pytest
+
from funasr.metrics.wer import compute_wer, compute_wer_by_line, print_cer_detail
+# Standalone copies of compute_wer_by_line kept for the runtime bundles and the
+# Aishell examples. They duplicate the helper instead of importing it, so they
+# need the same boundary coverage as the library version.
+DUPLICATE_WER_HELPERS = [
+ "runtime/python/utils/compute_wer.py",
+ "runtime/python/libtorch/funasr_torch/utils/compute_wer.py",
+ "examples/aishell/paraformer/utils/compute_wer.py",
+ "examples/aishell/e_paraformer/utils/compute_wer.py",
+]
+
+
+def _load_helper(relative_path):
+ path = Path(__file__).resolve().parent.parent / relative_path
+ spec = importlib.util.spec_from_file_location(
+ f"dup_{relative_path.replace('/', '_').removesuffix('.py')}", path
+ )
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected]("relative_path", DUPLICATE_WER_HELPERS)
+def test_duplicate_wer_helpers_handle_inputs_at_the_int16_boundary(relative_path):
+ helper = _load_helper(relative_path)
+
+ out_item = helper.compute_wer_by_line(["a"] * 32768, ["a"])
+
+ assert out_item["nwords"] == 1
+ assert out_item["cor"] == 1
+ assert out_item["wrong"] == 32767
+ assert out_item["ins"] == 32767
+
+
[email protected]("relative_path", DUPLICATE_WER_HELPERS)
+def test_duplicate_wer_helpers_agree_with_the_library(relative_path):
+ helper = _load_helper(relative_path)
+
+ for hyp, ref in [
+ (["a", "b", "c"], ["a", "x", "c"]),
+ (["uh"], []),
+ (["the", "cat", "sat"], ["the", "cat", "sat", "down"]),
+ ]:
+ assert {
+ k: int(v) for k, v in helper.compute_wer_by_line(hyp, ref).items()
+ } == {k: int(v) for k, v in compute_wer_by_line(hyp, ref).items()}
+
def test_cer_detail_for_empty_reference_does_not_divide_by_zero():
out_item = compute_wer_by_line(["uh"], [])