Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(metrics): report unscored references in the WER summary line (#3787)

Mohammad Hijjawifbbfe01800a8
diff --git a/examples/aishell/e_paraformer/utils/compute_wer.py b/examples/aishell/e_paraformer/utils/compute_wer.py
index b117f3216..fe728c993 100755
--- a/examples/aishell/e_paraformer/utils/compute_wer.py
+++ b/examples/aishell/e_paraformer/utils/compute_wer.py
@@ -85,9 +85,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):
)
cer_detail_writer.write(
"Scored "
- + str(len(hyp_dict))
+ + str(rst["Snt"])
+ " sentences, "
- + str(len(hyp_dict) - rst["Snt"])
+ + str(len(ref_dict) - rst["Snt"])
+ " not present in hyp."
+ "\n"
)
diff --git a/examples/aishell/paraformer/utils/compute_wer.py b/examples/aishell/paraformer/utils/compute_wer.py
index b117f3216..fe728c993 100755
--- a/examples/aishell/paraformer/utils/compute_wer.py
+++ b/examples/aishell/paraformer/utils/compute_wer.py
@@ -85,9 +85,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):
)
cer_detail_writer.write(
"Scored "
- + str(len(hyp_dict))
+ + str(rst["Snt"])
+ " sentences, "
- + str(len(hyp_dict) - rst["Snt"])
+ + str(len(ref_dict) - rst["Snt"])
+ " not present in hyp."
+ "\n"
)
diff --git a/funasr/metrics/wer.py b/funasr/metrics/wer.py
index f32cf9f03..918126be4 100755
--- a/funasr/metrics/wer.py
+++ b/funasr/metrics/wer.py
@@ -111,9 +111,9 @@ def compute_wer(
)
cer_detail_writer.write(
"Scored "
- + str(len(hyp_dict))
+ + str(rst["Snt"])
+ " sentences, "
- + str(len(hyp_dict) - rst["Snt"])
+ + str(len(ref_dict) - rst["Snt"])
+ " not present in hyp."
+ "\n"
)
diff --git a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
index e86399526..dce87a5e3 100755
--- a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
+++ b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py
@@ -81,9 +81,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):
)
cer_detail_writer.write(
"Scored "
- + str(len(hyp_dict))
+ + str(rst["Snt"])
+ " sentences, "
- + str(len(hyp_dict) - rst["Snt"])
+ + str(len(ref_dict) - rst["Snt"])
+ " not present in hyp."
+ "\n"
)
diff --git a/runtime/python/utils/compute_wer.py b/runtime/python/utils/compute_wer.py
index e86399526..dce87a5e3 100755
--- a/runtime/python/utils/compute_wer.py
+++ b/runtime/python/utils/compute_wer.py
@@ -81,9 +81,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):
)
cer_detail_writer.write(
"Scored "
- + str(len(hyp_dict))
+ + str(rst["Snt"])
+ " sentences, "
- + str(len(hyp_dict) - rst["Snt"])
+ + str(len(ref_dict) - rst["Snt"])
+ " not present in hyp."
+ "\n"
)
diff --git a/tests/test_metrics_wer.py b/tests/test_metrics_wer.py
index 9147f2fa3..c706a4177 100644
--- a/tests/test_metrics_wer.py
+++ b/tests/test_metrics_wer.py
@@ -94,3 +94,36 @@ def test_compute_wer_scores_all_utterances_when_one_reference_is_empty(tmp_path)
assert "utt3(nwords=2,cor=2,ins=0,del=0,sub=0) corr:100.00%,cer:0.00%" in report
assert "%WER 20.0 [ 1 / 5, 1 ins, 0 del, 0 sub ]" in report
assert "%SER 33.33 [ 1 / 3 ]" in report
+
+
+def _write_summary_fixture(tmp_path):
+ # utt2 has a reference but no hypothesis; utt9 has a hypothesis but no
+ # reference. Neither can be scored.
+ ref_file = tmp_path / "ref.txt"
+ hyp_file = tmp_path / "hyp.txt"
+ ref_file.write_text("utt1 the cat sat\nutt2 hello world\nutt3 good morning\n", encoding="utf-8")
+ hyp_file.write_text(
+ "utt1 the cat sat\nutt3 good morning\nutt9 stray output\n", encoding="utf-8"
+ )
+ return ref_file, hyp_file, tmp_path / "out.wer"
+
+
+def test_compute_wer_summary_counts_references_missing_from_hyp(tmp_path):
+ ref_file, hyp_file, cer_file = _write_summary_fixture(tmp_path)
+
+ compute_wer(str(ref_file), str(hyp_file), str(cer_file))
+
+ report = cer_file.read_text(encoding="utf-8")
+ assert "%SER 0.0 [ 0 / 2 ]" in report
+ assert "Scored 2 sentences, 1 not present in hyp." in report
+
+
[email protected]("relative_path", DUPLICATE_WER_HELPERS)
+def test_duplicate_wer_helpers_count_references_missing_from_hyp(relative_path, tmp_path):
+ helper = _load_helper(relative_path)
+ ref_file, hyp_file, cer_file = _write_summary_fixture(tmp_path)
+
+ helper.compute_wer(str(ref_file), str(hyp_file), str(cer_file))
+
+ report = cer_file.read_text(encoding="utf-8")
+ assert "Scored 2 sentences, 1 not present in hyp." in report