Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(metrics): report unscored references in the WER summary line (#3787)
Mohammad Hijjawifbbfe01800a8
diff --git a/examples/aishell/e_paraformer/utils/compute_wer.py b/examples/aishell/e_paraformer/utils/compute_wer.pyindex b117f3216..fe728c993 100755--- a/examples/aishell/e_paraformer/utils/compute_wer.py+++ b/examples/aishell/e_paraformer/utils/compute_wer.py@@ -85,9 +85,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):)cer_detail_writer.write("Scored "- + str(len(hyp_dict))+ + str(rst["Snt"])+ " sentences, "- + str(len(hyp_dict) - rst["Snt"])+ + str(len(ref_dict) - rst["Snt"])+ " not present in hyp."+ "\n")diff --git a/examples/aishell/paraformer/utils/compute_wer.py b/examples/aishell/paraformer/utils/compute_wer.pyindex b117f3216..fe728c993 100755--- a/examples/aishell/paraformer/utils/compute_wer.py+++ b/examples/aishell/paraformer/utils/compute_wer.py@@ -85,9 +85,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):)cer_detail_writer.write("Scored "- + str(len(hyp_dict))+ + str(rst["Snt"])+ " sentences, "- + str(len(hyp_dict) - rst["Snt"])+ + str(len(ref_dict) - rst["Snt"])+ " not present in hyp."+ "\n")diff --git a/funasr/metrics/wer.py b/funasr/metrics/wer.pyindex f32cf9f03..918126be4 100755--- a/funasr/metrics/wer.py+++ b/funasr/metrics/wer.py@@ -111,9 +111,9 @@ def compute_wer()cer_detail_writer.write("Scored "- + str(len(hyp_dict))+ + str(rst["Snt"])+ " sentences, "- + str(len(hyp_dict) - rst["Snt"])+ + str(len(ref_dict) - rst["Snt"])+ " not present in hyp."+ "\n")diff --git a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py b/runtime/python/libtorch/funasr_torch/utils/compute_wer.pyindex e86399526..dce87a5e3 100755--- a/runtime/python/libtorch/funasr_torch/utils/compute_wer.py+++ b/runtime/python/libtorch/funasr_torch/utils/compute_wer.py@@ -81,9 +81,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):)cer_detail_writer.write("Scored "- + str(len(hyp_dict))+ + str(rst["Snt"])+ " sentences, "- + str(len(hyp_dict) - rst["Snt"])+ + str(len(ref_dict) - rst["Snt"])+ " not present in hyp."+ "\n")diff --git a/runtime/python/utils/compute_wer.py b/runtime/python/utils/compute_wer.pyindex e86399526..dce87a5e3 100755--- a/runtime/python/utils/compute_wer.py+++ b/runtime/python/utils/compute_wer.py@@ -81,9 +81,9 @@ def compute_wer(ref_file, hyp_file, cer_detail_file):)cer_detail_writer.write("Scored "- + str(len(hyp_dict))+ + str(rst["Snt"])+ " sentences, "- + str(len(hyp_dict) - rst["Snt"])+ + str(len(ref_dict) - rst["Snt"])+ " not present in hyp."+ "\n")diff --git a/tests/test_metrics_wer.py b/tests/test_metrics_wer.pyindex 9147f2fa3..c706a4177 100644--- a/tests/test_metrics_wer.py+++ b/tests/test_metrics_wer.py@@ -94,3 +94,36 @@ def test_compute_wer_scores_all_utterances_when_one_reference_is_empty(tmp_path)assert "utt3(nwords=2,cor=2,ins=0,del=0,sub=0) corr:100.00%,cer:0.00%" in reportassert "%WER 20.0 [ 1 / 5, 1 ins, 0 del, 0 sub ]" in reportassert "%SER 33.33 [ 1 / 3 ]" in report+++def _write_summary_fixture(tmp_path):+ # utt2 has a reference but no hypothesis; utt9 has a hypothesis but no+ # reference. Neither can be scored.+ ref_file = tmp_path / "ref.txt"+ hyp_file = tmp_path / "hyp.txt"+ ref_file.write_text("utt1 the cat sat\nutt2 hello world\nutt3 good morning\n", encoding="utf-8")+ hyp_file.write_text(+ "utt1 the cat sat\nutt3 good morning\nutt9 stray output\n", encoding="utf-8"+ )+ return ref_file, hyp_file, tmp_path / "out.wer"+++def test_compute_wer_summary_counts_references_missing_from_hyp(tmp_path):+ ref_file, hyp_file, cer_file = _write_summary_fixture(tmp_path)++ compute_wer(str(ref_file), str(hyp_file), str(cer_file))++ report = cer_file.read_text(encoding="utf-8")+ assert "%SER 0.0 [ 0 / 2 ]" in report+ assert "Scored 2 sentences, 1 not present in hyp." in report++[email protected]("relative_path", DUPLICATE_WER_HELPERS)+def test_duplicate_wer_helpers_count_references_missing_from_hyp(relative_path, tmp_path):+ helper = _load_helper(relative_path)+ ref_file, hyp_file, cer_file = _write_summary_fixture(tmp_path)++ helper.compute_wer(str(ref_file), str(hyp_file), str(cer_file))++ report = cer_file.read_text(encoding="utf-8")+ assert "Scored 2 sentences, 1 not present in hyp." in report