Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: handle Unicode punctuation post-processing (#3238)
zhifu gao1d8080af99fa
diff --git a/fun_text_processing/text_normalization/data_loader_utils.py b/fun_text_processing/text_normalization/data_loader_utils.pyindex 9b145405f..b228a65ad 100644--- a/fun_text_processing/text_normalization/data_loader_utils.py+++ b/fun_text_processing/text_normalization/data_loader_utils.py@@ -284,7 +284,7 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: boolArgs:input: input text (original input to the NN, before normalization or tokenization)normalized_text: output text (output of the TN NN model)- add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time)+ add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation"""# in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly)# this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement@@ -296,12 +296,11 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: boolpunct_marks = [x for x in string.punctuation if x in input]if add_unicode_punct:- punct_unicode = [- chr(i)- for i in range(sys.maxunicode)- if category(chr(i)).startswith("P") and chr(i) not in punct_default and chr(i) in input- ]- punct_marks = punct_marks.extend(punct_unicode)+ punct_marks.extend(+ char+ for char in dict.fromkeys(input)+ if char not in string.punctuation and category(char).startswith("P")+ )for punct in punct_marks:try:diff --git a/tests/test_text_normalization_punctuation.py b/tests/test_text_normalization_punctuation.pynew file mode 100644index 000000000..436bd2eef--- /dev/null+++ b/tests/test_text_normalization_punctuation.py@@ -0,0 +1,16 @@+from fun_text_processing.text_normalization.data_loader_utils import post_process_punct+++def test_post_process_punct_preserves_ascii_punctuation_spacing():+ assert post_process_punct("test' example", "test 'example") == "test' example"+++def test_post_process_punct_handles_unicode_punctuation():+ assert (+ post_process_punct(+ "\u4f60\u597d\uff0c\u4e16\u754c\uff01",+ "\u4f60\u597d \uff0c \u4e16\u754c \uff01",+ add_unicode_punct=True,+ )+ == "\u4f60\u597d\uff0c\u4e16\u754c\uff01"+ )