Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix: handle Unicode punctuation post-processing (#3238)

zhifu gao1d8080af99fa
diff --git a/fun_text_processing/text_normalization/data_loader_utils.py b/fun_text_processing/text_normalization/data_loader_utils.py
index 9b145405f..b228a65ad 100644
--- a/fun_text_processing/text_normalization/data_loader_utils.py
+++ b/fun_text_processing/text_normalization/data_loader_utils.py
@@ -284,7 +284,7 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool
Args:
input: input text (original input to the NN, before normalization or tokenization)
normalized_text: output text (output of the TN NN model)
- add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation (increases post processing time)
+ add_unicode_punct: set to True to handle unicode punctuation marks as well as default string.punctuation
"""
# in the post-processing WFST graph "``" are repalced with '"" quotes (otherwise single quotes "`" won't be handled correctly)
# this function fixes spaces around them based on input sequence, so here we're making the same double quote replacement
@@ -296,12 +296,11 @@ def post_process_punct(input: str, normalized_text: str, add_unicode_punct: bool
punct_marks = [x for x in string.punctuation if x in input]
if add_unicode_punct:
- punct_unicode = [
- chr(i)
- for i in range(sys.maxunicode)
- if category(chr(i)).startswith("P") and chr(i) not in punct_default and chr(i) in input
- ]
- punct_marks = punct_marks.extend(punct_unicode)
+ punct_marks.extend(
+ char
+ for char in dict.fromkeys(input)
+ if char not in string.punctuation and category(char).startswith("P")
+ )
for punct in punct_marks:
try:
diff --git a/tests/test_text_normalization_punctuation.py b/tests/test_text_normalization_punctuation.py
new file mode 100644
index 000000000..436bd2eef
--- /dev/null
+++ b/tests/test_text_normalization_punctuation.py
@@ -0,0 +1,16 @@
+from fun_text_processing.text_normalization.data_loader_utils import post_process_punct
+
+
+def test_post_process_punct_preserves_ascii_punctuation_spacing():
+ assert post_process_punct("test' example", "test 'example") == "test' example"
+
+
+def test_post_process_punct_handles_unicode_punctuation():
+ assert (
+ post_process_punct(
+ "\u4f60\u597d\uff0c\u4e16\u754c\uff01",
+ "\u4f60\u597d \uff0c \u4e16\u754c \uff01",
+ add_unicode_punct=True,
+ )
+ == "\u4f60\u597d\uff0c\u4e16\u754c\uff01"
+ )