Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

Fix invalid escape sequences in text normalization and models (#3709)

mehdi Bda12182b6b80
diff --git a/fun_text_processing/text_normalization/normalize.py b/fun_text_processing/text_normalization/normalize.py
index c1cebdd09..7b10b65e6 100644
--- a/fun_text_processing/text_normalization/normalize.py
+++ b/fun_text_processing/text_normalization/normalize.py
@@ -351,7 +351,7 @@ class Normalizer:
upper_case_unicode = "\u0410-\u042F"
# Read and split transcript by utterance (roughly, sentences)
- split_pattern = f"(?<!\w\.\w.)(?<![A-Z{upper_case_unicode}][a-z{lower_case_unicode}]+\.)(?<![A-Z{upper_case_unicode}]\.)(?<=\.|\?|\!|\.”|\?”\!”)\s(?![0-9]+[a-z]*\.)"
+ split_pattern = f"(?<!\\w\\.\\w.)(?<![A-Z{upper_case_unicode}][a-z{lower_case_unicode}]+\\.)(?<![A-Z{upper_case_unicode}]\\.)(?<=\\.|\\?|\\!|\\.”|\\?”\\!”)\\s(?![0-9]+[a-z]*\\.)"
sentences = regex.split(split_pattern, text)
return sentences
diff --git a/funasr/models/fsmn_kws/encoder.py b/funasr/models/fsmn_kws/encoder.py
index ff4e2892d..407576bab 100755
--- a/funasr/models/fsmn_kws/encoder.py
+++ b/funasr/models/fsmn_kws/encoder.py
@@ -84,7 +84,7 @@ class LinearTransform(nn.Module):
dtype=torch.float32)
for i in range(self.output_dim):
line = fread.readline()
- splits = line.strip().strip('\[\]').strip().split()
+ splits = line.strip().strip('\\[\\]').strip().split()
assert len(splits) == self.input_dim
cols = torch.tensor([float(item) for item in splits],
dtype=torch.float32)
@@ -160,7 +160,7 @@ class AffineTransform(nn.Module):
dtype=torch.float32)
for i in range(self.output_dim):
line = fread.readline()
- splits = line.strip().strip('\[\]').strip().split()
+ splits = line.strip().strip('\\[\\]').strip().split()
assert len(splits) == self.input_dim
cols = torch.tensor([float(item) for item in splits],
dtype=torch.float32)
@@ -171,7 +171,7 @@ class AffineTransform(nn.Module):
linear_bias = self.state_dict()['linear.bias']
#print(linear_bias.shape)
bias_line = fread.readline()
- splits = bias_line.strip().strip('\[\]').strip().split()
+ splits = bias_line.strip().strip('\\[\\]').strip().split()
assert len(splits) == self.output_dim
new_bias = torch.tensor([float(item) for item in splits],
dtype=torch.float32)
@@ -331,7 +331,7 @@ class FSMNBlock(nn.Module):
self.dim = int(fsmn_split[1])
params_line = fread.readline()
- params_split = params_line.strip().strip('\[\]').strip().split()
+ params_split = params_line.strip().strip('\\[\\]').strip().split()
assert len(params_split) == 12
assert params_split[0] == '<LearnRateCoef>'
assert params_split[2] == '<LOrder>'
@@ -352,7 +352,7 @@ class FSMNBlock(nn.Module):
for i in range(self.lorder):
print('read conv_left weight -- %d' % i)
line = fread.readline()
- splits = line.strip().strip('\[\]').strip().split()
+ splits = line.strip().strip('\\[\\]').strip().split()
assert len(splits) == self.dim
cols = torch.tensor([float(item) for item in splits],
dtype=torch.float32)
@@ -375,7 +375,7 @@ class FSMNBlock(nn.Module):
for i in range(self.rorder):
print('read conv_right weight -- %d' % i)
line = fread.readline()
- splits = line.strip().strip('\[\]').strip().split()
+ splits = line.strip().strip('\\[\\]').strip().split()
assert len(splits) == self.dim
cols = torch.tensor([float(item) for item in splits],
dtype=torch.float32)
diff --git a/funasr/models/fun_asr_nano/tools/format5res.py b/funasr/models/fun_asr_nano/tools/format5res.py
index 3b11bb83a..619b9f67c 100644
--- a/funasr/models/fun_asr_nano/tools/format5res.py
+++ b/funasr/models/fun_asr_nano/tools/format5res.py
@@ -320,8 +320,8 @@ if __name__ == "__main__":
continue
name = tmp[0]
content = tmp[1]
- name = re.sub("\.pcm", "", name)
- name = re.sub("\.wav", "", name)
+ name = re.sub("\\.pcm", "", name)
+ name = re.sub("\\.wav", "", name)
content = recoformat(content)
content = numbersingle(content)
content = ch_number2digit(content)