Skip to content

SOTA-Class TTS at Compact Scale

Python

Normalize CJK line breaks without spaces

linux295650-ux7b86053ae756
diff --git a/audio8_tts_data.py b/audio8_tts_data.py
index 068286d..37dd739 100644
--- a/audio8_tts_data.py
+++ b/audio8_tts_data.py
@@ -18,14 +18,37 @@ import torch.nn.functional as F
NUM_CODEBOOKS = 10
CODEBOOK_SIZE = 4096
+_CJK_RANGES = (
+ "\u1100-\u11ff\u2e80-\u2fdf\u3000-\u303f\u3040-\u30ff\u3100-\u31ff"
+ "\u3400-\u4dbf\u4e00-\u9fff\ua960-\ua97f\uac00-\ud7a3\ud7b0-\ud7ff\uf900-\ufaff"
+ "\ufe30-\ufe4f\uff01-\uff9f\U00020000-\U0002fa1f"
+)
+_CJK_CHARACTER_RE = re.compile(rf"[{_CJK_RANGES}]")
+_LINE_BREAK_RE = re.compile(r"[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]")
+
+
+def _normalize_whitespace(text: str) -> str:
+ def replace(match: re.Match[str]) -> str:
+ left = text[match.start() - 1] if match.start() else ""
+ right = text[match.end()] if match.end() < len(text) else ""
+ if (
+ _LINE_BREAK_RE.search(match.group())
+ and _CJK_CHARACTER_RE.fullmatch(left)
+ and _CJK_CHARACTER_RE.fullmatch(right)
+ ):
+ return ""
+ return " "
+
+ return re.sub(r"\s+", replace, text).strip()
+
def clean_text(value: Any, *, field_name: str = "text") -> str:
"""Remove non-linguistic controls and normalize whitespace for inference."""
text = "".join(
- " " if char.isspace() else "" if unicodedata.category(char).startswith("C") else char
+ char if char.isspace() else "" if unicodedata.category(char).startswith("C") else char
for char in str(value)
)
- text = " ".join(text.split())
+ text = _normalize_whitespace(text)
if not text:
raise ValueError(f"{field_name} must not be empty")
return text
diff --git a/onnx_runtime/arktts_runtime/prompt.py b/onnx_runtime/arktts_runtime/prompt.py
index bad1590..b4190b6 100644
--- a/onnx_runtime/arktts_runtime/prompt.py
+++ b/onnx_runtime/arktts_runtime/prompt.py
@@ -7,13 +7,36 @@ from pathlib import Path
import numpy as np
from tokenizers import Tokenizer
+_CJK_RANGES = (
+ "\u1100-\u11ff\u2e80-\u2fdf\u3000-\u303f\u3040-\u30ff\u3100-\u31ff"
+ "\u3400-\u4dbf\u4e00-\u9fff\ua960-\ua97f\uac00-\ud7a3\ud7b0-\ud7ff\uf900-\ufaff"
+ "\ufe30-\ufe4f\uff01-\uff9f\U00020000-\U0002fa1f"
+)
+_CJK_CHARACTER_RE = re.compile(rf"[{_CJK_RANGES}]")
+_LINE_BREAK_RE = re.compile(r"[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]")
+
+
+def _normalize_whitespace(text: str) -> str:
+ def replace(match: re.Match[str]) -> str:
+ left = text[match.start() - 1] if match.start() else ""
+ right = text[match.end()] if match.end() < len(text) else ""
+ if (
+ _LINE_BREAK_RE.search(match.group())
+ and _CJK_CHARACTER_RE.fullmatch(left)
+ and _CJK_CHARACTER_RE.fullmatch(right)
+ ):
+ return ""
+ return " "
+
+ return re.sub(r"\s+", replace, text).strip()
+
def clean_text(text: str) -> str:
value = "".join(
- " " if char.isspace() else "" if unicodedata.category(char).startswith("C") else char
+ char if char.isspace() else "" if unicodedata.category(char).startswith("C") else char
for char in str(text)
)
- return " ".join(value.split())
+ return _normalize_whitespace(value)
def format_reference_text(text: str) -> str:
diff --git a/tests/test_clean_text.py b/tests/test_clean_text.py
new file mode 100644
index 0000000..2d62c2c
--- /dev/null
+++ b/tests/test_clean_text.py
@@ -0,0 +1,29 @@
+import pytest
+
+from audio8_tts_data import clean_text
+
+
+ ("value", "expected"),
+ [
+ ("你\n好", "你好"),
+ ("你 \t\n 好", "你好"),
+ ("你 好", "你 好"),
+ ("こんにちは。\n世界", "こんにちは。世界"),
+ ("안녕하세요\n세계", "안녕하세요세계"),
+ ("안녕하세요 세계", "안녕하세요 세계"),
+ ("hello\nworld", "hello world"),
+ ("你好\nworld", "你好 world"),
+ ("こんにちは\nworld", "こんにちは world"),
+ ("안녕하세요\nworld", "안녕하세요 world"),
+ ("hello\n世界", "hello 世界"),
+ ("甲\U00020000乙", "甲\U00020000乙"),
+ ],
+)
+def test_clean_text_normalizes_whitespace_by_script(value: str, expected: str) -> None:
+ assert clean_text(value) == expected
+
+
+def test_clean_text_rejects_whitespace_only_input() -> None:
+ with pytest.raises(ValueError, match="text must not be empty"):
+ clean_text(" \n\t")