Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Make japanese hiragana and katakana characters work with ACE. (#7997)

comfyanonymous5d3cc85e1383
diff --git a/comfy/sd.py b/comfy/sd.py
index c6b6a3b19..161d96f1e 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -441,7 +441,7 @@ class VAE:
self.working_dtypes = [torch.float16, torch.bfloat16, torch.float32]
elif "vocoder.backbone.channel_layers.0.0.bias" in sd: #Ace Step Audio
self.first_stage_model = comfy.ldm.ace.vae.music_dcae_pipeline.MusicDCAE(source_sample_rate=44100)
- self.memory_used_encode = lambda shape, dtype: (shape[2] * 300) * model_management.dtype_size(dtype)
+ self.memory_used_encode = lambda shape, dtype: (shape[2] * 330) * model_management.dtype_size(dtype)
self.memory_used_decode = lambda shape, dtype: (shape[2] * shape[3] * 87000) * model_management.dtype_size(dtype)
self.latent_channels = 8
self.output_channels = 2
diff --git a/comfy/text_encoders/ace.py b/comfy/text_encoders/ace.py
index b6fe451bd..d650bb10d 100644
--- a/comfy/text_encoders/ace.py
+++ b/comfy/text_encoders/ace.py
@@ -7,7 +7,7 @@ import torch
import logging
from tokenizers import Tokenizer
-from .ace_text_cleaners import multilingual_cleaners
+from .ace_text_cleaners import multilingual_cleaners, japanese_to_romaji
SUPPORT_LANGUAGES = {
"en": 259, "de": 260, "fr": 262, "es": 284, "it": 285,
@@ -65,6 +65,14 @@ class VoiceBpeTokenizer:
if "spa" in lang:
lang = "es"
+ try:
+ line_out = japanese_to_romaji(line)
+ if line_out != line:
+ lang = "ja"
+ line = line_out
+ except:
+ pass
+
try:
if structure_pattern.match(line):
token_idx = self.encode(line, "en")
diff --git a/comfy/text_encoders/ace_text_cleaners.py b/comfy/text_encoders/ace_text_cleaners.py
index ad3612e5f..cd31d8d8c 100644
--- a/comfy/text_encoders/ace_text_cleaners.py
+++ b/comfy/text_encoders/ace_text_cleaners.py
@@ -4,6 +4,131 @@
import re
+def japanese_to_romaji(japanese_text):
+ """
+ Convert Japanese hiragana and katakana to romaji (Latin alphabet representation).
+
+ Args:
+ japanese_text (str): Text containing hiragana and/or katakana characters
+
+ Returns:
+ str: The romaji (Latin alphabet) equivalent
+ """
+ # Dictionary mapping kana characters to their romaji equivalents
+ kana_map = {
+ # Katakana characters
+ 'ア': 'a', 'イ': 'i', 'ウ': 'u', 'エ': 'e', 'オ': 'o',
+ 'カ': 'ka', 'キ': 'ki', 'ク': 'ku', 'ケ': 'ke', 'コ': 'ko',
+ 'サ': 'sa', 'シ': 'shi', 'ス': 'su', 'セ': 'se', 'ソ': 'so',
+ 'タ': 'ta', 'チ': 'chi', 'ツ': 'tsu', 'テ': 'te', 'ト': 'to',
+ 'ナ': 'na', 'ニ': 'ni', 'ヌ': 'nu', 'ネ': 'ne', 'ノ': 'no',
+ 'ハ': 'ha', 'ヒ': 'hi', 'フ': 'fu', 'ヘ': 'he', 'ホ': 'ho',
+ 'マ': 'ma', 'ミ': 'mi', 'ム': 'mu', 'メ': 'me', 'モ': 'mo',
+ 'ヤ': 'ya', 'ユ': 'yu', 'ヨ': 'yo',
+ 'ラ': 'ra', 'リ': 'ri', 'ル': 'ru', 'レ': 're', 'ロ': 'ro',
+ 'ワ': 'wa', 'ヲ': 'wo', 'ン': 'n',
+
+ # Katakana voiced consonants
+ 'ガ': 'ga', 'ギ': 'gi', 'グ': 'gu', 'ゲ': 'ge', 'ゴ': 'go',
+ 'ザ': 'za', 'ジ': 'ji', 'ズ': 'zu', 'ゼ': 'ze', 'ゾ': 'zo',
+ 'ダ': 'da', 'ヂ': 'ji', 'ヅ': 'zu', 'デ': 'de', 'ド': 'do',
+ 'バ': 'ba', 'ビ': 'bi', 'ブ': 'bu', 'ベ': 'be', 'ボ': 'bo',
+ 'パ': 'pa', 'ピ': 'pi', 'プ': 'pu', 'ペ': 'pe', 'ポ': 'po',
+
+ # Katakana combinations
+ 'キャ': 'kya', 'キュ': 'kyu', 'キョ': 'kyo',
+ 'シャ': 'sha', 'シュ': 'shu', 'ショ': 'sho',
+ 'チャ': 'cha', 'チュ': 'chu', 'チョ': 'cho',
+ 'ニャ': 'nya', 'ニュ': 'nyu', 'ニョ': 'nyo',
+ 'ヒャ': 'hya', 'ヒュ': 'hyu', 'ヒョ': 'hyo',
+ 'ミャ': 'mya', 'ミュ': 'myu', 'ミョ': 'myo',
+ 'リャ': 'rya', 'リュ': 'ryu', 'リョ': 'ryo',
+ 'ギャ': 'gya', 'ギュ': 'gyu', 'ギョ': 'gyo',
+ 'ジャ': 'ja', 'ジュ': 'ju', 'ジョ': 'jo',
+ 'ビャ': 'bya', 'ビュ': 'byu', 'ビョ': 'byo',
+ 'ピャ': 'pya', 'ピュ': 'pyu', 'ピョ': 'pyo',
+
+ # Katakana small characters and special cases
+ 'ッ': '', # Small tsu (doubles the following consonant)
+ 'ャ': 'ya', 'ュ': 'yu', 'ョ': 'yo',
+
+ # Katakana extras
+ 'ヴ': 'vu', 'ファ': 'fa', 'フィ': 'fi', 'フェ': 'fe', 'フォ': 'fo',
+ 'ウィ': 'wi', 'ウェ': 'we', 'ウォ': 'wo',
+
+ # Hiragana characters
+ 'あ': 'a', 'い': 'i', 'う': 'u', 'え': 'e', 'お': 'o',
+ 'か': 'ka', 'き': 'ki', 'く': 'ku', 'け': 'ke', 'こ': 'ko',
+ 'さ': 'sa', 'し': 'shi', 'す': 'su', 'せ': 'se', 'そ': 'so',
+ 'た': 'ta', 'ち': 'chi', 'つ': 'tsu', 'て': 'te', 'と': 'to',
+ 'な': 'na', 'に': 'ni', 'ぬ': 'nu', 'ね': 'ne', 'の': 'no',
+ 'は': 'ha', 'ひ': 'hi', 'ふ': 'fu', 'へ': 'he', 'ほ': 'ho',
+ 'ま': 'ma', 'み': 'mi', 'む': 'mu', 'め': 'me', 'も': 'mo',
+ 'や': 'ya', 'ゆ': 'yu', 'よ': 'yo',
+ 'ら': 'ra', 'り': 'ri', 'る': 'ru', 'れ': 're', 'ろ': 'ro',
+ 'わ': 'wa', 'を': 'wo', 'ん': 'n',
+
+ # Hiragana voiced consonants
+ 'が': 'ga', 'ぎ': 'gi', 'ぐ': 'gu', 'げ': 'ge', 'ご': 'go',
+ 'ざ': 'za', 'じ': 'ji', 'ず': 'zu', 'ぜ': 'ze', 'ぞ': 'zo',
+ 'だ': 'da', 'ぢ': 'ji', 'づ': 'zu', 'で': 'de', 'ど': 'do',
+ 'ば': 'ba', 'び': 'bi', 'ぶ': 'bu', 'べ': 'be', 'ぼ': 'bo',
+ 'ぱ': 'pa', 'ぴ': 'pi', 'ぷ': 'pu', 'ぺ': 'pe', 'ぽ': 'po',
+
+ # Hiragana combinations
+ 'きゃ': 'kya', 'きゅ': 'kyu', 'きょ': 'kyo',
+ 'しゃ': 'sha', 'しゅ': 'shu', 'しょ': 'sho',
+ 'ちゃ': 'cha', 'ちゅ': 'chu', 'ちょ': 'cho',
+ 'にゃ': 'nya', 'にゅ': 'nyu', 'にょ': 'nyo',
+ 'ひゃ': 'hya', 'ひゅ': 'hyu', 'ひょ': 'hyo',
+ 'みゃ': 'mya', 'みゅ': 'myu', 'みょ': 'myo',
+ 'りゃ': 'rya', 'りゅ': 'ryu', 'りょ': 'ryo',
+ 'ぎゃ': 'gya', 'ぎゅ': 'gyu', 'ぎょ': 'gyo',
+ 'じゃ': 'ja', 'じゅ': 'ju', 'じょ': 'jo',
+ 'びゃ': 'bya', 'びゅ': 'byu', 'びょ': 'byo',
+ 'ぴゃ': 'pya', 'ぴゅ': 'pyu', 'ぴょ': 'pyo',
+
+ # Hiragana small characters and special cases
+ 'っ': '', # Small tsu (doubles the following consonant)
+ 'ゃ': 'ya', 'ゅ': 'yu', 'ょ': 'yo',
+
+ # Common punctuation and spaces
+ ' ': ' ', # Japanese space
+ '、': ', ', '。': '. ',
+ }
+
+ result = []
+ i = 0
+
+ while i < len(japanese_text):
+ # Check for small tsu (doubling the following consonant)
+ if i < len(japanese_text) - 1 and (japanese_text[i] == 'っ' or japanese_text[i] == 'ッ'):
+ if i < len(japanese_text) - 1 and japanese_text[i+1] in kana_map:
+ next_romaji = kana_map[japanese_text[i+1]]
+ if next_romaji and next_romaji[0] not in 'aiueon':
+ result.append(next_romaji[0]) # Double the consonant
+ i += 1
+ continue
+
+ # Check for combinations with small ya, yu, yo
+ if i < len(japanese_text) - 1 and japanese_text[i+1] in ('ゃ', 'ゅ', 'ょ', 'ャ', 'ュ', 'ョ'):
+ combo = japanese_text[i:i+2]
+ if combo in kana_map:
+ result.append(kana_map[combo])
+ i += 2
+ continue
+
+ # Regular character
+ if japanese_text[i] in kana_map:
+ result.append(kana_map[japanese_text[i]])
+ else:
+ # If it's not in our map, keep it as is (might be kanji, romaji, etc.)
+ result.append(japanese_text[i])
+
+ i += 1
+
+ return ''.join(result)
+
def number_to_text(num, ordinal=False):
"""
Convert a number (int or float) to its text representation.