High-Quality Voice Cloning TTS for 600+ Languages
Python
fix model loading without internet
Han Zhue7ef814012a6
diff --git a/omnivoice/cli/demo.py b/omnivoice/cli/demo.pyindex b14c3fc..f45ac77 100644--- a/omnivoice/cli/demo.py+++ b/omnivoice/cli/demo.py@@ -143,6 +143,12 @@ def build_parser() -> argparse.ArgumentParser:help="Skip loading Whisper ASR model. Reference text auto-transcription"" will be unavailable.",)+ parser.add_argument(+ "--asr-model",+ default="openai/whisper-large-v3-turbo",+ help="ASR model path or HuggingFace repo id"+ " (default: openai/whisper-large-v3-turbo).",+ )return parser@@ -523,6 +529,7 @@ def main(argv=None) -> int:device_map=device,dtype=torch.float16,load_asr=not args.no_asr,+ asr_model_name=args.asr_model,)print("Model loaded.")diff --git a/omnivoice/models/omnivoice.py b/omnivoice/models/omnivoice.pyindex 62a92f4..508182a 100644--- a/omnivoice/models/omnivoice.py+++ b/omnivoice/models/omnivoice.py@@ -38,9 +38,9 @@ from typing import Any, List, Optional, Unionimport numpy as npimport torch-import torchaudioimport torch.nn as nnimport torch.nn.functional as F+import torchaudiofrom torch.nn.attention.flex_attention import create_block_maskfrom transformers import (AutoFeatureExtractor,@@ -182,6 +182,14 @@ class OmniVoiceConfig(PretrainedConfig):self.audio_codebook_weights = audio_codebook_weights+def _resolve_model_path(name_or_path: str) -> str:+ if os.path.isdir(name_or_path):+ return name_or_path+ from huggingface_hub import snapshot_download++ return snapshot_download(name_or_path)++class OmniVoice(PreTrainedModel):_supports_flex_attn = True_supports_flash_attn_2 = True@@ -238,31 +246,23 @@ class OmniVoice(PreTrainedModel):logging.disable(logging.INFO)try:- model = super().from_pretrained(- pretrained_model_name_or_path, *args, **kwargs- )+ # Resolve to local path first; download only if not already cached+ resolved_path = _resolve_model_path(pretrained_model_name_or_path)- if not train_mode:- # Resolve local path for audio tokenizer subdirectory- if os.path.isdir(pretrained_model_name_or_path):- resolved_path = pretrained_model_name_or_path- else:- from huggingface_hub import snapshot_download+ model = super().from_pretrained(resolved_path, *args, **kwargs)- resolved_path = snapshot_download(pretrained_model_name_or_path)-- model.text_tokenizer = AutoTokenizer.from_pretrained(- pretrained_model_name_or_path- )+ if not train_mode:+ model.text_tokenizer = AutoTokenizer.from_pretrained(resolved_path)audio_tokenizer_path = os.path.join(resolved_path, "audio_tokenizer")if not os.path.isdir(audio_tokenizer_path):- # Fallback to the HuggingFace Hub path of transformers'- # HiggsAudioV2Tokenizer if the local subdirectory doesn't exist.- audio_tokenizer_path = "eustlb/higgs-audio-v2-tokenizer"+ audio_tokenizer_path = _resolve_model_path(+ "eustlb/higgs-audio-v2-tokenizer"+ )- # higgs-audio-v2-tokenizer does not support MPS (output channels > 65536)+ # higgs-audio-v2-tokenizer does not support MPS+ # (output channels > 65536)tokenizer_device = ("cpu" if str(model.device).startswith("mps") else model.device)@@ -292,7 +292,7 @@ class OmniVoice(PreTrainedModel):"""Load a Whisper ASR model for reference audio transcription.Args:- model_name: HuggingFace model name for the Whisper model.+ model_name: HuggingFace model name or local path for the Whisper model."""from transformers import pipeline as hf_pipeline@@ -300,6 +300,9 @@ class OmniVoice(PreTrainedModel):asr_dtype = (torch.float16 if str(self.device).startswith("cuda") else torch.float32)++ model_name = _resolve_model_path(model_name)+self._asr_pipe = hf_pipeline("automatic-speech-recognition",model=model_name,@@ -623,11 +626,13 @@ class OmniVoice(PreTrainedModel):waveform = np.mean(waveform, axis=0, keepdims=True)if sr != self.sampling_rate:waveform = torchaudio.functional.resample(- torch.from_numpy(waveform), orig_freq=sr, new_freq=self.sampling_rate,+ torch.from_numpy(waveform),+ orig_freq=sr,+ new_freq=self.sampling_rate,).numpy()ref_wav = waveform- ref_rms = float(np.sqrt(np.mean(ref_wav ** 2)))+ ref_rms = float(np.sqrt(np.mean(ref_wav**2)))if 0 < ref_rms < 0.1:ref_wav = ref_wav * 0.1 / ref_rms@@ -673,9 +678,7 @@ class OmniVoice(PreTrainedModel):clip_size = int(ref_wav.shape[-1] % chunk_size)ref_wav = ref_wav[:, :-clip_size] if clip_size > 0 else ref_wav# numpy → torch at tokenizer boundary- ref_wav_tensor = torch.from_numpy(ref_wav).to(- self.audio_tokenizer.device- )+ ref_wav_tensor = torch.from_numpy(ref_wav).to(self.audio_tokenizer.device)ref_audio_tokens = self.audio_tokenizer.encode(ref_wav_tensor.unsqueeze(0),).audio_codes.squeeze(diff --git a/omnivoice/training/builder.py b/omnivoice/training/builder.pyindex 28f3c71..848f4b7 100644--- a/omnivoice/training/builder.py+++ b/omnivoice/training/builder.py@@ -40,7 +40,7 @@ from omnivoice.data.batching import PackingIterableDatasetfrom omnivoice.data.collator import PackingDataCollatorfrom omnivoice.data.dataset import WebDatasetReader, prepare_data_manifests_from_jsonfrom omnivoice.data.processor import OmniVoiceSampleProcessor-from omnivoice.models.omnivoice import OmniVoice, OmniVoiceConfig+from omnivoice.models.omnivoice import OmniVoice, OmniVoiceConfig, _resolve_model_pathfrom omnivoice.training.config import TrainingConfiglogger = logging.getLogger(__name__)@@ -58,6 +58,7 @@ def build_model_and_tokenizer(if config.init_from_checkpointelse config.llm_name_or_path)+ tokenizer_path = _resolve_model_path(tokenizer_path)tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)if tokenizer.pad_token is None:tokenizer.pad_token = tokenizer.eos_token@@ -85,7 +86,8 @@ def build_model_and_tokenizer(train=True,)else:- llm_config = AutoConfig.from_pretrained(config.llm_name_or_path)+ resolved_llm = _resolve_model_path(config.llm_name_or_path)+ llm_config = AutoConfig.from_pretrained(resolved_llm)ov_config = OmniVoiceConfig(audio_vocab_size=config.audio_vocab_size,@@ -99,7 +101,7 @@ def build_model_and_tokenizer(hf_logging.set_verbosity_error() # suppress expected lm_head.weight warningsllm = AutoModel.from_pretrained(- config.llm_name_or_path,+ resolved_llm,attn_implementation="flex_attention",dtype=torch.float32,)