Skip to content

High-Quality Voice Cloning TTS for 600+ Languages

Python

fix model loading without internet

Han Zhue7ef814012a6
diff --git a/omnivoice/cli/demo.py b/omnivoice/cli/demo.py
index b14c3fc..f45ac77 100644
--- a/omnivoice/cli/demo.py
+++ b/omnivoice/cli/demo.py
@@ -143,6 +143,12 @@ def build_parser() -> argparse.ArgumentParser:
help="Skip loading Whisper ASR model. Reference text auto-transcription"
" will be unavailable.",
)
+ parser.add_argument(
+ "--asr-model",
+ default="openai/whisper-large-v3-turbo",
+ help="ASR model path or HuggingFace repo id"
+ " (default: openai/whisper-large-v3-turbo).",
+ )
return parser
@@ -523,6 +529,7 @@ def main(argv=None) -> int:
device_map=device,
dtype=torch.float16,
load_asr=not args.no_asr,
+ asr_model_name=args.asr_model,
)
print("Model loaded.")
diff --git a/omnivoice/models/omnivoice.py b/omnivoice/models/omnivoice.py
index 62a92f4..508182a 100644
--- a/omnivoice/models/omnivoice.py
+++ b/omnivoice/models/omnivoice.py
@@ -38,9 +38,9 @@ from typing import Any, List, Optional, Union
import numpy as np
import torch
-import torchaudio
import torch.nn as nn
import torch.nn.functional as F
+import torchaudio
from torch.nn.attention.flex_attention import create_block_mask
from transformers import (
AutoFeatureExtractor,
@@ -182,6 +182,14 @@ class OmniVoiceConfig(PretrainedConfig):
self.audio_codebook_weights = audio_codebook_weights
+def _resolve_model_path(name_or_path: str) -> str:
+ if os.path.isdir(name_or_path):
+ return name_or_path
+ from huggingface_hub import snapshot_download
+
+ return snapshot_download(name_or_path)
+
+
class OmniVoice(PreTrainedModel):
_supports_flex_attn = True
_supports_flash_attn_2 = True
@@ -238,31 +246,23 @@ class OmniVoice(PreTrainedModel):
logging.disable(logging.INFO)
try:
- model = super().from_pretrained(
- pretrained_model_name_or_path, *args, **kwargs
- )
+ # Resolve to local path first; download only if not already cached
+ resolved_path = _resolve_model_path(pretrained_model_name_or_path)
- if not train_mode:
- # Resolve local path for audio tokenizer subdirectory
- if os.path.isdir(pretrained_model_name_or_path):
- resolved_path = pretrained_model_name_or_path
- else:
- from huggingface_hub import snapshot_download
+ model = super().from_pretrained(resolved_path, *args, **kwargs)
- resolved_path = snapshot_download(pretrained_model_name_or_path)
-
- model.text_tokenizer = AutoTokenizer.from_pretrained(
- pretrained_model_name_or_path
- )
+ if not train_mode:
+ model.text_tokenizer = AutoTokenizer.from_pretrained(resolved_path)
audio_tokenizer_path = os.path.join(resolved_path, "audio_tokenizer")
if not os.path.isdir(audio_tokenizer_path):
- # Fallback to the HuggingFace Hub path of transformers'
- # HiggsAudioV2Tokenizer if the local subdirectory doesn't exist.
- audio_tokenizer_path = "eustlb/higgs-audio-v2-tokenizer"
+ audio_tokenizer_path = _resolve_model_path(
+ "eustlb/higgs-audio-v2-tokenizer"
+ )
- # higgs-audio-v2-tokenizer does not support MPS (output channels > 65536)
+ # higgs-audio-v2-tokenizer does not support MPS
+ # (output channels > 65536)
tokenizer_device = (
"cpu" if str(model.device).startswith("mps") else model.device
)
@@ -292,7 +292,7 @@ class OmniVoice(PreTrainedModel):
"""Load a Whisper ASR model for reference audio transcription.
Args:
- model_name: HuggingFace model name for the Whisper model.
+ model_name: HuggingFace model name or local path for the Whisper model.
"""
from transformers import pipeline as hf_pipeline
@@ -300,6 +300,9 @@ class OmniVoice(PreTrainedModel):
asr_dtype = (
torch.float16 if str(self.device).startswith("cuda") else torch.float32
)
+
+ model_name = _resolve_model_path(model_name)
+
self._asr_pipe = hf_pipeline(
"automatic-speech-recognition",
model=model_name,
@@ -623,11 +626,13 @@ class OmniVoice(PreTrainedModel):
waveform = np.mean(waveform, axis=0, keepdims=True)
if sr != self.sampling_rate:
waveform = torchaudio.functional.resample(
- torch.from_numpy(waveform), orig_freq=sr, new_freq=self.sampling_rate,
+ torch.from_numpy(waveform),
+ orig_freq=sr,
+ new_freq=self.sampling_rate,
).numpy()
ref_wav = waveform
- ref_rms = float(np.sqrt(np.mean(ref_wav ** 2)))
+ ref_rms = float(np.sqrt(np.mean(ref_wav**2)))
if 0 < ref_rms < 0.1:
ref_wav = ref_wav * 0.1 / ref_rms
@@ -673,9 +678,7 @@ class OmniVoice(PreTrainedModel):
clip_size = int(ref_wav.shape[-1] % chunk_size)
ref_wav = ref_wav[:, :-clip_size] if clip_size > 0 else ref_wav
# numpy → torch at tokenizer boundary
- ref_wav_tensor = torch.from_numpy(ref_wav).to(
- self.audio_tokenizer.device
- )
+ ref_wav_tensor = torch.from_numpy(ref_wav).to(self.audio_tokenizer.device)
ref_audio_tokens = self.audio_tokenizer.encode(
ref_wav_tensor.unsqueeze(0),
).audio_codes.squeeze(
diff --git a/omnivoice/training/builder.py b/omnivoice/training/builder.py
index 28f3c71..848f4b7 100644
--- a/omnivoice/training/builder.py
+++ b/omnivoice/training/builder.py
@@ -40,7 +40,7 @@ from omnivoice.data.batching import PackingIterableDataset
from omnivoice.data.collator import PackingDataCollator
from omnivoice.data.dataset import WebDatasetReader, prepare_data_manifests_from_json
from omnivoice.data.processor import OmniVoiceSampleProcessor
-from omnivoice.models.omnivoice import OmniVoice, OmniVoiceConfig
+from omnivoice.models.omnivoice import OmniVoice, OmniVoiceConfig, _resolve_model_path
from omnivoice.training.config import TrainingConfig
logger = logging.getLogger(__name__)
@@ -58,6 +58,7 @@ def build_model_and_tokenizer(
if config.init_from_checkpoint
else config.llm_name_or_path
)
+ tokenizer_path = _resolve_model_path(tokenizer_path)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
@@ -85,7 +86,8 @@ def build_model_and_tokenizer(
train=True,
)
else:
- llm_config = AutoConfig.from_pretrained(config.llm_name_or_path)
+ resolved_llm = _resolve_model_path(config.llm_name_or_path)
+ llm_config = AutoConfig.from_pretrained(resolved_llm)
ov_config = OmniVoiceConfig(
audio_vocab_size=config.audio_vocab_size,
@@ -99,7 +101,7 @@ def build_model_and_tokenizer(
hf_logging.set_verbosity_error() # suppress expected lm_head.weight warnings
llm = AutoModel.from_pretrained(
- config.llm_name_or_path,
+ resolved_llm,
attn_implementation="flex_attention",
dtype=torch.float32,
)