Skip to content

High-Quality Voice Cloning TTS for 600+ Languages

Python

Allow placing the ASR model on a chosen device (#224)

Chao Cao6e4339ffbb86
diff --git a/README.md b/README.md
index 2adcb00..e28416d 100644
--- a/README.md
+++ b/README.md
@@ -164,6 +164,8 @@ audio = model.generate(
# If you don't want to input `ref_text` manually, you can directly omit the `ref_text`.
# The model will use Whisper ASR to auto-transcribe it. To use a local copy (or
# a different Whisper model), pass `asr_model_name="..."` to `from_pretrained`.
+# To control which device Whisper is loaded on (e.g. another GPU in multi-GPU
+# setups, or the CPU), pass `asr_device="cuda:1"` (or `"cpu"`).
sf.write("out.wav", audio[0], 24000)
```
diff --git a/omnivoice/models/omnivoice.py b/omnivoice/models/omnivoice.py
index ff68aca..a9e0cbd 100644
--- a/omnivoice/models/omnivoice.py
+++ b/omnivoice/models/omnivoice.py
@@ -323,12 +323,14 @@ class OmniVoice(PreTrainedModel):
self.sampling_rate = None
self._asr_pipe = None
self._asr_model_name = "openai/whisper-large-v3-turbo"
+ self._asr_device = None
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
train_mode = kwargs.pop("train", False)
load_asr = kwargs.pop("load_asr", False)
asr_model_name = kwargs.pop("asr_model_name", None)
+ asr_device = kwargs.pop("asr_device", None)
# Suppress noisy INFO logs from transformers/huggingface_hub during loading
_prev_disable = logging.root.manager.disable
@@ -368,6 +370,8 @@ class OmniVoice(PreTrainedModel):
if asr_model_name is not None:
model._asr_model_name = asr_model_name
+ if asr_device is not None:
+ model._asr_device = asr_device
if load_asr:
model.load_asr_model()
finally:
@@ -379,7 +383,9 @@ class OmniVoice(PreTrainedModel):
# ASR support (optional, for auto-transcription)
# -------------------------------------------------------------------
- def load_asr_model(self, model_name: Optional[str] = None):
+ def load_asr_model(
+ self, model_name: Optional[str] = None, device: Optional[str] = None
+ ):
"""Load a Whisper ASR model for reference audio transcription.
Args:
@@ -387,28 +393,36 @@ class OmniVoice(PreTrainedModel):
model. Defaults to the ``asr_model_name`` passed to
:meth:`from_pretrained` (``openai/whisper-large-v3-turbo``
if unset).
+ device: Device to load the ASR model on (e.g. ``"cuda:1"`` or
+ ``"cpu"``). Defaults to the ``asr_device`` passed to
+ :meth:`from_pretrained`, falling back to the main model's
+ device (its first shard when sharded across GPUs).
"""
from transformers import pipeline as hf_pipeline
if model_name is None:
model_name = self._asr_model_name
+ if device is None:
+ device = self._asr_device if self._asr_device is not None else self.device
logger.info("Loading ASR model %s ...", model_name)
asr_dtype = (
- torch.float16
- if str(self.device).startswith(("cuda", "xpu"))
- else torch.float32
+ torch.float16 if str(device).startswith(("cuda", "xpu")) else torch.float32
)
model_name = _resolve_model_path(model_name)
+ # Use `device=` (single-device placement) rather than `device_map=`:
+ # pipeline() ignores a plain device string in `device_map` and
+ # auto-selects an accelerator, which is why the ASR model could not
+ # be moved off the default GPU (#180).
self._asr_pipe = hf_pipeline(
"automatic-speech-recognition",
model=model_name,
dtype=asr_dtype,
- device_map=self.device,
+ device=device,
)
- logger.info("ASR model loaded on %s.", self.device)
+ logger.info("ASR model loaded on %s.", device)
@torch.inference_mode()
def transcribe(