SOTA Open Source TTS
Support inference on mps device natively (#461)
Ftpscc7afe8d7bf6
diff --git a/.gitignore b/.gitignoreindex 88341de..007add1 100644--- a/.gitignore+++ b/.gitignore@@ -1,3 +1,4 @@+.DS_Store.pgx.*.pdm-python/fish_speech.egg-infodiff --git a/fish_speech/models/vqgan/modules/reference.py b/fish_speech/models/vqgan/modules/reference.pyindex 034d5c5..0d9c8c8 100644--- a/fish_speech/models/vqgan/modules/reference.py+++ b/fish_speech/models/vqgan/modules/reference.py@@ -4,6 +4,8 @@ import torchimport torch.nn.functional as Ffrom torch import nn+from fish_speech.utils import autocast_exclude_mps+from .wavenet import WaveNet@@ -96,7 +98,7 @@ class ReferenceEncoder(WaveNet):if __name__ == "__main__":- with torch.autocast(device_type="cpu", dtype=torch.bfloat16):+ with autocast_exclude_mps(device_type="cpu", dtype=torch.bfloat16):model = ReferenceEncoder(input_channels=128,output_channels=64,diff --git a/fish_speech/utils/__init__.py b/fish_speech/utils/__init__.pyindex c3e0106..0537851 100644--- a/fish_speech/utils/__init__.py+++ b/fish_speech/utils/__init__.py@@ -1,4 +1,5 @@from .braceexpand import braceexpand+from .context import autocast_exclude_mpsfrom .file import get_latest_checkpointfrom .instantiators import instantiate_callbacks, instantiate_loggersfrom .logger import RankedLogger@@ -18,4 +19,5 @@ __all__ = ["task_wrapper","braceexpand","get_latest_checkpoint",+ "autocast_exclude_mps",]diff --git a/fish_speech/utils/context.py b/fish_speech/utils/context.pynew file mode 100644index 0000000..f04a992--- /dev/null+++ b/fish_speech/utils/context.py@@ -0,0 +1,13 @@+from contextlib import nullcontext++import torch+++def autocast_exclude_mps(+ device_type: str, dtype: torch.dtype+) -> nullcontext | torch.autocast:+ return (+ nullcontext()+ if torch.backends.mps.is_available()+ else torch.autocast(device_type, dtype)+ )diff --git a/tools/api.py b/tools/api.pyindex 8a63792..9b6f3d2 100644--- a/tools/api.py+++ b/tools/api.py@@ -32,6 +32,7 @@ pyrootutils.setup_root(__file__, indicator=".project-root", pythonpath=True)# from fish_speech.models.vqgan.lit_module import VQGANfrom fish_speech.models.vqgan.modules.firefly import FireflyArchitecture+from fish_speech.utils import autocast_exclude_mpsfrom tools.auto_rerank import batch_asr, calculate_wer, is_chinese, load_modelfrom tools.llama.generate import (GenerateRequest,@@ -266,7 +267,7 @@ def inference(req: InvokeRequest):if result.action == "next":break- with torch.autocast(+ with autocast_exclude_mps(device_type=decoder_model.device.type, dtype=args.precision):fake_audios = decode_vq_tokens(diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex eaf195e..934c185 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -356,7 +356,10 @@ def load_model(checkpoint_path, device, precision, compile=False):if compile:logger.info("Compiling function...")decode_one_token = torch.compile(- decode_one_token, mode="reduce-overhead", fullgraph=True+ decode_one_token,+ fullgraph=True,+ backend="inductor" if torch.cuda.is_available() else "aot_eager",+ mode="reduce-overhead" if torch.cuda.is_available() else None,)return model.eval(), decode_one_tokendiff --git a/tools/webui.py b/tools/webui.pyindex 9e01233..da19f6b 100644--- a/tools/webui.py+++ b/tools/webui.py@@ -21,6 +21,7 @@ pyrootutils.setup_root(__file__, indicator=".project-root", pythonpath=True)from fish_speech.i18n import i18nfrom fish_speech.text.chn_text_norm.text import Text as ChnNormedText+from fish_speech.utils import autocast_exclude_mpsfrom tools.api import decode_vq_tokens, encode_referencefrom tools.auto_rerank import batch_asr, calculate_wer, is_chinese, load_modelfrom tools.llama.generate import (@@ -127,13 +128,8 @@ def inference(if result.action == "next":break- with torch.autocast(- device_type=(- "cpu"- if decoder_model.device.type == "mps"- else decoder_model.device.type- ),- dtype=args.precision,+ with autocast_exclude_mps(+ device_type=decoder_model.device.type, dtype=args.precision):fake_audios = decode_vq_tokens(decoder_model=decoder_model,