SOTA Open Source TTS
Add g2p config and semantic convert to api server
Lengyue3a08434bfb06
diff --git a/tools/api_server.py b/tools/api_server.pyindex 57e596d..b574031 100644--- a/tools/api_server.py+++ b/tools/api_server.py@@ -6,6 +6,7 @@ from http import HTTPStatusfrom threading import Lockfrom typing import Annotated, Literal, Optional+import librosaimport numpy as npimport soundfile as sfimport torch@@ -29,6 +30,7 @@ from pydantic import BaseModelfrom transformers import AutoTokenizerimport tools.llama.generate+from fish_speech.models.vqgan.utils import sequence_maskfrom tools.llama.generate import encode_tokens, generate, load_model@@ -133,7 +135,6 @@ class VQGANModel:logger.info("The vqgan model is removed from memory.")@torch.no_grad()- @torch.autocast(device_type="cuda", enabled=True)def sematic_to_wav(self, indices):model = self.modelindices = indices.to(model.device).long()@@ -170,6 +171,57 @@ class VQGANModel:return fake_audio, model.sampling_rate+ @torch.no_grad()+ def wav_to_semantic(self, audio):+ model = self.model+ # Load audio+ audio, _ = librosa.load(+ audio,+ sr=model.sampling_rate,+ mono=True,+ )+ audios = torch.from_numpy(audio).to(model.device)[None, None, :]+ logger.info(+ f"Loaded audio with {audios.shape[2] / model.sampling_rate:.2f} seconds"+ )++ # VQ Encoder+ audio_lengths = torch.tensor(+ [audios.shape[2]], device=model.device, dtype=torch.long+ )++ features = gt_mels = model.mel_transform(+ audios, sample_rate=model.sampling_rate+ )++ if model.downsample is not None:+ features = model.downsample(features)++ mel_lengths = audio_lengths // model.hop_length+ feature_lengths = (+ audio_lengths+ / model.hop_length+ / (model.downsample.total_strides if model.downsample is not None else 1)+ ).long()++ feature_masks = torch.unsqueeze(+ sequence_mask(feature_lengths, features.shape[2]), 1+ ).to(gt_mels.dtype)++ # vq_features is 50 hz, need to convert to true mel size+ text_features = model.mel_encoder(features, feature_masks)+ _, indices, _ = model.vq_encoder(text_features, feature_masks)++ if indices.ndim == 4 and indices.shape[1] == 1 and indices.shape[3] == 1:+ indices = indices[:, 0, :, 0]+ else:+ logger.error(f"Unknown indices shape: {indices.shape}")+ return++ logger.info(f"Generated indices of shape {indices.shape}")++ return indices+class LoadLlamaModelRequest(BaseModel):config_name: str = "text2semantic_finetune"@@ -275,6 +327,7 @@ class InvokeRequest(BaseModel):top_p: float = 0.5repetition_penalty: float = 1.5temperature: float = 0.7+ order: str = "zh,jp,en"use_g2p: bool = Trueseed: Optional[int] = Nonespeaker: Optional[str] = None@@ -299,19 +352,27 @@ def api_invoke_model(llama_model_manager = model["llama"]vqgan_model_manager = model["vqgan"]- # Lock- model["lock"].acquire()-device = llama_model_manager.deviceseed = req.seedprompt_tokens = req.prompt_tokenslogger.info(f"Device: {device}")- prompt_tokens = (- torch.from_numpy(np.load(prompt_tokens)).to(device)- if prompt_tokens is not None- else None- )+ if prompt_tokens is not None and prompt_tokens.endswith(".npy"):+ prompt_tokens = torch.from_numpy(np.load(prompt_tokens)).to(device)+ elif prompt_tokens is not None and prompt_tokens.endswith(".wav"):+ prompt_tokens = vqgan_model_manager.wav_to_semantic(prompt_tokens)+ elif prompt_tokens is not None:+ logger.error(f"Unknown prompt tokens: {prompt_tokens}")+ raise HTTPException(+ status_code=HTTPStatus.BAD_REQUEST,+ content="Unknown prompt tokens, it should be either .npy or .wav file.",+ )+ else:+ prompt_tokens = None++ # Lock+ model["lock"].acquire()+encoded = encode_tokens(llama_model_manager.tokenizer,req.text,@@ -321,6 +382,7 @@ def api_invoke_model(device=device,use_g2p=req.use_g2p,speaker=req.speaker,+ order=req.order,)prompt_length = encoded.size(1)logger.info(f"Encoded prompt shape: {encoded.shape}")diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex c84a69b..3a02bd6 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -268,12 +268,14 @@ def encode_tokens(prompt_tokens=None,use_g2p=False,speaker=None,+ order="zh,jp,en",):if prompt_text is not None:string = prompt_text + " " + stringif use_g2p:- prompt = g2p(string)+ order = order.split(",")+ prompt = g2p(string, order=order)prompt = [(f"<p:{i}>" if i not in pu_symbols and i != pad_symbol else i)for _, i in prompt@@ -382,6 +384,7 @@ def load_model(config_name, checkpoint_path, device, precision):@click.option("--use-g2p/--no-g2p", default=True)@click.option("--seed", type=int, default=42)@click.option("--speaker", type=str, default=None)[email protected]("--order", type=str, default="zh,jp,en")@click.option("--half/--no-half", default=False)def main(text: str,@@ -400,6 +403,7 @@ def main(use_g2p: bool,seed: int,speaker: Optional[str],+ order: str,half: bool,) -> None:device = "cuda"@@ -429,6 +433,7 @@ def main(device=device,use_g2p=use_g2p,speaker=speaker,+ order=order,)prompt_length = encoded.size(1)logger.info(f"Encoded prompt shape: {encoded.shape}")