SOTA Open Source TTS
add fastapi for inference (#12)
spicysama06a35aef5304
diff --git a/fish_speech/models/text2semantic/llama.py b/fish_speech/models/text2semantic/llama.pyindex 7d4c8a4..37f7dc6 100644--- a/fish_speech/models/text2semantic/llama.py+++ b/fish_speech/models/text2semantic/llama.py@@ -110,7 +110,9 @@ class Transformer(nn.Module):self.max_batch_size = -1self.max_seq_len = -1- def setup_caches(self, max_batch_size, max_seq_len, dtype=torch.bfloat16):+ def setup_caches(+ self, max_batch_size: int, max_seq_len: int, dtype: torch.dtype = torch.bfloat16+ ):if self.max_seq_len >= max_seq_len and self.max_batch_size >= max_batch_size:returndiff --git a/pyproject.toml b/pyproject.tomlindex ba21221..fdab719 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -33,7 +33,8 @@ dependencies = ["pyopenjtalk","wandb","tensorboard",- "grpcio>=1.58.0"+ "grpcio>=1.58.0",+ "kui>=1.6.0"][build-system]diff --git a/tools/api_server.py b/tools/api_server.pynew file mode 100644index 0000000..8ef177f--- /dev/null+++ b/tools/api_server.py@@ -0,0 +1,384 @@+import gc+import io+import time+import traceback+from http import HTTPStatus+from typing import Annotated, Any, Literal, Optional++import numpy as np+import soundfile as sf+import torch+import torch.nn.functional as F+from hydra import compose, initialize+from hydra.utils import instantiate+from kui.wsgi import (+ Body,+ HTTPException,+ HttpView,+ JSONResponse,+ Kui,+ OpenAPI,+ Path,+ StreamResponse,+ allow_cors,+)+from kui.wsgi.routing import MultimethodRoutes, Router+from loguru import logger+from pydantic import BaseModel+from transformers import AutoTokenizer++import tools.llama.generate+from tools.llama.generate import encode_tokens, generate, load_model+++# Define utils for web server+def http_execption_handler(exc: HTTPException):+ return JSONResponse(+ dict(+ statusCode=exc.status_code,+ message=exc.content,+ error=HTTPStatus(exc.status_code).phrase,+ ),+ exc.status_code,+ exc.headers,+ )+++def other_exception_handler(exc: "Exception"):+ traceback.print_exc()++ status = HTTPStatus.INTERNAL_SERVER_ERROR+ return JSONResponse(+ dict(statusCode=status, message=str(exc), error=status.phrase),+ status,+ )+++routes = MultimethodRoutes(base_class=HttpView)++# Define models+MODELS = {}+++class LlamaModel:+ def __init__(+ self,+ config_name: str,+ checkpoint_path: str,+ device,+ precision: str,+ tokenizer_path: str,+ compile: bool,+ ):+ self.device = device+ self.compile = compile++ self.t0 = time.time()+ self.precision = torch.bfloat16 if precision == "bfloat16" else torch.float16+ self.model = load_model(config_name, checkpoint_path, device, self.precision)+ self.model_size = sum(+ p.numel() for p in self.model.parameters() if p.requires_grad+ )++ torch.cuda.synchronize()+ logger.info(f"Time to load model: {time.time() - self.t0:.02f} seconds")++ if self.tokenizer is None:+ self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)++ if self.compile:+ logger.info("Compiling model ...")+ tools.llama.generate.decode_one_token = torch.compile(+ tools.llama.generate.decode_one_token,+ mode="reduce-overhead",+ fullgraph=True,+ )++ def __del__(self):+ self.model = None+ self.tokenizer = None++ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()++ logger.info("The llama is removed from memory.")+++class VQGANModel:+ def __init__(self, config_name: str, checkpoint_path: str):+ if self.cfg is None:+ with initialize(version_base="1.3", config_path="../fish_speech/configs"):+ self.cfg = compose(config_name=config_name)++ self.model = instantiate(self.cfg.model)+ state_dict = torch.load(+ checkpoint_path,+ map_location=self.model.device,+ )+ if "state_dict" in state_dict:+ state_dict = state_dict["state_dict"]+ self.model.load_state_dict(state_dict, strict=True)+ self.model.eval()+ self.model.cuda()+ logger.info("Restored model from checkpoint")++ def __del__(self):+ self.cfg = None+ self.model = None++ gc.collect()+ if torch.cuda.is_available():+ torch.cuda.empty_cache()++ logger.info("The vqgan model is removed from memory.")++ @torch.no_grad()+ @torch.autocast(device_type="cuda", enabled=True)+ def sematic_to_wav(self, indices):+ model = self.model+ indices = indices.to(model.device).long()+ indices = indices.unsqueeze(1).unsqueeze(-1)++ mel_lengths = indices.shape[2] * (+ model.downsample.total_strides if model.downsample is not None else 1+ )+ mel_lengths = torch.tensor([mel_lengths], device=model.device, dtype=torch.long)+ mel_masks = torch.ones(+ (1, 1, mel_lengths), device=model.device, dtype=torch.float32+ )++ text_features = model.vq_encoder.decode(indices)++ logger.info(+ f"VQ Encoded, indices: {indices.shape} equivalent to "+ + f"{1 / (mel_lengths[0] * model.hop_length / model.sampling_rate / indices.shape[2]):.2f} Hz"+ )++ text_features = F.interpolate(+ text_features, size=mel_lengths[0], mode="nearest"+ )++ # Sample mels+ decoded_mels = model.decoder(text_features, mel_masks)+ fake_audios = model.generator(decoded_mels)+ logger.info(+ f"Generated audio of shape {fake_audios.shape}, equivalent to {fake_audios.shape[-1] / model.sampling_rate:.2f} seconds"+ )++ # Save audio+ fake_audio = fake_audios[0, 0].cpu().numpy().astype(np.float32)++ return fake_audio, model.sampling_rate+++class LoadLlamaModelRequest(BaseModel):+ config_name: str = "text2semantic_finetune"+ checkpoint_path: str = "checkpoints/text2semantic-400m-v0.2-4k.pth"+ device: str = "cuda"+ precision: Literal["float16", "bfloat16"] = "bfloat16"+ tokenizer: str = "fishaudio/speech-lm-v1"+ compile: bool = True+++class LoadVQGANModelRequest(BaseModel):+ config_name: str = "vqgan_pretrain"+ checkpoint_path: str = "checkpoints/vqgan-v1.pth"+++class LoadModelResponse(BaseModel):+ name: str++[email protected]("/models/{name}")+def load_model(+ name: Annotated[str, Path("default")],+ llama: Annotated[LoadLlamaModelRequest, Body()],+ vqgan: Annotated[LoadVQGANModelRequest, Body()],+) -> Annotated[LoadModelResponse, JSONResponse[200, {}, LoadModelResponse]]:+ """+ Load model+ """++ if name in MODELS:+ del MODELS[name]++ logger.info("Loading model ...")+ new_model = {+ "llama": LlamaModel(+ config_name=llama.config_name,+ checkpoint_path=llama.checkpoint_path,+ device=llama.device,+ precision=llama.precision,+ tokenizer_path=llama.tokenizer,+ compile=llama.compile,+ ),+ "vqgan": VQGANModel(+ config_name=vqgan.config_name,+ checkpoint_path=vqgan.checkpoint_path,+ ),+ }++ MODELS[name] = new_model++ return LoadModelResponse(name=name)++[email protected]("/models/{name}")+def delete_model(+ name: Annotated[str, Path("default")],+) -> JSONResponse[200, {}, dict]:+ """+ Delete model+ """++ if name not in MODELS:+ raise HTTPException(+ status_code=HTTPStatus.BAD_REQUEST,+ content="Model not found.",+ )++ return JSONResponse(+ dict(message="Model deleted."),+ 200,+ )++[email protected]("/models")+def list_models() -> JSONResponse[200, {}, dict]:+ """+ List models+ """++ return JSONResponse(+ dict(models=list(MODELS.keys())),+ 200,+ )+++class InvokeRequest(BaseModel):+ text: str = "你说的对, 但是原神是一款由米哈游自主研发的开放世界手游."+ prompt_text: Optional[str] = None+ prompt_tokens: Optional[str] = None+ max_new_tokens: int = 0+ top_k: Optional[int] = None+ top_p: float = 0.5+ repetition_penalty: float = 1.5+ temperature: float = 0.7+ use_g2p: bool = True+ seed: Optional[int] = None+ speaker: Optional[str] = None++[email protected]("/models/{name}/invoke")+def invoke_model(+ name: Annotated[str, Path("default")],+ req: Annotated[InvokeRequest, Body(exclusive=True)],+):+ """+ Invoke model and generate audio+ """++ if name not in MODELS:+ raise HTTPException(+ status_code=HTTPStatus.NOT_FOUND,+ content="Cannot find model.",+ )++ model = MODELS[name]+ llama_model_manager = model["llama"]+ vqgan_model_manager = model["vqgan"]++ device = llama_model_manager.device+ seed = req.seed+ prompt_tokens = req.prompt_tokens+ logger.info(f"Device: {device}")++ prompt_tokens = (+ torch.from_numpy(np.load(prompt_tokens)).to(device)+ if prompt_tokens is not None+ else None+ )+ encoded = encode_tokens(+ llama_model_manager.tokenizer,+ req.text,+ prompt_text=req.prompt_text,+ prompt_tokens=prompt_tokens,+ bos=True,+ device=device,+ use_g2p=req.use_g2p,+ speaker=req.speaker,+ )+ prompt_length = encoded.size(1)+ logger.info(f"Encoded prompt shape: {encoded.shape}")++ if seed is not None:+ torch.manual_seed(seed)+ torch.cuda.manual_seed(seed)++ torch.cuda.synchronize()++ t0 = time.perf_counter()+ y = generate(+ model=llama_model_manager.model,+ prompt=encoded,+ max_new_tokens=req.max_new_tokens,+ eos_token_id=llama_model_manager.tokenizer.eos_token_id,+ precision=llama_model_manager.precision,+ temperature=req.temperature,+ top_k=req.top_k,+ top_p=req.top_p,+ repetition_penalty=req.repetition_penalty,+ )++ torch.cuda.synchronize()+ t = time.perf_counter() - t0++ tokens_generated = y.size(1) - prompt_length+ tokens_sec = tokens_generated / t+ logger.info(+ f"Generated {tokens_generated} tokens in {t:.02f} seconds, {tokens_sec:.02f} tokens/sec"+ )+ logger.info(+ f"Bandwidth achieved: {llama_model_manager.model_size * tokens_sec / 1e9:.02f} GB/s"+ )+ logger.info(f"GPU Memory used: {torch.cuda.max_memory_reserved() / 1e9:.02f} GB")++ codes = y[1:, prompt_length:-1]+ codes = codes - 2+ assert (codes >= 0).all(), "Codes should be >= 0"++ # --------------- llama end ------------+ audio, sr = vqgan_model_manager.sematic_to_wav(codes)+ # --------------- vqgan end ------------++ buffer = io.BytesIO()+ sf.write(buffer, audio, sr, format="wav")++ return StreamResponse(+ iterable=[buffer.getvalue()],+ headers={+ "Content-Disposition": "attachment; filename=generated.wav",+ "Content-Type": "audio/wav",+ },+ )+++# Define Kui app+app = Kui(+ exception_handlers={+ HTTPException: http_execption_handler,+ Exception: other_exception_handler,+ },+)+app.router = Router(+ [],+ http_middlewares=[+ app.exception_middleware,+ allow_cors(),+ ],+)++# Swagger UI & routes+app.router << ("/v1" // routes)+app.router << ("/docs" // OpenAPI().routes)diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex b8ff491..e7b5cce 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -180,10 +180,10 @@ def decode_n_tokens(enable_flash=False, enable_mem_efficient=False, enable_math=True): # Actually better for Inductor to codegen attention herenext_token = decode_one_token(- model,- cur_token,- input_pos,- window,+ model=model,+ x=cur_token,+ input_pos=input_pos,+ previous_tokens=window,**sampling_kwargs,)@@ -434,6 +434,8 @@ def main(logger.info(f"Encoded prompt shape: {encoded.shape}")torch.manual_seed(seed)+ torch.cuda.manual_seed(seed)+if compile:global decode_one_tokendecode_one_token = torch.compile(