SOTA Open Source TTS
fix(inference): GPU memory leakage bug (#1040) (#1073)
胖虎遛二狗54458f7cd5d9
diff --git a/fish_speech/models/text2semantic/inference.py b/fish_speech/models/text2semantic/inference.pyindex 466e446..d940be4 100644--- a/fish_speech/models/text2semantic/inference.py+++ b/fish_speech/models/text2semantic/inference.py@@ -3,10 +3,9 @@ import queueimport threadingimport timeimport traceback-from contextlib import nullcontextfrom dataclasses import dataclassfrom pathlib import Path-from typing import Literal, Optional, Tuple, Union+from typing import Callable, Literal, Optional, Tuple, Unionimport clickimport numpy as np@@ -106,17 +105,17 @@ def decode_one_token_ar(repetition_penalty: torch.Tensor,audio_masks: torch.Tensor,audio_parts: torch.Tensor,- previous_tokens: torch.Tensor = None,+ previous_tokens: Optional[torch.Tensor] = None,) -> torch.Tensor:# print(x, torch.count_nonzero(vq_masks))- x = model.forward_generate(+ forward_result = model.forward_generate(x,input_pos,audio_masks=audio_masks,audio_parts=audio_parts,)- logits = x.logits # [:, -1:]- hidden_states = x.hidden_states # [:, -1:]+ logits = forward_result.logits # [:, -1:]+ hidden_states = forward_result.hidden_states # [:, -1:]codebooks = [sample(@@ -130,10 +129,11 @@ def decode_one_token_ar()[0]]- # Cleanup the cache+ # Only clear cache for fast_layers, avoid clearing main model cachefor layer in model.fast_layers:- layer.attention.kv_cache.k_cache.fill_(0)- layer.attention.kv_cache.v_cache.fill_(0)+ if hasattr(layer, "attention") and hasattr(layer.attention, "kv_cache"):+ layer.attention.kv_cache.k_cache.fill_(0)+ layer.attention.kv_cache.v_cache.fill_(0)input_pos = torch.tensor([0], device=hidden_states.device, dtype=torch.long)model.forward_generate_fast(hidden_states, input_pos)@@ -167,11 +167,15 @@ def decode_one_token_ar(codebooks.append(a)codebooks = torch.stack(codebooks, dim=1)++ # Only delete references, let Python GC handle cleanup+ del logits, hidden_states, forward_result+return codebooks.Tdef decode_n_tokens(- model: NaiveTransformer,+ model: DualARTransformer,cur_token: torch.Tensor,input_pos: torch.Tensor,num_new_tokens: int,@@ -220,6 +224,9 @@ def decode_n_tokens(if cur_token[0, 0, -1] == model.tokenizer.get_token_id(IM_END_TOKEN):break+ # Only clean up the large tensor+ del cur_token+return previous_tokens[:, : i + 1]@@ -227,7 +234,7 @@ def decode_n_tokens(@torch.inference_mode()def generate(*,- model: BaseTransformer,+ model: DualARTransformer,prompt: torch.Tensor,max_new_tokens: int,audio_masks: torch.Tensor,@@ -259,28 +266,51 @@ def generate(max_new_tokens = T_new - Tdevice, dtype = prompt.device, prompt.dtype- with torch.device(device):- model.setup_caches(- max_batch_size=num_samples,- max_seq_len=model.config.max_seq_len,- dtype=next(model.parameters()).dtype,- )++ # Critical fix: Only set up cache on first run or when necessary+ if not hasattr(model, "_cache_setup_done") or not model._cache_setup_done:+ with torch.device(device):+ model.setup_caches(+ max_batch_size=1, # Fixed to 1, avoid dynamic changes+ max_seq_len=model.config.max_seq_len,+ dtype=next(model.parameters()).dtype,+ )+ model._cache_setup_done = Truecodebook_dim = 1 + model.config.num_codebooks- input_pos = torch.arange(0, T, device=device)++ # Create new tensor each time, but try to reuse memory+ input_pos = torch.arange(0, T, device=device, dtype=torch.long)empty = torch.empty((codebook_dim, model.config.max_seq_len), dtype=dtype, device=device)empty[:, :T] = promptseq = empty- temperature = torch.tensor(- sampling_kwargs["temperature"], device=device, dtype=torch.bfloat16+ # Use pre-created fixed parameter tensors+ temperature = getattr(+ model, "fixed_temperature", torch.tensor(0.8, device=device, dtype=torch.float))- top_p = torch.tensor(sampling_kwargs["top_p"], device=device, dtype=torch.bfloat16)- repetition_penalty = torch.tensor(- sampling_kwargs["repetition_penalty"], device=device, dtype=torch.bfloat16+ top_p = getattr(+ model, "fixed_top_p", torch.tensor(0.8, device=device, dtype=torch.float))+ repetition_penalty = getattr(+ model,+ "fixed_repetition_penalty",+ torch.tensor(1.1, device=device, dtype=torch.float),+ )++ # If different parameter values are needed, directly modify existing tensors+ temp_val = sampling_kwargs.get("temperature", 0.7)+ top_p_val = sampling_kwargs.get("top_p", 0.7)+ rep_val = sampling_kwargs.get("repetition_penalty", 1.5)++ if abs(temperature.item() - temp_val) > 1e-6:+ temperature.fill_(temp_val)+ if abs(top_p.item() - top_p_val) > 1e-6:+ top_p.fill_(top_p_val)+ if abs(repetition_penalty.item() - rep_val) > 1e-6:+ repetition_penalty.fill_(rep_val)prefill_decode = decode_one_token_ar@@ -296,7 +326,9 @@ def generate()seq[:, T : T + 1] = first_token+ # Recreate input_posinput_pos = torch.tensor([T], device=device, dtype=torch.int)+x = decode_n_tokens(model,first_token.view(1, codebook_dim, -1),@@ -311,6 +343,10 @@ def generate()seq = seq[:, : T + 1 + x.size(1)]seq[:, T + 1 :] = x++ # Clean up temporary variables+ del first_token, x, prompt, empty, input_pos+return seq@@ -327,19 +363,18 @@ def init_model(checkpoint_path, device, precision, compile=False):else:raise ValueError("Unsupported model type")- # Initialize cache- with torch.device(device):- model.setup_caches(- max_batch_size=1,- max_seq_len=model.config.max_seq_len,- dtype=next(model.parameters()).dtype,- )+ # Pre-create fixed parameter tensors to avoid runtime creation+ model.fixed_temperature = torch.tensor(0.7, device=device, dtype=torch.float)+ model.fixed_top_p = torch.tensor(0.7, device=device, dtype=torch.float)+ model.fixed_repetition_penalty = torch.tensor(1.5, device=device, dtype=torch.float)++ # Mark whether cache has been initialized+ model._cache_setup_done = Falseif compile:logger.info("Compiling function...")decode_one_token = torch.compile(decode_one_token,- # mode="max-autotune-no-cudagraphs",backend="inductor" if torch.cuda.is_available() else "aot_eager",mode="reduce-overhead" if torch.cuda.is_available() else None,fullgraph=True,@@ -358,19 +393,19 @@ class GenerateResponse:def generate_long(*,model,- device: str | torch.device,- decode_one_token: callable,+ device: Union[str, torch.device],+ decode_one_token: Callable,text: str,num_samples: int = 1,max_new_tokens: int = 0,- top_p: int = 0.8,+ top_p: float = 0.8,repetition_penalty: float = 1.1,temperature: float = 0.8,compile: bool = False,iterative_prompt: bool = True,chunk_length: int = 512,- prompt_text: Optional[str | list[str]] = None,- prompt_tokens: Optional[torch.Tensor | list[torch.Tensor]] = None,+ prompt_text: Optional[Union[str, list[str]]] = None,+ prompt_tokens: Optional[Union[torch.Tensor, list[torch.Tensor]]] = None,):assert 0 < top_p <= 1, "top_p must be in (0, 1]"assert 0 < repetition_penalty < 2, "repetition_penalty must be in (0, 2)"@@ -381,11 +416,13 @@ def generate_long(prompt_text = [prompt_text]prompt_tokens = [prompt_tokens]- assert use_prompt is False or len(prompt_text) == len(- prompt_tokens- ), "Prompt text and tokens must have the same length"+ if use_prompt:+ assert len(prompt_text) == len(+ prompt_tokens+ ), "Prompt text and tokens must have the same length"- prompt_tokens = [i.cpu() for i in prompt_tokens]+ if prompt_tokens:+ prompt_tokens = [i.cpu() for i in prompt_tokens]model_size = sum(p.numel() for p in model.parameters() if p.requires_grad)tokenizer = model.tokenizer@@ -419,14 +456,6 @@ def generate_long(encoded = encoded.to(device=device)logger.info(f"Encoded text: {text}")- # Move temperature, top_p, repetition_penalty to device- # This is important so that changing params doesn't trigger recompile- temperature = torch.tensor(temperature, device=device, dtype=torch.float)- top_p = torch.tensor(top_p, device=device, dtype=torch.float)- repetition_penalty = torch.tensor(- repetition_penalty, device=device, dtype=torch.float- )-for sample_idx in range(num_samples):if torch.cuda.is_available():torch.cuda.synchronize()@@ -436,6 +465,7 @@ def generate_long(prompt_length = encoded.size(1)t0 = time.perf_counter()+y = generate(model=model,prompt=encoded,@@ -469,26 +499,26 @@ def generate_long()# Put the generated tokens- # since there is <im_end>, we remove last tokencodes = y[1:, prompt_length:-1].clone()assert (codes >= 0).all(), f"Negative code found"decoded = y[:, prompt_length:].clone()- # But for global encoding, we should keep the <im_end> token-global_encoded.append(decoded.cpu())assert (codes >= 0).all(), f"Negative code found: {codes}"+yield GenerateResponse(action="sample", codes=codes, text=text)seg_idx += 1- # This indicates the end of the current sample+ # Force GPU memory cleanup+ del y, decoded, codes+yield GenerateResponse(action="next")@dataclassclass WrappedGenerateResponse:status: Literal["success", "error"]- response: Optional[GenerateResponse | Exception] = None+ response: Optional[Union[GenerateResponse, Exception]] = None@dataclass@@ -533,9 +563,17 @@ def launch_thread_safe_queue(response_queue.put(WrappedGenerateResponse(status="success", response=chunk))++ # Only clear cache after complete request batch+ if torch.cuda.is_available():+ torch.cuda.empty_cache()+except Exception as e:logger.error(traceback.format_exc())response_queue.put(WrappedGenerateResponse(status="error", response=e))+ # Clear cache on error+ if torch.cuda.is_available():+ torch.cuda.empty_cache()threading.Thread(target=worker, daemon=True).start()init_event.wait()@@ -575,8 +613,8 @@ def launch_thread_safe_queue(@click.option("--output-dir", type=Path, default="temp")def main(text: str,- prompt_text: Optional[list[str]],- prompt_tokens: Optional[list[Path]],+ prompt_text: Optional[tuple[str, ...]],+ prompt_tokens: Optional[tuple[Path, ...]],num_samples: int,max_new_tokens: int,top_p: int,@@ -594,7 +632,11 @@ def main(os.makedirs(output_dir, exist_ok=True)precision = torch.half if half else torch.bfloat16- if prompt_text is not None and len(prompt_text) != len(prompt_tokens):+ if (+ prompt_text is not None+ and prompt_tokens is not None+ and len(prompt_text) != len(prompt_tokens)+ ):raise ValueError(f"Number of prompt text ({len(prompt_text)}) and prompt tokens ({len(prompt_tokens)}) should be the same")@@ -615,8 +657,9 @@ def main(logger.info(f"Time to load model: {time.time() - t0:.02f} seconds")+ prompt_tokens_list = Noneif prompt_tokens is not None:- prompt_tokens = [torch.from_numpy(np.load(p)) for p in prompt_tokens]+ prompt_tokens_list = [torch.from_numpy(np.load(p)) for p in prompt_tokens]torch.manual_seed(seed)@@ -636,8 +679,8 @@ def main(compile=compile,iterative_prompt=iterative_prompt,chunk_length=chunk_length,- prompt_text=prompt_text,- prompt_tokens=prompt_tokens,+ prompt_text=list(prompt_text) if prompt_text else None,+ prompt_tokens=prompt_tokens_list,)idx = 0