SOTA Open Source TTS
Fix breakdown infer (#534)
spicysama6e95d2ae3e58
diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex ad9c549..d717ce7 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -605,7 +605,7 @@ def launch_thread_safe_queue(multiple=True,)@click.option("--num-samples", type=int, default=1)[email protected]("--max-new-tokens", type=int, default=0)[email protected]("--max-new-tokens", type=int, default=1024)@click.option("--top-p", type=float, default=0.7)@click.option("--repetition-penalty", type=float, default=1.2)@click.option("--temperature", type=float, default=0.7)@@ -650,7 +650,10 @@ def main(model, decode_one_token = load_model(checkpoint_path, device, precision, compile=compile)-+ with torch.device(device):+ model.setup_caches(+ max_batch_size=1, max_seq_len=2048, dtype=next(model.parameters()).dtype+ )if torch.cuda.is_available():torch.cuda.synchronize()