SOTA Open Source TTS
Add queue to support streaming
Lengyuedcbe986fc96e
diff --git a/tools/api.py b/tools/api.pyindex 3672a7e..68593bd 100644--- a/tools/api.py+++ b/tools/api.py@@ -1,5 +1,6 @@import base64import io+import queueimport threadingimport tracebackfrom argparse import ArgumentParser@@ -114,17 +115,26 @@ def inference(req: InvokeRequest):)payload = dict(- event=threading.Event(),+ response_queue=queue.Queue(),request=request,)llama_queue.put(payload)- # Wait for the result- payload["event"].wait()- if payload["success"] is False:- raise payload["response"]+ codes = []+ while True:+ result = payload["response_queue"].get()+ if result == "next":+ # TODO: handle next sentence+ continue- codes = payload["response"][0]+ if result == "done":+ if payload["success"] is False:+ raise payload["response"]+ break++ codes.append(result)++ codes = torch.cat(codes, dim=1)# VQGAN Inferencefeature_lengths = torch.tensor([codes.shape[1]], device=vqgan_model.device)diff --git a/tools/llama/generate.py b/tools/llama/generate.pyindex 36b6596..82d04a4 100644--- a/tools/llama/generate.py+++ b/tools/llama/generate.py@@ -470,16 +470,14 @@ def generate_long(texts = split_text(text, chunk_length) if iterative_prompt else [text]if use_prompt:- encoded.append(- encode_tokens(- tokenizer,- prompt_text,- prompt_tokens=prompt_tokens,- bos=True,- device=device,- speaker=speaker,- num_codebooks=model.config.num_codebooks,- )+ encoded_prompts = encode_tokens(+ tokenizer,+ prompt_text,+ prompt_tokens=prompt_tokens,+ bos=True,+ device=device,+ speaker=speaker,+ num_codebooks=model.config.num_codebooks,)for idx, text in enumerate(texts):@@ -501,10 +499,6 @@ def generate_long(all_codes = []seg_idx = 0- if use_prompt:- seg_idx = 1- global_encoded.append(encoded[0])-while seg_idx < len(encoded):logger.info(f"Generating sentence {seg_idx + 1}/{len(encoded)} of sample {sample_idx + 1}/{num_samples}"@@ -531,6 +525,9 @@ def generate_long(else:partial_encoded = global_encoded+ if use_prompt:+ partial_encoded = [encoded_prompts] + partial_encoded+cat_encoded = torch.cat(partial_encoded, dim=1)prompt_length = cat_encoded.size(1)@@ -593,7 +590,7 @@ def generate_long(if is_streaming:# This indicates the end of the current sample- yield None+ yield "next"else:all_codes = torch.cat(all_codes, dim=1)assert (all_codes >= 0).all(), f"Negative code found: {codes}"@@ -623,20 +620,21 @@ def launch_thread_safe_queue(breakkwargs = item["request"]- event = item["event"]+ response_queue = item["response_queue"]try:item["success"] = True- item["response"] = list(- generate_long(- model=model, decode_one_token=decode_one_token, **kwargs- )- )+ for chunk in generate_long(+ model=model, decode_one_token=decode_one_token, **kwargs+ ):+ response_queue.put(chunk)++ response_queue.put("done")except Exception as e:item["success"] = Falseitem["response"] = e- event.set()+ response_queue.put("done")threading.Thread(target=worker, daemon=True).start()init_event.wait()diff --git a/tools/webui.py b/tools/webui.pyindex 7c41d84..829dd7f 100644--- a/tools/webui.py+++ b/tools/webui.py@@ -1,6 +1,7 @@import gcimport htmlimport os+import queueimport threadingfrom argparse import ArgumentParserfrom pathlib import Path@@ -119,17 +120,26 @@ def inference()payload = dict(- event=threading.Event(),+ response_queue=queue.Queue(),request=request,)llama_queue.put(payload)- # Wait for the result- payload["event"].wait()- if payload["success"] is False:- raise payload["response"]+ codes = []+ while True:+ result = payload["response_queue"].get()+ if result == "next":+ # TODO: handle next sentence+ continue- codes = payload["response"][0]+ if result == "done":+ if payload["success"] is False:+ raise payload["response"]+ break++ codes.append(result)++ codes = torch.cat(codes, dim=1)# VQGAN Inferencefeature_lengths = torch.tensor([codes.shape[1]], device=vqgan_model.device)