Open-Source Frontier Voice AI
Python
feat: nginx-based data parallel for optimal ASR throughput
Jianwei Yu3817f74d46e3
diff --git a/docs/vibevoice-vllm-asr.md b/docs/vibevoice-vllm-asr.mdindex fa43735..bde87c8 100644--- a/docs/vibevoice-vllm-asr.md+++ b/docs/vibevoice-vllm-asr.md@@ -47,7 +47,9 @@ The launcher supports two types of GPU parallelism via `--tp` and `--dp` flags:### Data Parallel (Recommended for scaling throughput)-Run 4 independent replicas on 4 GPUs — vLLM automatically distributes incoming requests:+Run 4 independent replicas on 4 GPUs with automatic load balancing behind a single port.+When `--dp N` is specified (N > 1), the launcher automatically starts N independent vLLM+processes behind an nginx reverse proxy for optimal throughput:```bashdocker run -d --gpus '"device=0,1,2,3"' --name vibevoice-vllm \@@ -62,6 +64,21 @@ docker run -d --gpus '"device=0,1,2,3"' --name vibevoice-vllm \-c "python3 /app/vllm_plugin/scripts/start_server.py --dp 4"```+Run on all 8 GPUs:++```bash+docker run -d --gpus all --name vibevoice-vllm \+ --ipc=host \+ -p 8000:8000 \+ -e VIBEVOICE_FFMPEG_MAX_CONCURRENCY=64 \+ -e PYTORCH_ALLOC_CONF=expandable_segments:True \+ -v $(pwd):/app \+ -w /app \+ --entrypoint bash \+ vllm/vllm-openai:v0.14.1 \+ -c "python3 /app/vllm_plugin/scripts/start_server.py --dp 8"+```+### Tensor ParallelSplit a single model across 2 GPUs (useful if GPU memory is limited):diff --git a/vllm_plugin/scripts/start_server.py b/vllm_plugin/scripts/start_server.pyindex 78b85ef..375c29e 100644--- a/vllm_plugin/scripts/start_server.py+++ b/vllm_plugin/scripts/start_server.py@@ -9,14 +9,21 @@ One-click deployment script that handles:4. Generating tokenizer files5. Starting vLLM server+For DP > 1, launches N independent vLLM processes behind an nginx+reverse proxy for optimal throughput (avoids single-process HTTP+bottleneck of vLLM's built-in DP coordinator).+Usage:python3 start_server.py [--model MODEL_ID] [--port PORT]"""import argparseimport os+import signalimport subprocessimport sys+import textwrap+import timedef run_command(cmd: list[str], description: str, shell: bool = False) -> None:@@ -77,26 +84,21 @@ def generate_tokenizer(model_path: str) -> None:)-def start_vllm_server(model_path: str, port: int,- tensor_parallel_size: int = 1,- data_parallel_size: int = 1) -> None:- """Start vLLM server (replaces current process)."""- print(f"\n{'='*60}")- print(f" Starting vLLM server on port {port}")- print(f" Tensor Parallel (TP): {tensor_parallel_size}")- print(f" Data Parallel (DP): {data_parallel_size}")- print(f"{'='*60}\n")-- vllm_cmd = [+def _build_vllm_cmd(model_path: str, port: int,+ tensor_parallel_size: int = 1,+ data_parallel_size: int = 1,+ max_num_seqs: int = 64,+ max_model_len: int = 65536,+ gpu_memory_utilization: float = 0.8) -> list[str]:+ """Build the vllm serve command."""+ return ["vllm", "serve", model_path,"--served-model-name", "vibevoice","--trust-remote-code","--dtype", "bfloat16",- "--max-num-seqs", "64",- "--max-model-len", "65536",- # "--max-num-batched-tokens", "32768",- "--gpu-memory-utilization", "0.8",- # "--enforce-eager",+ "--max-num-seqs", str(max_num_seqs),+ "--max-model-len", str(max_model_len),+ "--gpu-memory-utilization", str(gpu_memory_utilization),"--no-enable-prefix-caching","--enable-chunked-prefill","--chat-template-content-format", "openai",@@ -105,10 +107,208 @@ def start_vllm_server(model_path: str, port: int,"--allowed-local-media-path", "/app","--port", str(port),]+++def start_vllm_server(model_path: str, port: int,+ tensor_parallel_size: int = 1,+ data_parallel_size: int = 1,+ max_num_seqs: int = 64,+ max_model_len: int = 65536,+ gpu_memory_utilization: float = 0.8) -> None:+ """Start a single vLLM server (replaces current process)."""+ print(f"\n{'='*60}")+ print(f" Starting vLLM server on port {port}")+ print(f" Tensor Parallel (TP): {tensor_parallel_size}")+ print(f" Data Parallel (DP): {data_parallel_size}")+ print(f" Max Num Seqs: {max_num_seqs}")+ print(f" Max Model Len: {max_model_len}")+ print(f" GPU Mem Utilization: {gpu_memory_utilization}")+ print(f"{'='*60}\n")+ vllm_cmd = _build_vllm_cmd(+ model_path, port,+ tensor_parallel_size=tensor_parallel_size,+ data_parallel_size=data_parallel_size,+ max_num_seqs=max_num_seqs,+ max_model_len=max_model_len,+ gpu_memory_utilization=gpu_memory_utilization,+ )os.execvp("vllm", vllm_cmd)+def _install_nginx() -> None:+ """Install nginx if not already available."""+ if subprocess.run(["which", "nginx"], capture_output=True).returncode != 0:+ run_command(["apt-get", "update"], "Updating package list for nginx")+ run_command(+ ["apt-get", "install", "-y", "nginx"],+ "Installing nginx for load balancing"+ )+++def _write_nginx_config(frontend_port: int, backend_ports: list[int]) -> str:+ """Write nginx config for round-robin load balancing."""+ backends = "\n".join(f" server 127.0.0.1:{p};" for p in backend_ports)+ config = textwrap.dedent(f"""\+ worker_processes auto;+ worker_rlimit_nofile 65536;+ error_log /dev/stderr warn;+ pid /tmp/nginx.pid;++ events {{+ worker_connections 8192;+ }}++ http {{+ access_log off;++ upstream vllm_backends {{+ least_conn;+ {backends}+ }}++ server {{+ listen {frontend_port};+ client_max_body_size 200m;+ client_body_buffer_size 10m;+ proxy_buffering on;+ proxy_buffer_size 64k;+ proxy_buffers 16 64k;++ location / {{+ proxy_pass http://vllm_backends;+ proxy_read_timeout 600s;+ proxy_connect_timeout 10s;+ proxy_send_timeout 600s;+ proxy_http_version 1.1;+ proxy_set_header Connection "";+ }}+ }}+ }}+ """)+ config_path = "/tmp/nginx_vllm.conf"+ with open(config_path, "w") as f:+ f.write(config)+ return config_path+++def start_dp_server(model_path: str, frontend_port: int,+ data_parallel_size: int,+ tensor_parallel_size: int = 1,+ max_num_seqs: int = 64,+ max_model_len: int = 65536,+ gpu_memory_utilization: float = 0.8) -> None:+ """Start multiple vLLM workers behind nginx for data parallelism.++ Launches N independent vLLM processes (one per GPU group) on internal+ ports, with an nginx reverse proxy on the frontend port for load+ balancing. This avoids the single-process HTTP bottleneck of vLLM's+ built-in DP coordinator when handling large audio payloads.+ """+ import torch+ num_gpus = torch.cuda.device_count()+ gpus_per_replica = tensor_parallel_size+ total_gpus_needed = data_parallel_size * gpus_per_replica+ assert num_gpus >= total_gpus_needed, (+ f"Need {total_gpus_needed} GPUs (dp={data_parallel_size} × tp={tensor_parallel_size}) "+ f"but only {num_gpus} available"+ )++ _install_nginx()++ # Assign internal ports: frontend_port + 100, +101, ...+ backend_ports = [frontend_port + 100 + i for i in range(data_parallel_size)]++ print(f"\n{'='*60}")+ print(f" Starting DP server with nginx load balancing")+ print(f" Frontend port: {frontend_port} (nginx)")+ print(f" Backend ports: {backend_ports}")+ print(f" Data Parallel: {data_parallel_size}")+ print(f" Tensor Parallel: {tensor_parallel_size}")+ print(f" GPUs per replica: {gpus_per_replica}")+ print(f" Max Num Seqs: {max_num_seqs}")+ print(f" Max Model Len: {max_model_len}")+ print(f"{'='*60}\n")++ # Write nginx config+ nginx_conf = _write_nginx_config(frontend_port, backend_ports)++ # Launch vLLM workers+ workers: list[subprocess.Popen] = []+ for rank in range(data_parallel_size):+ gpu_start = rank * gpus_per_replica+ gpu_ids = ",".join(str(gpu_start + j) for j in range(gpus_per_replica))+ port = backend_ports[rank]++ env = os.environ.copy()+ env["CUDA_VISIBLE_DEVICES"] = gpu_ids++ vllm_cmd = _build_vllm_cmd(+ model_path, port,+ tensor_parallel_size=tensor_parallel_size,+ data_parallel_size=1, # Each worker is dp=1+ max_num_seqs=max_num_seqs,+ max_model_len=max_model_len,+ gpu_memory_utilization=gpu_memory_utilization,+ )++ print(f" Launching worker rank={rank} on GPU(s) {gpu_ids}, port {port}")+ proc = subprocess.Popen(vllm_cmd, env=env)+ workers.append(proc)++ # Start nginx+ print(f"\n Starting nginx on port {frontend_port} ...")+ nginx_proc = subprocess.Popen(+ ["nginx", "-c", nginx_conf, "-g", "daemon off;"]+ )++ # Wait for all backends to be ready+ print(" Waiting for all backends to be ready ...")+ import urllib.request+ for port in backend_ports:+ url = f"http://127.0.0.1:{port}/v1/models"+ for attempt in range(600): # up to 10 minutes+ try:+ urllib.request.urlopen(url, timeout=2)+ print(f" ✅ Backend on port {port} is ready")+ break+ except Exception:+ time.sleep(1)+ else:+ print(f" ❌ Backend on port {port} failed to start")++ print(f"\n{'='*60}")+ print(f" ✅ VibeVoice DP server ready on port {frontend_port}")+ print(f" {data_parallel_size} replicas behind nginx load balancer")+ print(f"{'='*60}\n")++ # Handle shutdown: forward signals to all children+ def _shutdown(signum, frame):+ print("\nShutting down ...")+ nginx_proc.terminate()+ for w in workers:+ w.terminate()+ for w in workers:+ w.wait(timeout=10)+ nginx_proc.wait(timeout=5)+ sys.exit(0)++ signal.signal(signal.SIGTERM, _shutdown)+ signal.signal(signal.SIGINT, _shutdown)++ # Wait for any child to exit (indicates a failure)+ while True:+ for i, w in enumerate(workers):+ ret = w.poll()+ if ret is not None:+ print(f" ❌ Worker {i} exited with code {ret}")+ _shutdown(None, None)+ if nginx_proc.poll() is not None:+ print(f" ❌ nginx exited with code {nginx_proc.returncode}")+ _shutdown(None, None)+ time.sleep(1)++def main():parser = argparse.ArgumentParser(description="VibeVoice vLLM ASR Server - One-Click Deployment",@@ -121,7 +321,7 @@ Examples:# Use custom portpython3 start_server.py --port 8080- # Data parallel: 4 independent replicas on 4 GPUs (load balancing)+ # Data parallel: 4 replicas on 4 GPUs (nginx load balancing)python3 start_server.py --dp 4# Tensor parallel: split model across 2 GPUs@@ -166,6 +366,27 @@ Examples:dest="data_parallel_size",help="Data parallel size: run N independent model replicas for load balancing (default: 1)")+ parser.add_argument(+ "--max-num-seqs",+ type=int,+ default=64,+ dest="max_num_seqs",+ help="Maximum number of sequences per batch (default: 64)"+ )+ parser.add_argument(+ "--max-model-len",+ type=int,+ default=65536,+ dest="max_model_len",+ help="Maximum model context length (default: 65536)"+ )+ parser.add_argument(+ "--gpu-memory-utilization",+ type=float,+ default=0.8,+ dest="gpu_memory_utilization",+ help="GPU memory utilization fraction (default: 0.8)"+ )args = parser.parse_args()print("\n" + "="*60)@@ -186,10 +407,25 @@ Examples:if not args.skip_tokenizer:generate_tokenizer(model_path)- # Step 5: Start vLLM server- start_vllm_server(model_path, args.port,- tensor_parallel_size=args.tensor_parallel_size,- data_parallel_size=args.data_parallel_size)+ # Step 5: Start server+ if args.data_parallel_size > 1:+ start_dp_server(+ model_path, args.port,+ data_parallel_size=args.data_parallel_size,+ tensor_parallel_size=args.tensor_parallel_size,+ max_num_seqs=args.max_num_seqs,+ max_model_len=args.max_model_len,+ gpu_memory_utilization=args.gpu_memory_utilization,+ )+ else:+ start_vllm_server(+ model_path, args.port,+ tensor_parallel_size=args.tensor_parallel_size,+ data_parallel_size=1,+ max_num_seqs=args.max_num_seqs,+ max_model_len=args.max_model_len,+ gpu_memory_utilization=args.gpu_memory_utilization,+ )if __name__ == "__main__":