Skip to content

Open-Source Frontier Voice AI

Python

feat: nginx-based data parallel for optimal ASR throughput

Jianwei Yu3817f74d46e3
diff --git a/docs/vibevoice-vllm-asr.md b/docs/vibevoice-vllm-asr.md
index fa43735..bde87c8 100644
--- a/docs/vibevoice-vllm-asr.md
+++ b/docs/vibevoice-vllm-asr.md
@@ -47,7 +47,9 @@ The launcher supports two types of GPU parallelism via `--tp` and `--dp` flags:
### Data Parallel (Recommended for scaling throughput)
-Run 4 independent replicas on 4 GPUs — vLLM automatically distributes incoming requests:
+Run 4 independent replicas on 4 GPUs with automatic load balancing behind a single port.
+When `--dp N` is specified (N > 1), the launcher automatically starts N independent vLLM
+processes behind an nginx reverse proxy for optimal throughput:
```bash
docker run -d --gpus '"device=0,1,2,3"' --name vibevoice-vllm \
@@ -62,6 +64,21 @@ docker run -d --gpus '"device=0,1,2,3"' --name vibevoice-vllm \
-c "python3 /app/vllm_plugin/scripts/start_server.py --dp 4"
```
+Run on all 8 GPUs:
+
+```bash
+docker run -d --gpus all --name vibevoice-vllm \
+ --ipc=host \
+ -p 8000:8000 \
+ -e VIBEVOICE_FFMPEG_MAX_CONCURRENCY=64 \
+ -e PYTORCH_ALLOC_CONF=expandable_segments:True \
+ -v $(pwd):/app \
+ -w /app \
+ --entrypoint bash \
+ vllm/vllm-openai:v0.14.1 \
+ -c "python3 /app/vllm_plugin/scripts/start_server.py --dp 8"
+```
+
### Tensor Parallel
Split a single model across 2 GPUs (useful if GPU memory is limited):
diff --git a/vllm_plugin/scripts/start_server.py b/vllm_plugin/scripts/start_server.py
index 78b85ef..375c29e 100644
--- a/vllm_plugin/scripts/start_server.py
+++ b/vllm_plugin/scripts/start_server.py
@@ -9,14 +9,21 @@ One-click deployment script that handles:
4. Generating tokenizer files
5. Starting vLLM server
+For DP > 1, launches N independent vLLM processes behind an nginx
+reverse proxy for optimal throughput (avoids single-process HTTP
+bottleneck of vLLM's built-in DP coordinator).
+
Usage:
python3 start_server.py [--model MODEL_ID] [--port PORT]
"""
import argparse
import os
+import signal
import subprocess
import sys
+import textwrap
+import time
def run_command(cmd: list[str], description: str, shell: bool = False) -> None:
@@ -77,26 +84,21 @@ def generate_tokenizer(model_path: str) -> None:
)
-def start_vllm_server(model_path: str, port: int,
- tensor_parallel_size: int = 1,
- data_parallel_size: int = 1) -> None:
- """Start vLLM server (replaces current process)."""
- print(f"\n{'='*60}")
- print(f" Starting vLLM server on port {port}")
- print(f" Tensor Parallel (TP): {tensor_parallel_size}")
- print(f" Data Parallel (DP): {data_parallel_size}")
- print(f"{'='*60}\n")
-
- vllm_cmd = [
+def _build_vllm_cmd(model_path: str, port: int,
+ tensor_parallel_size: int = 1,
+ data_parallel_size: int = 1,
+ max_num_seqs: int = 64,
+ max_model_len: int = 65536,
+ gpu_memory_utilization: float = 0.8) -> list[str]:
+ """Build the vllm serve command."""
+ return [
"vllm", "serve", model_path,
"--served-model-name", "vibevoice",
"--trust-remote-code",
"--dtype", "bfloat16",
- "--max-num-seqs", "64",
- "--max-model-len", "65536",
- # "--max-num-batched-tokens", "32768",
- "--gpu-memory-utilization", "0.8",
- # "--enforce-eager",
+ "--max-num-seqs", str(max_num_seqs),
+ "--max-model-len", str(max_model_len),
+ "--gpu-memory-utilization", str(gpu_memory_utilization),
"--no-enable-prefix-caching",
"--enable-chunked-prefill",
"--chat-template-content-format", "openai",
@@ -105,10 +107,208 @@ def start_vllm_server(model_path: str, port: int,
"--allowed-local-media-path", "/app",
"--port", str(port),
]
+
+
+def start_vllm_server(model_path: str, port: int,
+ tensor_parallel_size: int = 1,
+ data_parallel_size: int = 1,
+ max_num_seqs: int = 64,
+ max_model_len: int = 65536,
+ gpu_memory_utilization: float = 0.8) -> None:
+ """Start a single vLLM server (replaces current process)."""
+ print(f"\n{'='*60}")
+ print(f" Starting vLLM server on port {port}")
+ print(f" Tensor Parallel (TP): {tensor_parallel_size}")
+ print(f" Data Parallel (DP): {data_parallel_size}")
+ print(f" Max Num Seqs: {max_num_seqs}")
+ print(f" Max Model Len: {max_model_len}")
+ print(f" GPU Mem Utilization: {gpu_memory_utilization}")
+ print(f"{'='*60}\n")
+ vllm_cmd = _build_vllm_cmd(
+ model_path, port,
+ tensor_parallel_size=tensor_parallel_size,
+ data_parallel_size=data_parallel_size,
+ max_num_seqs=max_num_seqs,
+ max_model_len=max_model_len,
+ gpu_memory_utilization=gpu_memory_utilization,
+ )
os.execvp("vllm", vllm_cmd)
+def _install_nginx() -> None:
+ """Install nginx if not already available."""
+ if subprocess.run(["which", "nginx"], capture_output=True).returncode != 0:
+ run_command(["apt-get", "update"], "Updating package list for nginx")
+ run_command(
+ ["apt-get", "install", "-y", "nginx"],
+ "Installing nginx for load balancing"
+ )
+
+
+def _write_nginx_config(frontend_port: int, backend_ports: list[int]) -> str:
+ """Write nginx config for round-robin load balancing."""
+ backends = "\n".join(f" server 127.0.0.1:{p};" for p in backend_ports)
+ config = textwrap.dedent(f"""\
+ worker_processes auto;
+ worker_rlimit_nofile 65536;
+ error_log /dev/stderr warn;
+ pid /tmp/nginx.pid;
+
+ events {{
+ worker_connections 8192;
+ }}
+
+ http {{
+ access_log off;
+
+ upstream vllm_backends {{
+ least_conn;
+ {backends}
+ }}
+
+ server {{
+ listen {frontend_port};
+ client_max_body_size 200m;
+ client_body_buffer_size 10m;
+ proxy_buffering on;
+ proxy_buffer_size 64k;
+ proxy_buffers 16 64k;
+
+ location / {{
+ proxy_pass http://vllm_backends;
+ proxy_read_timeout 600s;
+ proxy_connect_timeout 10s;
+ proxy_send_timeout 600s;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ }}
+ }}
+ }}
+ """)
+ config_path = "/tmp/nginx_vllm.conf"
+ with open(config_path, "w") as f:
+ f.write(config)
+ return config_path
+
+
+def start_dp_server(model_path: str, frontend_port: int,
+ data_parallel_size: int,
+ tensor_parallel_size: int = 1,
+ max_num_seqs: int = 64,
+ max_model_len: int = 65536,
+ gpu_memory_utilization: float = 0.8) -> None:
+ """Start multiple vLLM workers behind nginx for data parallelism.
+
+ Launches N independent vLLM processes (one per GPU group) on internal
+ ports, with an nginx reverse proxy on the frontend port for load
+ balancing. This avoids the single-process HTTP bottleneck of vLLM's
+ built-in DP coordinator when handling large audio payloads.
+ """
+ import torch
+ num_gpus = torch.cuda.device_count()
+ gpus_per_replica = tensor_parallel_size
+ total_gpus_needed = data_parallel_size * gpus_per_replica
+ assert num_gpus >= total_gpus_needed, (
+ f"Need {total_gpus_needed} GPUs (dp={data_parallel_size} × tp={tensor_parallel_size}) "
+ f"but only {num_gpus} available"
+ )
+
+ _install_nginx()
+
+ # Assign internal ports: frontend_port + 100, +101, ...
+ backend_ports = [frontend_port + 100 + i for i in range(data_parallel_size)]
+
+ print(f"\n{'='*60}")
+ print(f" Starting DP server with nginx load balancing")
+ print(f" Frontend port: {frontend_port} (nginx)")
+ print(f" Backend ports: {backend_ports}")
+ print(f" Data Parallel: {data_parallel_size}")
+ print(f" Tensor Parallel: {tensor_parallel_size}")
+ print(f" GPUs per replica: {gpus_per_replica}")
+ print(f" Max Num Seqs: {max_num_seqs}")
+ print(f" Max Model Len: {max_model_len}")
+ print(f"{'='*60}\n")
+
+ # Write nginx config
+ nginx_conf = _write_nginx_config(frontend_port, backend_ports)
+
+ # Launch vLLM workers
+ workers: list[subprocess.Popen] = []
+ for rank in range(data_parallel_size):
+ gpu_start = rank * gpus_per_replica
+ gpu_ids = ",".join(str(gpu_start + j) for j in range(gpus_per_replica))
+ port = backend_ports[rank]
+
+ env = os.environ.copy()
+ env["CUDA_VISIBLE_DEVICES"] = gpu_ids
+
+ vllm_cmd = _build_vllm_cmd(
+ model_path, port,
+ tensor_parallel_size=tensor_parallel_size,
+ data_parallel_size=1, # Each worker is dp=1
+ max_num_seqs=max_num_seqs,
+ max_model_len=max_model_len,
+ gpu_memory_utilization=gpu_memory_utilization,
+ )
+
+ print(f" Launching worker rank={rank} on GPU(s) {gpu_ids}, port {port}")
+ proc = subprocess.Popen(vllm_cmd, env=env)
+ workers.append(proc)
+
+ # Start nginx
+ print(f"\n Starting nginx on port {frontend_port} ...")
+ nginx_proc = subprocess.Popen(
+ ["nginx", "-c", nginx_conf, "-g", "daemon off;"]
+ )
+
+ # Wait for all backends to be ready
+ print(" Waiting for all backends to be ready ...")
+ import urllib.request
+ for port in backend_ports:
+ url = f"http://127.0.0.1:{port}/v1/models"
+ for attempt in range(600): # up to 10 minutes
+ try:
+ urllib.request.urlopen(url, timeout=2)
+ print(f" ✅ Backend on port {port} is ready")
+ break
+ except Exception:
+ time.sleep(1)
+ else:
+ print(f" ❌ Backend on port {port} failed to start")
+
+ print(f"\n{'='*60}")
+ print(f" ✅ VibeVoice DP server ready on port {frontend_port}")
+ print(f" {data_parallel_size} replicas behind nginx load balancer")
+ print(f"{'='*60}\n")
+
+ # Handle shutdown: forward signals to all children
+ def _shutdown(signum, frame):
+ print("\nShutting down ...")
+ nginx_proc.terminate()
+ for w in workers:
+ w.terminate()
+ for w in workers:
+ w.wait(timeout=10)
+ nginx_proc.wait(timeout=5)
+ sys.exit(0)
+
+ signal.signal(signal.SIGTERM, _shutdown)
+ signal.signal(signal.SIGINT, _shutdown)
+
+ # Wait for any child to exit (indicates a failure)
+ while True:
+ for i, w in enumerate(workers):
+ ret = w.poll()
+ if ret is not None:
+ print(f" ❌ Worker {i} exited with code {ret}")
+ _shutdown(None, None)
+ if nginx_proc.poll() is not None:
+ print(f" ❌ nginx exited with code {nginx_proc.returncode}")
+ _shutdown(None, None)
+ time.sleep(1)
+
+
def main():
parser = argparse.ArgumentParser(
description="VibeVoice vLLM ASR Server - One-Click Deployment",
@@ -121,7 +321,7 @@ Examples:
# Use custom port
python3 start_server.py --port 8080
- # Data parallel: 4 independent replicas on 4 GPUs (load balancing)
+ # Data parallel: 4 replicas on 4 GPUs (nginx load balancing)
python3 start_server.py --dp 4
# Tensor parallel: split model across 2 GPUs
@@ -166,6 +366,27 @@ Examples:
dest="data_parallel_size",
help="Data parallel size: run N independent model replicas for load balancing (default: 1)"
)
+ parser.add_argument(
+ "--max-num-seqs",
+ type=int,
+ default=64,
+ dest="max_num_seqs",
+ help="Maximum number of sequences per batch (default: 64)"
+ )
+ parser.add_argument(
+ "--max-model-len",
+ type=int,
+ default=65536,
+ dest="max_model_len",
+ help="Maximum model context length (default: 65536)"
+ )
+ parser.add_argument(
+ "--gpu-memory-utilization",
+ type=float,
+ default=0.8,
+ dest="gpu_memory_utilization",
+ help="GPU memory utilization fraction (default: 0.8)"
+ )
args = parser.parse_args()
print("\n" + "="*60)
@@ -186,10 +407,25 @@ Examples:
if not args.skip_tokenizer:
generate_tokenizer(model_path)
- # Step 5: Start vLLM server
- start_vllm_server(model_path, args.port,
- tensor_parallel_size=args.tensor_parallel_size,
- data_parallel_size=args.data_parallel_size)
+ # Step 5: Start server
+ if args.data_parallel_size > 1:
+ start_dp_server(
+ model_path, args.port,
+ data_parallel_size=args.data_parallel_size,
+ tensor_parallel_size=args.tensor_parallel_size,
+ max_num_seqs=args.max_num_seqs,
+ max_model_len=args.max_model_len,
+ gpu_memory_utilization=args.gpu_memory_utilization,
+ )
+ else:
+ start_vllm_server(
+ model_path, args.port,
+ tensor_parallel_size=args.tensor_parallel_size,
+ data_parallel_size=1,
+ max_num_seqs=args.max_num_seqs,
+ max_model_len=args.max_model_len,
+ gpu_memory_utilization=args.gpu_memory_utilization,
+ )
if __name__ == "__main__":