Open-Source Frontier Voice AI
Python
fix
YingboHAO0508c3e86fd3
diff --git a/vllm_plugin/__init__.py b/vllm_plugin/__init__.pyindex b696a45..989acb4 100644--- a/vllm_plugin/__init__.py+++ b/vllm_plugin/__init__.py@@ -15,7 +15,6 @@ from vibevoice.modular.configuration_vibevoice import VibeVoiceConfigfrom vibevoice.modular.modular_vibevoice_text_tokenizer import VibeVoiceASRTextTokenizerFastfrom .model import VibeVoiceForCausalLM-from .inputs import vibevoice_audio_input_mapperdef register_vibevoice():diff --git a/vllm_plugin/model.py b/vllm_plugin/model.pyindex 178fb74..2ca3cd2 100644--- a/vllm_plugin/model.py+++ b/vllm_plugin/model.py@@ -5,17 +5,11 @@ This module implements the VibeVoice ASR model with full vLLM multimodal registrintegration for speech-to-text inference."""-from typing import List, Optional, Tuple, Union, Dict, Any, Iterable, Mapping, Sequence, ClassVar, Literal-import json-import math+from typing import List, Optional, Tuple, Union, Dict, Any, Iterable, Mapping, Sequenceimport os-import sys-from pathlib import Pathimport torchimport torch.nn as nnimport numpy as np-from io import BytesIO-import tempfileimport base64@@ -29,32 +23,12 @@ import base64from vibevoice.processor.audio_utils import load_audio_use_ffmpeg, load_audio_bytes_use_ffmpeg, AudioNormalizer-def _suffix_from_media_type(media_type: str | None) -> str:- if not media_type:- return ".bin"- mt = media_type.lower().strip()- if mt in ("audio/wav", "audio/x-wav", "audio/wave"):- return ".wav"- if mt in ("audio/mpeg", "audio/mp3", "audio/x-mp3"):- return ".mp3"- if mt in ("audio/flac",):- return ".flac"- if mt in ("audio/ogg", "audio/opus"):- return ".ogg"- if mt in ("audio/mp4", "audio/m4a"):- return ".m4a"- if mt in ("video/mp4",):- return ".mp4"- return ".bin"---def _ffmpeg_load_bytes(data: bytes, *, media_type: str | None = None) -> tuple[np.ndarray, int]:- """Load audio bytes using FFmpeg.+def _ffmpeg_load_bytes(data: bytes) -> tuple[np.ndarray, int]:+ """Load audio bytes using FFmpeg via stdin-pipe decoding.Returns:Tuple of (audio_waveform, sample_rate). Sample rate is always 24000."""- # Prefer stdin-pipe decoding to avoid temp-file IO under high concurrency.audio, sr = load_audio_bytes_use_ffmpeg(data, resample=True, target_sr=24000)normalizer = AudioNormalizer()audio = normalizer(audio)@@ -79,10 +53,10 @@ class _PatchedAudioMediaIO(_OriginalAudioMediaIO):"""AudioMediaIO implementation using FFmpeg for audio decoding."""def load_bytes(self, data: bytes) -> tuple[np.ndarray, int]:- return _ffmpeg_load_bytes(data, media_type=None)+ return _ffmpeg_load_bytes(data)def load_base64(self, media_type: str, data: str) -> tuple[np.ndarray, int]:- return _ffmpeg_load_bytes(base64.b64decode(data), media_type=media_type)+ return _ffmpeg_load_bytes(base64.b64decode(data))def load_file(self, filepath) -> tuple[np.ndarray, int]:return _ffmpeg_load_file(filepath)@@ -96,17 +70,13 @@ _vllm_utils_module.AudioMediaIO = _PatchedAudioMediaIO# ============================================================================-from transformers import Qwen2Config, BatchFeature+from transformers import BatchFeaturefrom transformers.models.whisper import WhisperFeatureExtractor-from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM-from vllm.model_executor.model_loader.weight_utils import default_weight_loader-from vllm.config import VllmConfig, ModelConfig-from vllm.config.speech_to_text import SpeechToTextConfig+from vllm.config import VllmConfigfrom vllm.multimodal import MULTIMODAL_REGISTRYfrom vllm.multimodal.parse import MultiModalDataParserfrom vllm.sequence import IntermediateTensorsfrom vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP, MultiModalEmbeddings-from vllm.inputs import PromptTypefrom vllm.model_executor.models.utils import (init_vllm_registered_model,maybe_prefix,@@ -873,17 +843,6 @@ class VibeVoiceForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):with a causal language model for text generation."""- # SupportsTranscription interface- supports_transcription: ClassVar[Literal[True]] = True- supports_transcription_only: ClassVar[bool] = False- supports_segment_timestamp: ClassVar[bool] = False-- # Supported languages (Chinese as primary target)- supported_languages: ClassVar[Mapping[str, str]] = {- "zh": "Chinese",- "en": "English",- }-@classmethoddef get_placeholder_str(cls, modality: str, i: int) -> str | None:"""Return the placeholder string format for a given modality.@@ -897,112 +856,10 @@ class VibeVoiceForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):return "<|AUDIO|>"raise ValueError(f"Unsupported modality: {modality}")- @classmethod- def get_generation_prompt(- cls,- audio: np.ndarray,- stt_config: SpeechToTextConfig,- model_config: ModelConfig,- language: str | None,- task_type: Literal["transcribe", "translate"],- request_prompt: str,- to_language: str | None,- ) -> PromptType:- """Get the prompt for the ASR model.-- Generates a chat-formatted prompt for speech-to-text transcription- with JSON output format.- """- # If user provides custom prompt, use it- if request_prompt:- return request_prompt-- # Calculate audio duration for the prompt- # Audio should be at 24kHz, so duration = len(audio) / 24000- duration = len(audio) / 24000 if audio is not None else 10.0-- system_prompt = "You are a helpful assistant that transcribes audio input into text output in JSON format."- show_keys = ["Start time", "End time", "Speaker ID", "Content"]- user_suffix = (- f"This is a {duration:.2f} seconds audio, please transcribe it with these keys: "- + ", ".join(show_keys)- )-- # IMPORTANT: keep <|AUDIO|> as the only placeholder token here.- # `_get_prompt_updates` expands it into repeated `<|AUDIO|>` placeholders.- user_content = "<|AUDIO|>\n" + user_suffix-- prompt = (- f"<|im_start|>system\n{system_prompt}<|im_end|>\n"- f"<|im_start|>user\n{user_content}<|im_end|>\n"- "<|im_start|>assistant\n"- )- return prompt-- @classmethod- def get_speech_to_text_config(- cls, model_config: ModelConfig, task_type: Literal["transcribe", "translate"]- ) -> SpeechToTextConfig:- """Get the speech to text config for the ASR model."""- return SpeechToTextConfig(- language=None, # Auto-detect or use request language- task_type=task_type,- )-- @classmethod- def get_num_audio_tokens(- cls,- audio_duration_s: float,- stt_config: SpeechToTextConfig,- model_config: ModelConfig,- ) -> int | None:- """Estimate number of audio tokens from duration.-- Returns the number of audio EMBEDDING positions (speech_pad_id tokens).- Note: _get_prompt_updates actually generates:- [speech_start_id] + [speech_pad_id] * N + [speech_end_id] + [newline_id]- So total prompt tokens = N + 3, but this returns N (the embedding count).- """- sampling_rate = 24000- compress_ratio = 3200- samples = int(audio_duration_s * sampling_rate)- num_tokens = int(np.ceil(samples / compress_ratio))- return num_tokens-- @classmethod- def get_other_languages(cls) -> Mapping[str, str]:- """Get languages from Whisper map not natively supported."""- # Import LANGUAGES from vllm- try:- from vllm.transformers_utils.tokenizer import LANGUAGES- except ImportError:- # Fallback to empty dict if import fails- return {}- return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}-- @classmethod- def validate_language(cls, language: str | None) -> str | None:- """Validate the language code."""- if language is None or language in cls.supported_languages:- return language- elif language in cls.get_other_languages():- print(f"Warning: Language {language!r} is not natively supported")- return language- else:- raise ValueError(- f"Unsupported language: {language!r}. "- f"Supported: {list(cls.supported_languages.keys())}"- )-def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):super().__init__()config = vllm_config.model_config.hf_configself.config = config-- # Keep a copy of the resolved model path for any custom weight-loading- # logic (e.g., loading audio encoder weights in fp32 directly from- # safetensors shards).- self._model_path = vllm_config.model_config.modelself.audio_encoder = VibeVoiceAudioEncoder(config)@@ -1100,14 +957,14 @@ class VibeVoiceForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):# Process each audio through the VibeVoice encoderembeddings = []- # Get model device and dtype for alignment+ # Get model device for tensor placement.+ # dtype is NOT set here — audio_encoder.forward() handles it internally:+ # input: converted to fp32 (self._audio_encoder_dtype)+ # output: converted to bfloat16 (self._lm_dtype)try:device = next(self.audio_encoder.parameters()).device- dtype = next(self.audio_encoder.parameters()).dtypeexcept StopIteration:- # Fallback if no parameters (shouldn't happen)device = torch.device("cuda" if torch.cuda.is_available() else "cpu")- dtype = torch.bfloat16# Handle both stacked tensor and list of tensors# vLLM batches as: [batch_size, 1, seq_len] or [batch_size, seq_len]@@ -1138,7 +995,7 @@ class VibeVoiceForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):if not isinstance(audio_tensor, torch.Tensor):audio_tensor = torch.tensor(audio_tensor)- # Let vLLM handle dtype (bfloat16 by default)+ # Only place on correct device; audio_encoder.forward() handles dtypeaudio_tensor = audio_tensor.to(device=device)# Get actual length if available, otherwise use full length@@ -1294,35 +1151,31 @@ class VibeVoiceForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):intermediate_tensors: Intermediate tensors for pipeline parallelism.inputs_embeds: Pre-computed embeddings (from multimodal merge or decode)."""- try:- # PRIORITY: Use inputs_embeds if provided (from vLLM multimodal merge or decode)- # Only compute from input_ids if inputs_embeds is not available- if inputs_embeds is None and input_ids is not None:- # Compute embeddings from input_ids- inputs_embeds = self.get_input_embeddings()(input_ids)-- # If we have intermediate tensors (pipeline parallelism), don't use inputs_embeds- if intermediate_tensors is not None:- inputs_embeds = None-- # Get the inner model - handle both wrapped and direct language models- language_model = self.language_model- if hasattr(language_model, "language_model"):- language_model = language_model.language_model-- # Call the language model's model (Qwen2Model)- # vLLM V1 passes kv_caches and attn_metadata via context, not arguments- # IMPORTANT: Pass input_ids=None when using inputs_embeds to avoid double embedding- hidden_states = language_model.model(- input_ids=None, # Always None when we have inputs_embeds- positions=positions,- intermediate_tensors=intermediate_tensors,- inputs_embeds=inputs_embeds- )- return hidden_states-- except Exception as e:- raise+ # PRIORITY: Use inputs_embeds if provided (from vLLM multimodal merge or decode)+ # Only compute from input_ids if inputs_embeds is not available+ if inputs_embeds is None and input_ids is not None:+ # Compute embeddings from input_ids+ inputs_embeds = self.get_input_embeddings()(input_ids)++ # If we have intermediate tensors (pipeline parallelism), don't use inputs_embeds+ if intermediate_tensors is not None:+ inputs_embeds = None++ # Get the inner model - handle both wrapped and direct language models+ language_model = self.language_model+ if hasattr(language_model, "language_model"):+ language_model = language_model.language_model++ # Call the language model's model (Qwen2Model)+ # vLLM V1 passes kv_caches and attn_metadata via context, not arguments+ # IMPORTANT: Pass input_ids=None when using inputs_embeds to avoid double embedding+ hidden_states = language_model.model(+ input_ids=None, # Always None when we have inputs_embeds+ positions=positions,+ intermediate_tensors=intermediate_tensors,+ inputs_embeds=inputs_embeds+ )+ return hidden_states# Alias for training checkpoint compatibility