Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: add GLM-ASR vLLM inference engine

xiaoyunchong.xyc65ab846c3483
diff --git a/funasr/auto/auto_model_vllm.py b/funasr/auto/auto_model_vllm.py
index 70bdec94f..c3d69879c 100644
--- a/funasr/auto/auto_model_vllm.py
+++ b/funasr/auto/auto_model_vllm.py
@@ -246,6 +246,15 @@ class AutoModelVLLM:
max_model_len=max_model_len, enforce_eager=enforce_eager,
**kwargs,
)
+ elif self.model_type in ("GLMASR", "glmasr"):
+ from funasr.models.glm_asr.inference_vllm import GLMASRVLLMEngine
+ self._engine = GLMASRVLLMEngine(
+ model_dir=self.model_dir, device=device, dtype=dtype,
+ tensor_parallel_size=tensor_parallel_size,
+ gpu_memory_utilization=gpu_memory_utilization,
+ max_model_len=max_model_len,
+ **kwargs,
+ )
elif self.model_type in ("LLMASR", "LLMASRNAR"):
self._engine = self._build_llmasr_engine(
config, tensor_parallel_size, gpu_memory_utilization,
diff --git a/funasr/models/glm_asr/inference_vllm.py b/funasr/models/glm_asr/inference_vllm.py
new file mode 100644
index 000000000..d2eaac625
--- /dev/null
+++ b/funasr/models/glm_asr/inference_vllm.py
@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""GLM-ASR vLLM inference engine.
+
+Architecture: audio_tower (Whisper-like) + multi_modal_projector + language_model (Llama)
+Strategy: audio_tower + projector in PyTorch, language_model in vLLM via EmbedsPrompt.
+
+Usage:
+ from funasr.models.glm_asr.inference_vllm import GLMASRVLLMEngine
+
+ engine = GLMASRVLLMEngine.from_pretrained("zai-org/GLM-ASR-Nano-2512")
+ results = engine.generate(inputs=["audio.wav"])
+ print(results[0]["text"])
+"""
+
+import glob
+import json
+import logging
+import os
+import re
+import shutil
+import time
+
+import numpy as np
+import torch
+
+logger = logging.getLogger(__name__)
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+
+def prepare_glmasr_vllm_dir(model_dir: str) -> str:
+ """Extract language_model weights into vLLM-compatible Llama format."""
+ output_dir = os.path.join(model_dir, "language_model_vllm")
+
+ if glob.glob(os.path.join(output_dir, "*.safetensors")):
+ logger.info(f"vLLM LM weights already at {output_dir}")
+ return output_dir
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ from safetensors import safe_open
+ from safetensors.torch import save_file
+
+ st_files = sorted(glob.glob(os.path.join(model_dir, "*.safetensors")))
+ lm_state = {}
+ for st_file in st_files:
+ with safe_open(st_file, framework="pt") as f:
+ for key in f.keys():
+ if key.startswith("language_model."):
+ lm_state[key[len("language_model."):]] = f.get_tensor(key)
+
+ if not lm_state:
+ raise RuntimeError("No language_model weights found in safetensors")
+
+ logger.info(f"Extracted {len(lm_state)} LM tensors")
+ save_file(lm_state, os.path.join(output_dir, "model.safetensors"))
+
+ with open(os.path.join(model_dir, "config.json")) as f:
+ full_config = json.load(f)
+ text_config = full_config["text_config"]
+ text_config["architectures"] = ["LlamaForCausalLM"]
+ text_config["model_type"] = "llama"
+ with open(os.path.join(output_dir, "config.json"), "w") as f:
+ json.dump(text_config, f, indent=2)
+
+ for fname in os.listdir(model_dir):
+ if "tokenizer" in fname or fname == "generation_config.json":
+ src = os.path.join(model_dir, fname)
+ dst = os.path.join(output_dir, fname)
+ if os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+
+ index = {
+ "metadata": {"total_size": sum(v.numel() * v.element_size() for v in lm_state.values())},
+ "weight_map": {k: "model.safetensors" for k in lm_state.keys()},
+ }
+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump(index, f, indent=2)
+
+ logger.info(f"Saved vLLM LM to {output_dir}")
+ return output_dir
+
+
+class GLMASRVLLMEngine:
+ """GLM-ASR with vLLM backend.
+
+ Audio tower + projector run in PyTorch on a single device.
+ Language model is decoded by vLLM with PagedAttention for high throughput.
+
+ Args:
+ model_dir: Path to GLM-ASR model directory.
+ device: Device for audio encoder.
+ dtype: Compute dtype ("bf16", "fp16", "fp32").
+ tensor_parallel_size: GPUs for vLLM tensor parallelism.
+ gpu_memory_utilization: GPU memory fraction for vLLM KV cache.
+ max_model_len: Maximum sequence length for vLLM.
+ """
+
+ def __init__(self, model_dir, device="cuda:0", dtype="bf16",
+ tensor_parallel_size=1, gpu_memory_utilization=0.5,
+ max_model_len=4096, **kwargs):
+ from vllm import LLM
+ from transformers import AutoProcessor, AutoConfig, AutoModel as HFAutoModel
+
+ self.device = device
+ self.torch_dtype = dtype_map.get(dtype, torch.bfloat16)
+ self.model_dir = model_dir
+
+ logger.info(f"Loading GLM-ASR audio components from {model_dir}")
+ full_model = HFAutoModel.from_pretrained(
+ model_dir, dtype=self.torch_dtype, device_map=device, trust_remote_code=True
+ )
+ full_model.eval()
+
+ self.audio_tower = full_model.audio_tower
+ self.multi_modal_projector = full_model.multi_modal_projector
+ self.get_audio_features = full_model.get_audio_features
+ self.embed_tokens = full_model.language_model.get_input_embeddings()
+ self._full_model_config = full_model.config
+
+ # Free LM weights from GPU (vLLM loads its own copy)
+ del full_model.language_model
+ torch.cuda.empty_cache()
+
+ self.processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+
+ # Prepare and load vLLM engine
+ vllm_dir = prepare_glmasr_vllm_dir(model_dir)
+ logger.info(f"Initializing vLLM LM from {vllm_dir}")
+ self.vllm_engine = LLM(
+ model=vllm_dir,
+ enable_prompt_embeds=True,
+ tensor_parallel_size=tensor_parallel_size,
+ gpu_memory_utilization=gpu_memory_utilization,
+ max_model_len=max_model_len,
+ dtype={"bf16": "bfloat16", "fp16": "float16", "fp32": "auto"}.get(dtype, dtype),
+ trust_remote_code=True,
+ )
+ self.tokenizer = self.vllm_engine.get_tokenizer()
+ logger.info("GLM-ASR vLLM engine ready")
+
+ @torch.no_grad()
+ def _encode_audio(self, audio_input):
+ """Encode a single audio input through audio_tower + projector.
+
+ Returns:
+ audio_embeds: (1, T, hidden_size) tensor
+ """
+ import librosa
+
+ if isinstance(audio_input, str):
+ audio, _ = librosa.load(audio_input, sr=16000)
+ elif isinstance(audio_input, np.ndarray):
+ audio = audio_input.astype(np.float32)
+ elif isinstance(audio_input, torch.Tensor):
+ audio = audio_input.cpu().numpy().astype(np.float32)
+ else:
+ raise ValueError(f"Unsupported audio type: {type(audio_input)}")
+
+ inputs = self.processor.feature_extractor(audio, sampling_rate=16000, return_tensors="pt")
+ input_features = inputs["input_features"].to(self.device, dtype=self.torch_dtype)
+ feat_len = input_features.shape[-1]
+ input_features_mask = torch.ones(1, feat_len, dtype=torch.long, device=self.device)
+
+ audio_outputs = self.get_audio_features(
+ input_features, input_features_mask, return_dict=True
+ )
+ audio_embeds = audio_outputs.pooler_output
+ return audio_embeds.unsqueeze(0)
+
+ def _build_prompt_embeds(self, audio_embeds, prompt="转录以下音频内容"):
+ """Build [prefix_text_emb | audio_emb | suffix_text_emb]."""
+ prefix_text = "<|user|>\n<|begin_of_audio|>"
+ suffix_text = f"<|end_of_audio|><|user|>\n{prompt}<|assistant|>\n"
+
+ prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)
+ suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)
+ prefix_emb = self.embed_tokens(torch.tensor(prefix_ids, dtype=torch.long, device=self.device))
+ suffix_emb = self.embed_tokens(torch.tensor(suffix_ids, dtype=torch.long, device=self.device))
+
+ audio_emb = audio_embeds[0] if audio_embeds.dim() == 3 else audio_embeds
+ return torch.cat([prefix_emb, audio_emb, suffix_emb], dim=0)
+
+ def generate(self, inputs, prompt="转录以下音频内容", max_new_tokens=500, **kwargs):
+ """Run batch ASR inference.
+
+ Args:
+ inputs: Audio file path(s), numpy arrays, or tensors.
+ prompt: Instruction prompt for ASR.
+ max_new_tokens: Maximum tokens to generate per sample.
+
+ Returns:
+ List of {"key": str, "text": str}
+ """
+ from vllm import SamplingParams
+ from vllm.inputs.data import EmbedsPrompt
+
+ if isinstance(inputs, (str, np.ndarray, torch.Tensor)):
+ inputs = [inputs]
+
+ sampling_params = SamplingParams(
+ max_tokens=max_new_tokens,
+ temperature=0,
+ skip_special_tokens=True,
+ )
+
+ t0 = time.perf_counter()
+ prompts = []
+ for audio_input in inputs:
+ audio_embeds = self._encode_audio(audio_input)
+ full_embeds = self._build_prompt_embeds(audio_embeds, prompt=prompt)
+ prompts.append(EmbedsPrompt(prompt_embeds=full_embeds.float()))
+ t1 = time.perf_counter()
+ logger.info(f"Audio encoding: {len(inputs)} samples in {t1-t0:.3f}s")
+
+ outputs = self.vllm_engine.generate(prompts, sampling_params, use_tqdm=False)
+ t2 = time.perf_counter()
+ logger.info(f"vLLM generation: {t2-t1:.3f}s")
+
+ results = []
+ for i, output in enumerate(outputs):
+ token_ids = list(output.outputs[0].token_ids)
+ text = self.tokenizer.decode(token_ids, skip_special_tokens=True)
+ text = re.sub(r'\s+', ' ', text).strip()
+ key = os.path.splitext(os.path.basename(inputs[i]))[0] if isinstance(inputs[i], str) else f"sample_{i}"
+ results.append({"key": key, "text": text})
+
+ return results
+
+ @classmethod
+ def from_pretrained(cls, model="zai-org/GLM-ASR-Nano-2512", hub="ms",
+ device="cuda:0", dtype="bf16", **kwargs):
+ """Load from hub or local path."""
+ if os.path.isdir(model):
+ model_dir = model
+ else:
+ if hub in ("ms", "modelscope"):
+ from modelscope.hub.snapshot_download import snapshot_download
+ model_dir = snapshot_download(model, revision=kwargs.pop("revision", "master"))
+ else:
+ from huggingface_hub import snapshot_download
+ model_dir = snapshot_download(model)
+ return cls(model_dir=model_dir, device=device, dtype=dtype, **kwargs)