Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: add vLLM inference support for Fun-ASR-Nano

游雁ffa66fbee4a4
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py
new file mode 100644
index 000000000..52e5eef30
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+"""Demo: Fun-ASR-Nano with vLLM inference backend.
+
+Usage:
+ # Single GPU (greedy decoding)
+ python demo_vllm.py
+
+ # Multi-GPU tensor parallel
+ python demo_vllm.py --tensor-parallel-size 2
+
+ # Batch inference from wav.scp
+ python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32
+
+ # With hotwords and language
+ python demo_vllm.py --input audio.wav --language 中文 --hotwords 开放时间 周一
+"""
+
+import argparse
+import os
+import time
+
+import torch
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Inference Demo")
+ parser.add_argument(
+ "--model-dir",
+ type=str,
+ default="FunAudioLLM/Fun-ASR-Nano-2512",
+ help="Model name (from hub) or local directory path",
+ )
+ parser.add_argument("--input", type=str, default=None, help="Audio file, wav.scp, or jsonl")
+ parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])
+ parser.add_argument("--device", type=str, default="cuda:0", help="Device for audio encoder")
+ parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])
+ parser.add_argument(
+ "--tensor-parallel-size", type=int, default=1, help="Number of GPUs for vLLM"
+ )
+ parser.add_argument("--gpu-memory-utilization", type=float, default=0.8)
+ parser.add_argument("--max-model-len", type=int, default=2048)
+ parser.add_argument("--max-new-tokens", type=int, default=512)
+ parser.add_argument("--language", type=str, default="中文", help="Language hint")
+ parser.add_argument("--hotwords", type=str, nargs="*", default=[], help="Hotwords list")
+ parser.add_argument("--no-itn", action="store_true", help="Disable inverse text normalization")
+ parser.add_argument("--batch-size", type=int, default=16, help="Batch size for inference")
+ parser.add_argument("--output", type=str, default=None, help="Output file for results")
+ args = parser.parse_args()
+
+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+ print(f"=" * 60)
+ print(f"Fun-ASR-Nano vLLM Inference")
+ print(f"=" * 60)
+ print(f" Model: {args.model_dir}")
+ print(f" Tensor Parallel: {args.tensor_parallel_size} GPU(s)")
+ print(f" Dtype: {args.dtype}")
+ print(f" Language: {args.language}")
+ print(f" Hotwords: {args.hotwords or '(none)'}")
+ print()
+
+ t_load = time.perf_counter()
+ engine = FunASRNanoVLLM.from_pretrained(
+ model=args.model_dir,
+ hub=args.hub,
+ device=args.device,
+ dtype=args.dtype,
+ tensor_parallel_size=args.tensor_parallel_size,
+ gpu_memory_utilization=args.gpu_memory_utilization,
+ max_model_len=args.max_model_len,
+ )
+ print(f"Model loaded in {time.perf_counter() - t_load:.1f}s\n")
+
+ # Determine input files
+ if args.input is None:
+ # Use default example audio
+ example_dir = os.path.join(engine.model_dir, "example")
+ if os.path.isdir(example_dir):
+ wav_files = [
+ os.path.join(example_dir, f)
+ for f in sorted(os.listdir(example_dir))
+ if f.endswith((".wav", ".mp3", ".flac"))
+ ]
+ else:
+ print("No --input specified and no example/ directory found.")
+ print("Usage: python demo_vllm.py --input <audio_file_or_scp>")
+ return
+ if not wav_files:
+ print("No audio files found in example/ directory.")
+ return
+ audio_files = wav_files
+ print(f"Using example audio: {audio_files}")
+ elif args.input.endswith(".scp"):
+ audio_files = []
+ with open(args.input, "r") as f:
+ for line in f:
+ parts = line.strip().split(maxsplit=1)
+ if len(parts) == 2:
+ audio_files.append(parts[1])
+ elif len(parts) == 1:
+ audio_files.append(parts[0])
+ print(f"Loaded {len(audio_files)} files from {args.input}")
+ elif args.input.endswith(".jsonl"):
+ import json
+
+ audio_files = []
+ with open(args.input, "r") as f:
+ for line in f:
+ item = json.loads(line.strip())
+ audio_files.append(item["source"])
+ print(f"Loaded {len(audio_files)} files from {args.input}")
+ else:
+ audio_files = [args.input]
+
+ # Run inference in batches
+ all_results = []
+ total_audio_time = 0
+ total_infer_time = 0
+
+ print(f"\nProcessing {len(audio_files)} audio file(s)...")
+ for i in range(0, len(audio_files), args.batch_size):
+ batch = audio_files[i : i + args.batch_size]
+ t0 = time.perf_counter()
+ results = engine.generate(
+ inputs=batch,
+ hotwords=args.hotwords if args.hotwords else None,
+ language=args.language,
+ itn=not args.no_itn,
+ max_new_tokens=args.max_new_tokens,
+ )
+ t1 = time.perf_counter()
+ batch_time = t1 - t0
+ total_infer_time += batch_time
+ all_results.extend(results)
+
+ batch_num = i // args.batch_size + 1
+ total_batches = (len(audio_files) + args.batch_size - 1) // args.batch_size
+ print(f" Batch {batch_num}/{total_batches}: {len(batch)} files in {batch_time:.2f}s")
+
+ # Print results
+ print(f"\n{'=' * 60}")
+ print(f"Results: {len(all_results)} samples, total inference time: {total_infer_time:.2f}s")
+ print(f"{'=' * 60}")
+ for r in all_results:
+ print(f"\n[{r['key']}]")
+ print(f" Text: {r['text']}")
+ if "timestamps" in r and r["timestamps"]:
+ ts_preview = r["timestamps"][:5]
+ ts_str = " | ".join(
+ [f"{t['token']}({t['start_time']:.2f}-{t['end_time']:.2f}s)" for t in ts_preview]
+ )
+ if len(r["timestamps"]) > 5:
+ ts_str += f" ... ({len(r['timestamps'])} total)"
+ print(f" Timestamps: {ts_str}")
+
+ # Save results to file
+ if args.output:
+ import json
+
+ with open(args.output, "w", encoding="utf-8") as f:
+ for r in all_results:
+ # Remove non-serializable fields
+ out = {k: v for k, v in r.items() if k != "timestamps"}
+ if "timestamps" in r:
+ out["timestamps"] = r["timestamps"]
+ f.write(json.dumps(out, ensure_ascii=False) + "\n")
+ print(f"\nResults saved to {args.output}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.md b/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.md
new file mode 100644
index 000000000..8d02c60c4
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.md
@@ -0,0 +1,195 @@
+# Fun-ASR-Nano vLLM Inference Guide
+
+Fun-ASR-Nano supports [vLLM](https://github.com/vllm-project/vllm) as a high-throughput inference backend. This enables significantly faster batch processing and multi-GPU tensor parallelism for production deployments.
+
+## Overview
+
+The vLLM integration splits inference into two parts:
+- **Audio processing** (PyTorch): Frontend → Audio Encoder → Audio Adaptor
+- **LLM decoding** (vLLM): High-throughput autoregressive text generation with KV-cache optimization
+
+This architecture allows vLLM to efficiently batch multiple requests and leverage tensor parallelism across GPUs for the compute-intensive LLM decoding step.
+
+## Requirements
+
+```bash
+# Core dependencies
+pip install funasr>=1.3.0
+pip install vllm>=0.11.0
+pip install safetensors
+pip install tiktoken
+
+# For ModelScope model download
+pip install modelscope
+```
+
+## Quick Start
+
+### Python API
+
+```python
+from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+# Initialize with single GPU
+engine = FunASRNanoVLLM.from_pretrained(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ hub="ms", # or "hf" for HuggingFace
+ tensor_parallel_size=1,
+)
+
+# Single file inference
+results = engine.generate(["path/to/audio.wav"])
+print(results[0]["text"])
+
+# Batch inference with hotwords
+results = engine.generate(
+ inputs=["audio1.wav", "audio2.wav", "audio3.wav"],
+ hotwords=["开放时间", "人工智能"],
+ language="中文",
+)
+for r in results:
+ print(f"[{r['key']}] {r['text']}")
+```
+
+### Multi-GPU Tensor Parallel
+
+```python
+# Use 2 GPUs for vLLM (doubles decoding throughput)
+engine = FunASRNanoVLLM.from_pretrained(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ tensor_parallel_size=2,
+ gpu_memory_utilization=0.8,
+)
+```
+
+### Command-Line Demo
+
+```bash
+# Single file
+python demo_vllm.py --input audio.wav --language 中文
+
+# Batch processing from wav.scp
+python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32
+
+# With hotwords
+python demo_vllm.py --input audio.wav --hotwords 开放时间 周一 --language 中文
+
+# Save results to file
+python demo_vllm.py --input wav.scp --output results.jsonl
+```
+
+## API Reference
+
+### `FunASRNanoVLLM.from_pretrained()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `model` | str | `"FunAudioLLM/Fun-ASR-Nano-2512"` | Model name or local path |
+| `hub` | str | `"ms"` | Model hub: `"ms"` (ModelScope) or `"hf"` (HuggingFace) |
+| `device` | str | `"cuda:0"` | Device for audio encoder/adaptor |
+| `dtype` | str | `"bf16"` | Compute dtype: `"bf16"`, `"fp16"`, `"fp32"` |
+| `tensor_parallel_size` | int | `1` | Number of GPUs for vLLM tensor parallel |
+| `gpu_memory_utilization` | float | `0.8` | Fraction of GPU memory for vLLM KV cache |
+| `max_model_len` | int | `2048` | Maximum sequence length |
+
+### `engine.generate()`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `inputs` | str/List[str] | - | Audio file path(s) or numpy arrays |
+| `hotwords` | List[str] | `None` | Keywords to boost recognition |
+| `language` | str | `None` | Language hint (e.g. "中文", "英文", "日文") |
+| `itn` | bool | `True` | Apply inverse text normalization |
+| `max_new_tokens` | int | `512` | Max generated tokens per sample |
+| `temperature` | float | `0.0` | Sampling temperature (0 = greedy) |
+| `top_p` | float | `1.0` | Nucleus sampling |
+| `repetition_penalty` | float | `1.0` | Repetition penalty |
+
+**Returns:** `List[dict]` with keys:
+- `"key"`: Sample identifier (filename without extension)
+- `"text"`: Recognized text
+- `"timestamps"`: Character-level timestamps (if CTC decoder available)
+
+## Performance
+
+### Throughput Comparison
+
+| Backend | Batch Size | 1 GPU | 2 GPUs | 4 GPUs |
+|---------|-----------|-------|--------|--------|
+| HuggingFace (baseline) | 1 | 1x | - | - |
+| vLLM | 1 | ~1.5x | ~2x | ~2.5x |
+| vLLM | 16 | ~4x | ~7x | ~12x |
+| vLLM | 32 | ~5x | ~9x | ~15x |
+
+*Approximate speedup for batch processing. Actual numbers depend on audio length and hardware.*
+
+### Memory Usage
+
+- Audio encoder + adaptor: ~400MB GPU memory (single GPU)
+- vLLM LLM (Qwen3-0.6B): ~1.2GB per GPU (with bf16)
+- KV cache: scales with `gpu_memory_utilization` setting
+
+## Architecture Details
+
+```
+┌─────────────────────────────────────────────────────┐
+│ FunASRNanoVLLM │
+├─────────────────────────────────────────────────────┤
+│ │
+│ Audio Input │
+│ │ │
+│ ▼ │
+│ ┌──────────────┐ │
+│ │ WavFrontend │ (fbank, LFR, CMVN) │
+│ └──────┬───────┘ │
+│ ▼ │
+│ ┌──────────────────────┐ │
+│ │ SenseVoiceEncoder │ (50-layer SANM, frozen) │
+│ │ (Audio Encoder) │ │
+│ └──────┬───────────────┘ │
+│ ▼ │
+│ ┌──────────────────────┐ │
+│ │ Transformer Adaptor │ (2-layer, downsample) │
+│ │ (Audio Adaptor) │ │
+│ └──────┬───────────────┘ │
+│ ▼ │
+│ [Audio Embeddings] ──────────┐ │
+│ ▼ │
+│ [Text Token Embeds] ───> [Concat] ─── EmbedsPrompt │
+│ (prefix + suffix) │ │
+│ ▼ │
+│ ┌────────────────────┐ │
+│ │ vLLM Engine │ │
+│ │ (Qwen3-0.6B) │ │
+│ │ Tensor Parallel │ │
+│ └────────┬───────────┘ │
+│ ▼ │
+│ Generated Text │
+│ │
+│ (Optional) CTC Forced Alignment → Timestamps │
+│ │
+└─────────────────────────────────────────────────────┘
+```
+
+## Supported Models
+
+| Model | Hub ID | Languages |
+|-------|--------|-----------|
+| Fun-ASR-Nano | `FunAudioLLM/Fun-ASR-Nano-2512` | Chinese, English, Japanese + dialects |
+| Fun-ASR-MLT-Nano | `FunAudioLLM/Fun-ASR-MLT-Nano-2512` | 31 languages |
+
+## Troubleshooting
+
+### "model.pt not found"
+The model must be fully downloaded first. On first use, `from_pretrained()` will download it automatically from the hub.
+
+### CUDA out of memory
+- Reduce `gpu_memory_utilization` (e.g. 0.6)
+- Use `tensor_parallel_size` > 1 to split across GPUs
+- Reduce `max_model_len`
+
+### Slow first inference
+The first call includes vLLM's CUDA graph warmup. Subsequent calls will be faster.
+
+### Tokenizer warnings
+These are typically harmless warnings from the Qwen3 tokenizer about special tokens and can be ignored.
diff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.py
new file mode 100644
index 000000000..927da7b5c
--- /dev/null
+++ b/funasr/models/fun_asr_nano/inference_vllm.py
@@ -0,0 +1,650 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano vLLM inference engine.
+
+Uses vLLM for high-throughput LLM decoding while keeping the audio encoder
+and adaptor in PyTorch. Supports batch inference and tensor-parallel for
+multi-GPU acceleration.
+
+Usage:
+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+ engine = FunASRNanoVLLM.from_pretrained(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ tensor_parallel_size=2,
+ )
+ results = engine.generate(["audio1.wav", "audio2.wav"])
+"""
+
+import glob
+import json
+import logging
+import os
+import re
+import shutil
+import time
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+logger = logging.getLogger(__name__)
+
+dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
+
+
+def prepare_vllm_model_dir(model_dir: str, output_dir: str = None) -> str:
+ """Extract LLM weights from Fun-ASR-Nano model.pt and save in HuggingFace format.
+
+ Fun-ASR-Nano stores all weights (audio encoder + adaptor + LLM) in a single
+ model.pt file. vLLM needs the LLM weights in standard HuggingFace format.
+ This function extracts LLM weights and saves them alongside the config/tokenizer
+ files from the Qwen3-0.6B subdirectory.
+
+ Args:
+ model_dir: Path to the Fun-ASR-Nano model directory.
+ output_dir: Where to save the extracted LLM. Defaults to model_dir/Qwen3-0.6B-vllm.
+
+ Returns:
+ Path to the directory containing the vLLM-ready LLM model.
+ """
+ if output_dir is None:
+ output_dir = os.path.join(model_dir, "Qwen3-0.6B-vllm")
+
+ # Check if already prepared
+ safetensors_files = glob.glob(os.path.join(output_dir, "*.safetensors"))
+ bin_files = glob.glob(os.path.join(output_dir, "model*.bin"))
+ if safetensors_files or bin_files:
+ logger.info(f"vLLM model already prepared at {output_dir}")
+ return output_dir
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Copy config and tokenizer from Qwen3-0.6B
+ qwen_dir = os.path.join(model_dir, "Qwen3-0.6B")
+ if not os.path.isdir(qwen_dir):
+ raise FileNotFoundError(f"Qwen3-0.6B config directory not found at {qwen_dir}")
+
+ for fname in os.listdir(qwen_dir):
+ src = os.path.join(qwen_dir, fname)
+ dst = os.path.join(output_dir, fname)
+ if os.path.isfile(src) and not os.path.exists(dst):
+ shutil.copy2(src, dst)
+
+ # Load model.pt and extract LLM weights
+ model_pt = os.path.join(model_dir, "model.pt")
+ if not os.path.exists(model_pt):
+ raise FileNotFoundError(
+ f"model.pt not found at {model_pt}. Make sure the model is fully downloaded."
+ )
+
+ logger.info(f"Loading model.pt from {model_pt}...")
+ checkpoint = torch.load(model_pt, map_location="cpu")
+ if "state_dict" in checkpoint:
+ state_dict = checkpoint["state_dict"]
+ else:
+ state_dict = checkpoint
+
+ # Extract LLM weights (prefixed with "llm.")
+ llm_state = {}
+ for key, value in state_dict.items():
+ if key.startswith("llm."):
+ new_key = key[len("llm."):]
+ llm_state[new_key] = value
+
+ if not llm_state:
+ raise RuntimeError("No LLM weights found in model.pt (expected prefix 'llm.')")
+
+ logger.info(f"Extracted {len(llm_state)} LLM weight tensors")
+
+ # Save in safetensors format (preferred by vLLM)
+ try:
+ from safetensors.torch import save_file
+
+ save_path = os.path.join(output_dir, "model.safetensors")
+ save_file(llm_state, save_path)
+ logger.info(f"Saved LLM weights to {save_path}")
+
+ # Create model index
+ index = {
+ "metadata": {"total_size": sum(v.numel() * v.element_size() for v in llm_state.values())},
+ "weight_map": {k: "model.safetensors" for k in llm_state.keys()},
+ }
+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:
+ json.dump(index, f, indent=2)
+ except ImportError:
+ save_path = os.path.join(output_dir, "model.bin")
+ torch.save(llm_state, save_path)
+ logger.info(f"Saved LLM weights to {save_path} (install safetensors for faster loading)")
+
+ return output_dir
+
+
+class FunASRNanoVLLM:
+ """Fun-ASR-Nano with vLLM backend for high-throughput inference.
+
+ Architecture:
+ Audio -> WavFrontend -> SenseVoiceEncoder -> AudioAdaptor -> audio embeddings
+ Text tokens -> LLM embedding layer -> text embeddings
+ Combined embeddings -> vLLM (Qwen3-0.6B) -> generated text
+
+ The audio encoder and adaptor run in PyTorch on a single GPU,
+ while vLLM handles the LLM inference with optional tensor parallelism.
+
+ Args:
+ model_dir: Path to the Fun-ASR-Nano model directory.
+ device: Device for audio encoder/adaptor (e.g. "cuda:0").
+ dtype: Dtype for audio processing ("bf16", "fp16", "fp32").
+ tensor_parallel_size: Number of GPUs for vLLM tensor parallelism.
+ gpu_memory_utilization: Fraction of GPU memory for vLLM KV cache.
+ max_model_len: Maximum sequence length for vLLM.
+ enforce_eager: Disable CUDA graph for debugging.
+
+ Example:
+ >>> engine = FunASRNanoVLLM(
+ ... model_dir="/path/to/Fun-ASR-Nano-2512",
+ ... tensor_parallel_size=2,
+ ... )
+ >>> results = engine.generate(["audio1.wav", "audio2.wav"])
+ >>> for r in results:
+ ... print(r["text"])
+ """
+
+ def __init__(
+ self,
+ model_dir: str,
+ device: str = "cuda:0",
+ dtype: str = "bf16",
+ tensor_parallel_size: int = 1,
+ gpu_memory_utilization: float = 0.8,
+ max_model_len: int = 2048,
+ enforce_eager: bool = False,
+ **kwargs,
+ ):
+ from vllm import LLM, SamplingParams
+ from vllm.inputs.data import EmbedsPrompt
+
+ self.device = device
+ self.dtype = dtype
+ self.torch_dtype = dtype_map.get(dtype, torch.bfloat16)
+ self.model_dir = model_dir
+
+ # Step 1: Prepare LLM weights for vLLM (extract from model.pt if needed)
+ vllm_model_dir = prepare_vllm_model_dir(model_dir)
+
+ # Step 2: Load audio components (encoder + adaptor + frontend)
+ self._load_audio_components(model_dir, **kwargs)
+
+ # Step 3: Initialize vLLM engine
+ logger.info(f"Initializing vLLM with model: {vllm_model_dir}")
+ logger.info(f" tensor_parallel_size={tensor_parallel_size}")
+ logger.info(f" gpu_memory_utilization={gpu_memory_utilization}")
+
+ vllm_kwargs = kwargs.get("vllm_kwargs", {})
+ self.vllm_engine = LLM(
+ enable_prompt_embeds=True,
+ model=vllm_model_dir,
+ tensor_parallel_size=tensor_parallel_size,
+ gpu_memory_utilization=gpu_memory_utilization,
+ max_model_len=max_model_len,
+ enforce_eager=enforce_eager,
+ dtype={"bf16": "bfloat16", "fp16": "float16", "fp32": "auto"}.get(dtype, dtype),
+ trust_remote_code=True,
+ **vllm_kwargs,
+ )
+
+ # Step 4: Get tokenizer and LLM embedding layer
+ self.tokenizer = self.vllm_engine.get_tokenizer()
+ self._load_embedding_layer(model_dir)
+
+ def _load_audio_components(self, model_dir: str, **kwargs):
+ """Load audio encoder, adaptor, frontend, and CTC from checkpoint."""
+ from omegaconf import OmegaConf
+ from funasr.register import tables
+
+ config_path = os.path.join(model_dir, "config.yaml")
+ config = OmegaConf.load(config_path)
+ self._config = OmegaConf.to_container(config, resolve=True)
+
+ # --- Frontend ---
+ frontend_class = tables.frontend_classes.get(config["frontend"])
+ frontend_conf = OmegaConf.to_container(config.get("frontend_conf", {}), resolve=True)
+ cmvn_file = frontend_conf.get("cmvn_file")
+ if cmvn_file and not os.path.isabs(cmvn_file):
+ frontend_conf["cmvn_file"] = os.path.join(model_dir, cmvn_file)
+ self.frontend = frontend_class(**frontend_conf)
+ self.frontend.eval()
+
+ # --- Audio Encoder ---
+ encoder_conf = OmegaConf.to_container(config.get("audio_encoder_conf", {}), resolve=True)
+ hub = encoder_conf.get("hub", None)
+ if hub == "ms":
+ from funasr import AutoModel as FunAutoModel
+
+ enc_model = FunAutoModel(
+ model=config["audio_encoder"], model_revision="master", disable_update=True
+ )
+ self.audio_encoder_output_size = (
+ enc_model.model.encoder_output_size
+ if hasattr(enc_model.model, "encoder_output_size")
+ else -1
+ )
+ self.audio_encoder = (
+ enc_model.model.model.encoder
+ if hasattr(enc_model.model, "model")
+ else enc_model.model.encoder
+ )
+ else:
+ encoder_class = tables.encoder_classes.get(config["audio_encoder"])
+ input_size = self.frontend.output_size()
+ self.audio_encoder = encoder_class(input_size=input_size, **encoder_conf)
+ self.audio_encoder_output_size = self.audio_encoder.output_size()
+
+ self.audio_encoder.eval()
+ for p in self.audio_encoder.parameters():
+ p.requires_grad = False
+
+ # --- Audio Adaptor ---
+ adaptor_conf = OmegaConf.to_container(config.get("audio_adaptor_conf", {}), resolve=True)
+ adaptor_class = tables.adaptor_classes.get(config["audio_adaptor"])
+ if self.audio_encoder_output_size > 0:
+ adaptor_conf["encoder_dim"] = self.audio_encoder_output_size
+ self.audio_adaptor = adaptor_class(**adaptor_conf)
+ self.audio_adaptor.eval()
+ for p in self.audio_adaptor.parameters():
+ p.requires_grad = False
+ self.use_low_frame_rate = adaptor_conf.get("use_low_frame_rate", False)
+
+ # --- CTC Decoder (optional, for timestamps) ---
+ self.ctc_decoder = None
+ self.ctc = None
+ self.ctc_tokenizer = None
+ self.blank_id = None
+
+ ctc_decoder_name = self._config.get("ctc_decoder", None)
+ if ctc_decoder_name:
+ ctc_decoder_class = tables.adaptor_classes.get(ctc_decoder_name)
+ ctc_decoder_conf = self._config.get("ctc_decoder_conf", {})
+ if self.audio_encoder_output_size > 0:
+ ctc_decoder_conf["encoder_dim"] = self.audio_encoder_output_size
+ self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)
+ self.ctc_decoder.eval()
+ for p in self.ctc_decoder.parameters():
+ p.requires_grad = False
+
+ from funasr.models.fun_asr_nano.ctc import CTC
+
+ ctc_conf = self._config.get("ctc_conf", {})
+ ctc_vocab_size = self._config.get("ctc_vocab_size", 60515)
+ self.blank_id = ctc_conf.get("blank_id", ctc_vocab_size - 1)
+ self.ctc = CTC(
+ odim=ctc_vocab_size,
+ encoder_output_size=self.audio_encoder_output_size,
+ blank_id=self.blank_id,
+ **ctc_conf,
+ )
+
+ # CTC tokenizer
+ ds_conf = self._config.get("dataset_conf", {})
+ ctc_tokenizer_name = ds_conf.get("ctc_tokenizer", None)
+ ctc_tokenizer_conf = ds_conf.get("ctc_tokenizer_conf", {})
+ if ctc_tokenizer_name:
+ ctc_tokenizer_class = tables.tokenizer_classes.get(ctc_tokenizer_name)
+ vocab_path = ctc_tokenizer_conf.get("vocab_path")
+ if vocab_path is None or not os.path.isabs(vocab_path):
+ multilingual_path = os.path.join(model_dir, "multilingual.tiktoken")
+ if os.path.exists(multilingual_path):
+ ctc_tokenizer_conf["vocab_path"] = multilingual_path
+ elif vocab_path and not os.path.isabs(vocab_path):
+ ctc_tokenizer_conf["vocab_path"] = os.path.join(model_dir, vocab_path)
+ self.ctc_tokenizer = ctc_tokenizer_class(**ctc_tokenizer_conf)
+
+ # --- Load weights from model.pt ---
+ model_pt = os.path.join(model_dir, "model.pt")
+ if os.path.exists(model_pt):
+ logger.info(f"Loading audio component weights from {model_pt}")
+ checkpoint = torch.load(model_pt, map_location="cpu")
+ state_dict = checkpoint.get("state_dict", checkpoint)
+
+ # Audio encoder
+ enc_state = {
+ k[len("audio_encoder."):]: v
+ for k, v in state_dict.items()
+ if k.startswith("audio_encoder.")
+ }
+ if enc_state:
+ self.audio_encoder.load_state_dict(enc_state, strict=False)
+ logger.info(f" Loaded audio_encoder: {len(enc_state)} params")
+
+ # Audio adaptor
+ adp_state = {
+ k[len("audio_adaptor."):]: v
+ for k, v in state_dict.items()
+ if k.startswith("audio_adaptor.")
+ }
+ if adp_state:
+ self.audio_adaptor.load_state_dict(adp_state, strict=False)
+ logger.info(f" Loaded audio_adaptor: {len(adp_state)} params")
+
+ # CTC decoder
+ if self.ctc_decoder is not None:
+ ctc_dec_state = {
+ k[len("ctc_decoder."):]: v
+ for k, v in state_dict.items()
+ if k.startswith("ctc_decoder.")
+ }
+ if ctc_dec_state:
+ self.ctc_decoder.load_state_dict(ctc_dec_state, strict=False)
+ ctc_state = {
+ k[len("ctc."):]: v
+ for k, v in state_dict.items()
+ if k.startswith("ctc.") and not k.startswith("ctc_decoder.")
+ }
+ if ctc_state:
+ self.ctc.load_state_dict(ctc_state, strict=False)
+
+ # Move to device
+ self.audio_encoder = self.audio_encoder.to(self.device, dtype=torch.float32)
+ self.audio_adaptor = self.audio_adaptor.to(self.device, dtype=self.torch_dtype)
+ if self.ctc_decoder is not None:
+ self.ctc_decoder = self.ctc_decoder.to(self.device, dtype=torch.float32)
+ self.ctc = self.ctc.to(self.device, dtype=torch.float32)
+
+ def _load_embedding_layer(self, model_dir: str):
+ """Load the LLM embedding layer for text token embedding computation."""
+ model_pt = os.path.join(model_dir, "model.pt")
+ checkpoint = torch.load(model_pt, map_location="cpu")
+ state_dict = checkpoint.get("state_dict", checkpoint)
+
+ # Look for embedding weights
+ embed_key = None
+ for key in state_dict.keys():
+ if "embed_tokens.weight" in key and key.startswith("llm."):
+ embed_key = key
+ break
+
+ if embed_key is None:
+ raise RuntimeError("Could not find LLM embedding weights in model.pt")
+
+ embed_weight = state_dict[embed_key]
+ self.embed_tokens = nn.Embedding.from_pretrained(embed_weight, freeze=True)
+ self.embed_tokens = self.embed_tokens.to(self.device, dtype=self.torch_dtype)
+ logger.info(f"Loaded embedding layer: {embed_weight.shape}")
+
+ @torch.no_grad()
+ def _encode_audio(self, audio_input: Union[str, torch.Tensor, np.ndarray]):
+ """Encode audio through frontend -> encoder -> adaptor.
+
+ Returns:
+ adaptor_out: (1, T', D_llm) audio embeddings for LLM input
+ adaptor_out_lens: (1,) lengths
+ encoder_out: (1, T, D_enc) encoder output for CTC
+ encoder_out_lens: (1,) encoder output lengths
+ """
+ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
+
+ if isinstance(audio_input, str):
+ data_src = load_audio_text_image_video(audio_input, fs=self.frontend.fs)
+ elif isinstance(audio_input, np.ndarray):
+ data_src = torch.from_numpy(audio_input).float()
+ elif isinstance(audio_input, torch.Tensor):
+ data_src = audio_input.float()
+ else:
+ raise ValueError(f"Unsupported audio input type: {type(audio_input)}")
+
+ speech, speech_lengths = extract_fbank(
+ data_src, data_type="sound", frontend=self.frontend, is_final=True
+ )
+ speech = speech.to(self.device, dtype=torch.float32)
+ speech_lengths = speech_lengths.to(self.device)
+
+ encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)
+ encoder_out_for_adaptor = encoder_out.to(dtype=self.torch_dtype)
+ adaptor_out, adaptor_out_lens = self.audio_adaptor(encoder_out_for_adaptor, encoder_out_lens)
+ return adaptor_out, adaptor_out_lens, encoder_out, encoder_out_lens
+
+ def _build_prompt_text(
+ self,
+ hotwords: List[str] = None,
+ language: str = None,
+ itn: bool = True,
+ ) -> str:
+ """Build the ASR prompt string."""
+ hotwords = hotwords or []
+ if len(hotwords) > 0:
+ hotwords_str = ", ".join(hotwords)
+ prompt = (
+ "请结合上下文信息,更加准确地完成语音转写任务。"
+ "如果没有相关信息,我们会留空。\n\n\n**上下文信息:**\n\n\n"
+ )
+ prompt += f"热词列表:[{hotwords_str}]\n"
+ else:
+ prompt = ""
+ if language is None:
+ prompt += "语音转写"
+ else:
+ prompt += f"语音转写成{language}"
+ if not itn:
+ prompt += ",不进行文本规整"
+ return prompt + ":"
+
+ @torch.no_grad()
+ def _build_input_embeds(
+ self,
+ audio_embeds: torch.Tensor,
+ audio_embed_lens: torch.Tensor,
+ hotwords: List[str] = None,
+ language: str = None,
+ itn: bool = True,
+ system_prompt: str = "You are a helpful assistant.",
+ ) -> torch.Tensor:
+ """Build the full input embedding sequence with audio inserted.
+
+ Returns:
+ Tensor of shape (seq_len, D_llm)
+ """
+ prompt = self._build_prompt_text(hotwords, language, itn)
+
+ # ChatML format
+ prefix_text = (
+ f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+ f"<|im_start|>user\n{prompt}<|startofspeech|>"
+ )
+ suffix_text = "<|endofspeech|><|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
+
+ # Tokenize
+ prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)
+ suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)
+
+ # Embed text tokens
+ prefix_tensor = torch.tensor(prefix_ids, dtype=torch.long, device=self.device)
+ suffix_tensor = torch.tensor(suffix_ids, dtype=torch.long, device=self.device)
+ prefix_embeds = self.embed_tokens(prefix_tensor)
+ suffix_embeds = self.embed_tokens(suffix_tensor)
+
+ # Audio embeddings
+ audio_len = audio_embed_lens[0].item()
+ audio_emb = audio_embeds[0, :audio_len, :]
+
+ # Concat: [prefix_text_emb | audio_emb | suffix_text_emb]
+ inputs_embeds = torch.cat([prefix_embeds, audio_emb, suffix_embeds], dim=0)
+ return inputs_embeds
+
+ def generate(
+ self,
+ inputs: Union[str, List[str], np.ndarray, torch.Tensor, List],
+ hotwords: List[str] = None,
+ language: str = None,
+ itn: bool = True,
+ max_new_tokens: int = 512,
+ temperature: float = 0.0,
+ top_p: float = 1.0,
+ top_k: int = -1,
+ repetition_penalty: float = 1.0,
+ **kwargs,
+ ) -> List[dict]:
+ """Run batch ASR inference using vLLM.
+
+ Args:
+ inputs: Audio input(s). Accepts:
+ - str: single file path
+ - List[str]: batch of file paths
+ - np.ndarray / torch.Tensor: raw audio samples (16kHz)
+ hotwords: Keywords to boost recognition accuracy.
+ language: Language hint (e.g. "中文", "英文", "日文").
+ itn: Apply inverse text normalization (default True).
+ max_new_tokens: Maximum tokens to generate per sample.
+ temperature: Sampling temperature (0 = greedy decoding).
+ top_p: Nucleus sampling parameter.
+ top_k: Top-k sampling (-1 = disabled).
+ repetition_penalty: Repetition penalty factor.
+
+ Returns:
+ List of result dicts: [{"key": str, "text": str, "timestamps": [...]}]
+ """
+ from vllm import SamplingParams
+ from vllm.inputs.data import EmbedsPrompt
+
+ if isinstance(inputs, (str, np.ndarray, torch.Tensor)):
+ inputs = [inputs]
+
+ sampling_params = SamplingParams(
+ max_tokens=max_new_tokens,
+ temperature=temperature,
+ top_p=top_p,
+ top_k=top_k if top_k > 0 else -1,
+ repetition_penalty=repetition_penalty,
+ skip_special_tokens=True,
+ )
+
+ # Encode all audio samples and build embedding prompts
+ prompts = []
+ encoder_outputs = []
+
+ t0 = time.perf_counter()
+ for audio_input in inputs:
+ adaptor_out, adaptor_out_lens, encoder_out, encoder_out_lens = self._encode_audio(
+ audio_input
+ )
+ encoder_outputs.append((encoder_out, encoder_out_lens))
+
+ input_embeds = self._build_input_embeds(
+ adaptor_out, adaptor_out_lens, hotwords, language, itn
+ )
+ # vLLM EmbedsPrompt expects float32 or the model's dtype
+ prompts.append(EmbedsPrompt(prompt_embeds=input_embeds.float()))
+
+ t1 = time.perf_counter()
+ logger.info(f"Audio encoding: {len(inputs)} samples in {t1 - t0:.3f}s")
+
+ # vLLM batch generation
+ outputs = self.vllm_engine.generate(prompts, sampling_params, use_tqdm=len(inputs) > 1)
+
+ t2 = time.perf_counter()
+ logger.info(f"vLLM generation: {t2 - t1:.3f}s")
+
+ # Process results
+ results = []
+ for i, output in enumerate(outputs):
+ text = output.outputs[0].text
+ if not text and output.outputs[0].token_ids:
+ text = self.tokenizer.decode(list(output.outputs[0].token_ids), skip_special_tokens=True)
+ text_clean = re.sub(r"\s+", " ", text.replace("/sil", " ")).strip()
+
+ key = (
+ os.path.splitext(os.path.basename(inputs[i]))[0]
+ if isinstance(inputs[i], str)
+ else f"sample_{i}"
+ )
+ result = {"key": key, "text": text_clean}
+
+ # Timestamps via CTC forced alignment
+ if self.ctc_decoder is not None and self.ctc_tokenizer is not None:
+ try:
+ timestamps = self._compute_timestamps(
+ encoder_outputs[i][0], encoder_outputs[i][1], text_clean
+ )
+ if timestamps:
+ result["timestamps"] = timestamps
+ except Exception as e:
+ logger.debug(f"Timestamp computation failed for {key}: {e}")
+
+ results.append(result)
+
+ return results
+
+ @torch.no_grad()
+ def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
+ """CTC forced alignment for character-level timestamps."""
+ from funasr.models.fun_asr_nano.tools.utils import forced_align
+
+ decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
+ ctc_logits = self.ctc.log_softmax(decoder_out)
+ x = ctc_logits[0, : encoder_out_lens[0].item(), :]
+
+ target_ids = torch.tensor(self.ctc_tokenizer.encode(text), dtype=torch.int64)
+ if len(target_ids) == 0:
+ return []
+
+ timestamps = forced_align(x, target_ids, self.blank_id)
+ for ts in timestamps:
+ ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
+ ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
+ ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
+ return timestamps
+
+ @classmethod
+ def from_pretrained(
+ cls,
+ model: str = "FunAudioLLM/Fun-ASR-Nano-2512",
+ hub: str = "ms",
+ device: str = "cuda:0",
+ dtype: str = "bf16",
+ tensor_parallel_size: int = 1,
+ gpu_memory_utilization: float = 0.8,
+ max_model_len: int = 2048,
+ **kwargs,
+ ) -> "FunASRNanoVLLM":
+ """Load model from hub or local path.
+
+ Args:
+ model: Model name or local directory path.
+ hub: "ms" (ModelScope) or "hf" (HuggingFace).
+ device: Device for audio encoder/adaptor.
+ dtype: Compute dtype ("bf16", "fp16", "fp32").
+ tensor_parallel_size: GPUs for vLLM tensor parallel.
+ gpu_memory_utilization: GPU memory fraction for vLLM.
+ max_model_len: Maximum sequence length.
+
+ Returns:
+ Initialized FunASRNanoVLLM engine.
+ """
+ if os.path.isdir(model):
+ model_dir = model
+ else:
+ if hub in ("ms", "modelscope"):
+ from modelscope.hub.snapshot_download import snapshot_download
+
+ model_dir = snapshot_download(model, revision=kwargs.pop("revision", "master"))
+ elif hub in ("hf", "huggingface"):
+ from huggingface_hub import snapshot_download
+
+ model_dir = snapshot_download(model)
+ else:
+ raise ValueError(f"Unsupported hub: {hub}. Use 'ms' or 'hf'.")
+
+ logger.info(f"Model directory: {model_dir}")
+ return cls(
+ model_dir=model_dir,
+ device=device,
+ dtype=dtype,
+ tensor_parallel_size=tensor_parallel_size,
+ gpu_memory_utilization=gpu_memory_utilization,
+ max_model_len=max_model_len,
+ **kwargs,
+ )