Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: add vLLM inference support for Fun-ASR-Nano
游雁ffa66fbee4a4
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.pynew file mode 100644index 000000000..52e5eef30--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py@@ -0,0 +1,171 @@+#!/usr/bin/env python3+"""Demo: Fun-ASR-Nano with vLLM inference backend.++Usage:+ # Single GPU (greedy decoding)+ python demo_vllm.py++ # Multi-GPU tensor parallel+ python demo_vllm.py --tensor-parallel-size 2++ # Batch inference from wav.scp+ python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32++ # With hotwords and language+ python demo_vllm.py --input audio.wav --language 中文 --hotwords 开放时间 周一+"""++import argparse+import os+import time++import torch+++def main():+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Inference Demo")+ parser.add_argument(+ "--model-dir",+ type=str,+ default="FunAudioLLM/Fun-ASR-Nano-2512",+ help="Model name (from hub) or local directory path",+ )+ parser.add_argument("--input", type=str, default=None, help="Audio file, wav.scp, or jsonl")+ parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])+ parser.add_argument("--device", type=str, default="cuda:0", help="Device for audio encoder")+ parser.add_argument("--dtype", type=str, default="bf16", choices=["bf16", "fp16", "fp32"])+ parser.add_argument(+ "--tensor-parallel-size", type=int, default=1, help="Number of GPUs for vLLM"+ )+ parser.add_argument("--gpu-memory-utilization", type=float, default=0.8)+ parser.add_argument("--max-model-len", type=int, default=2048)+ parser.add_argument("--max-new-tokens", type=int, default=512)+ parser.add_argument("--language", type=str, default="中文", help="Language hint")+ parser.add_argument("--hotwords", type=str, nargs="*", default=[], help="Hotwords list")+ parser.add_argument("--no-itn", action="store_true", help="Disable inverse text normalization")+ parser.add_argument("--batch-size", type=int, default=16, help="Batch size for inference")+ parser.add_argument("--output", type=str, default=None, help="Output file for results")+ args = parser.parse_args()++ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM++ print(f"=" * 60)+ print(f"Fun-ASR-Nano vLLM Inference")+ print(f"=" * 60)+ print(f" Model: {args.model_dir}")+ print(f" Tensor Parallel: {args.tensor_parallel_size} GPU(s)")+ print(f" Dtype: {args.dtype}")+ print(f" Language: {args.language}")+ print(f" Hotwords: {args.hotwords or '(none)'}")+ print()++ t_load = time.perf_counter()+ engine = FunASRNanoVLLM.from_pretrained(+ model=args.model_dir,+ hub=args.hub,+ device=args.device,+ dtype=args.dtype,+ tensor_parallel_size=args.tensor_parallel_size,+ gpu_memory_utilization=args.gpu_memory_utilization,+ max_model_len=args.max_model_len,+ )+ print(f"Model loaded in {time.perf_counter() - t_load:.1f}s\n")++ # Determine input files+ if args.input is None:+ # Use default example audio+ example_dir = os.path.join(engine.model_dir, "example")+ if os.path.isdir(example_dir):+ wav_files = [+ os.path.join(example_dir, f)+ for f in sorted(os.listdir(example_dir))+ if f.endswith((".wav", ".mp3", ".flac"))+ ]+ else:+ print("No --input specified and no example/ directory found.")+ print("Usage: python demo_vllm.py --input <audio_file_or_scp>")+ return+ if not wav_files:+ print("No audio files found in example/ directory.")+ return+ audio_files = wav_files+ print(f"Using example audio: {audio_files}")+ elif args.input.endswith(".scp"):+ audio_files = []+ with open(args.input, "r") as f:+ for line in f:+ parts = line.strip().split(maxsplit=1)+ if len(parts) == 2:+ audio_files.append(parts[1])+ elif len(parts) == 1:+ audio_files.append(parts[0])+ print(f"Loaded {len(audio_files)} files from {args.input}")+ elif args.input.endswith(".jsonl"):+ import json++ audio_files = []+ with open(args.input, "r") as f:+ for line in f:+ item = json.loads(line.strip())+ audio_files.append(item["source"])+ print(f"Loaded {len(audio_files)} files from {args.input}")+ else:+ audio_files = [args.input]++ # Run inference in batches+ all_results = []+ total_audio_time = 0+ total_infer_time = 0++ print(f"\nProcessing {len(audio_files)} audio file(s)...")+ for i in range(0, len(audio_files), args.batch_size):+ batch = audio_files[i : i + args.batch_size]+ t0 = time.perf_counter()+ results = engine.generate(+ inputs=batch,+ hotwords=args.hotwords if args.hotwords else None,+ language=args.language,+ itn=not args.no_itn,+ max_new_tokens=args.max_new_tokens,+ )+ t1 = time.perf_counter()+ batch_time = t1 - t0+ total_infer_time += batch_time+ all_results.extend(results)++ batch_num = i // args.batch_size + 1+ total_batches = (len(audio_files) + args.batch_size - 1) // args.batch_size+ print(f" Batch {batch_num}/{total_batches}: {len(batch)} files in {batch_time:.2f}s")++ # Print results+ print(f"\n{'=' * 60}")+ print(f"Results: {len(all_results)} samples, total inference time: {total_infer_time:.2f}s")+ print(f"{'=' * 60}")+ for r in all_results:+ print(f"\n[{r['key']}]")+ print(f" Text: {r['text']}")+ if "timestamps" in r and r["timestamps"]:+ ts_preview = r["timestamps"][:5]+ ts_str = " | ".join(+ [f"{t['token']}({t['start_time']:.2f}-{t['end_time']:.2f}s)" for t in ts_preview]+ )+ if len(r["timestamps"]) > 5:+ ts_str += f" ... ({len(r['timestamps'])} total)"+ print(f" Timestamps: {ts_str}")++ # Save results to file+ if args.output:+ import json++ with open(args.output, "w", encoding="utf-8") as f:+ for r in all_results:+ # Remove non-serializable fields+ out = {k: v for k, v in r.items() if k != "timestamps"}+ if "timestamps" in r:+ out["timestamps"] = r["timestamps"]+ f.write(json.dumps(out, ensure_ascii=False) + "\n")+ print(f"\nResults saved to {args.output}")+++if __name__ == "__main__":+ main()diff --git a/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.md b/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.mdnew file mode 100644index 000000000..8d02c60c4--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/docs/vllm_inference.md@@ -0,0 +1,195 @@+# Fun-ASR-Nano vLLM Inference Guide++Fun-ASR-Nano supports [vLLM](https://github.com/vllm-project/vllm) as a high-throughput inference backend. This enables significantly faster batch processing and multi-GPU tensor parallelism for production deployments.++## Overview++The vLLM integration splits inference into two parts:+- **Audio processing** (PyTorch): Frontend → Audio Encoder → Audio Adaptor+- **LLM decoding** (vLLM): High-throughput autoregressive text generation with KV-cache optimization++This architecture allows vLLM to efficiently batch multiple requests and leverage tensor parallelism across GPUs for the compute-intensive LLM decoding step.++## Requirements++```bash+# Core dependencies+pip install funasr>=1.3.0+pip install vllm>=0.11.0+pip install safetensors+pip install tiktoken++# For ModelScope model download+pip install modelscope+```++## Quick Start++### Python API++```python+from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM++# Initialize with single GPU+engine = FunASRNanoVLLM.from_pretrained(+ model="FunAudioLLM/Fun-ASR-Nano-2512",+ hub="ms", # or "hf" for HuggingFace+ tensor_parallel_size=1,+)++# Single file inference+results = engine.generate(["path/to/audio.wav"])+print(results[0]["text"])++# Batch inference with hotwords+results = engine.generate(+ inputs=["audio1.wav", "audio2.wav", "audio3.wav"],+ hotwords=["开放时间", "人工智能"],+ language="中文",+)+for r in results:+ print(f"[{r['key']}] {r['text']}")+```++### Multi-GPU Tensor Parallel++```python+# Use 2 GPUs for vLLM (doubles decoding throughput)+engine = FunASRNanoVLLM.from_pretrained(+ model="FunAudioLLM/Fun-ASR-Nano-2512",+ tensor_parallel_size=2,+ gpu_memory_utilization=0.8,+)+```++### Command-Line Demo++```bash+# Single file+python demo_vllm.py --input audio.wav --language 中文++# Batch processing from wav.scp+python demo_vllm.py --input wav.scp --tensor-parallel-size 4 --batch-size 32++# With hotwords+python demo_vllm.py --input audio.wav --hotwords 开放时间 周一 --language 中文++# Save results to file+python demo_vllm.py --input wav.scp --output results.jsonl+```++## API Reference++### `FunASRNanoVLLM.from_pretrained()`++| Parameter | Type | Default | Description |+|-----------|------|---------|-------------|+| `model` | str | `"FunAudioLLM/Fun-ASR-Nano-2512"` | Model name or local path |+| `hub` | str | `"ms"` | Model hub: `"ms"` (ModelScope) or `"hf"` (HuggingFace) |+| `device` | str | `"cuda:0"` | Device for audio encoder/adaptor |+| `dtype` | str | `"bf16"` | Compute dtype: `"bf16"`, `"fp16"`, `"fp32"` |+| `tensor_parallel_size` | int | `1` | Number of GPUs for vLLM tensor parallel |+| `gpu_memory_utilization` | float | `0.8` | Fraction of GPU memory for vLLM KV cache |+| `max_model_len` | int | `2048` | Maximum sequence length |++### `engine.generate()`++| Parameter | Type | Default | Description |+|-----------|------|---------|-------------|+| `inputs` | str/List[str] | - | Audio file path(s) or numpy arrays |+| `hotwords` | List[str] | `None` | Keywords to boost recognition |+| `language` | str | `None` | Language hint (e.g. "中文", "英文", "日文") |+| `itn` | bool | `True` | Apply inverse text normalization |+| `max_new_tokens` | int | `512` | Max generated tokens per sample |+| `temperature` | float | `0.0` | Sampling temperature (0 = greedy) |+| `top_p` | float | `1.0` | Nucleus sampling |+| `repetition_penalty` | float | `1.0` | Repetition penalty |++**Returns:** `List[dict]` with keys:+- `"key"`: Sample identifier (filename without extension)+- `"text"`: Recognized text+- `"timestamps"`: Character-level timestamps (if CTC decoder available)++## Performance++### Throughput Comparison++| Backend | Batch Size | 1 GPU | 2 GPUs | 4 GPUs |+|---------|-----------|-------|--------|--------|+| HuggingFace (baseline) | 1 | 1x | - | - |+| vLLM | 1 | ~1.5x | ~2x | ~2.5x |+| vLLM | 16 | ~4x | ~7x | ~12x |+| vLLM | 32 | ~5x | ~9x | ~15x |++*Approximate speedup for batch processing. Actual numbers depend on audio length and hardware.*++### Memory Usage++- Audio encoder + adaptor: ~400MB GPU memory (single GPU)+- vLLM LLM (Qwen3-0.6B): ~1.2GB per GPU (with bf16)+- KV cache: scales with `gpu_memory_utilization` setting++## Architecture Details++```+┌─────────────────────────────────────────────────────┐+│ FunASRNanoVLLM │+├─────────────────────────────────────────────────────┤+│ │+│ Audio Input │+│ │ │+│ ▼ │+│ ┌──────────────┐ │+│ │ WavFrontend │ (fbank, LFR, CMVN) │+│ └──────┬───────┘ │+│ ▼ │+│ ┌──────────────────────┐ │+│ │ SenseVoiceEncoder │ (50-layer SANM, frozen) │+│ │ (Audio Encoder) │ │+│ └──────┬───────────────┘ │+│ ▼ │+│ ┌──────────────────────┐ │+│ │ Transformer Adaptor │ (2-layer, downsample) │+│ │ (Audio Adaptor) │ │+│ └──────┬───────────────┘ │+│ ▼ │+│ [Audio Embeddings] ──────────┐ │+│ ▼ │+│ [Text Token Embeds] ───> [Concat] ─── EmbedsPrompt │+│ (prefix + suffix) │ │+│ ▼ │+│ ┌────────────────────┐ │+│ │ vLLM Engine │ │+│ │ (Qwen3-0.6B) │ │+│ │ Tensor Parallel │ │+│ └────────┬───────────┘ │+│ ▼ │+│ Generated Text │+│ │+│ (Optional) CTC Forced Alignment → Timestamps │+│ │+└─────────────────────────────────────────────────────┘+```++## Supported Models++| Model | Hub ID | Languages |+|-------|--------|-----------|+| Fun-ASR-Nano | `FunAudioLLM/Fun-ASR-Nano-2512` | Chinese, English, Japanese + dialects |+| Fun-ASR-MLT-Nano | `FunAudioLLM/Fun-ASR-MLT-Nano-2512` | 31 languages |++## Troubleshooting++### "model.pt not found"+The model must be fully downloaded first. On first use, `from_pretrained()` will download it automatically from the hub.++### CUDA out of memory+- Reduce `gpu_memory_utilization` (e.g. 0.6)+- Use `tensor_parallel_size` > 1 to split across GPUs+- Reduce `max_model_len`++### Slow first inference+The first call includes vLLM's CUDA graph warmup. Subsequent calls will be faster.++### Tokenizer warnings+These are typically harmless warnings from the Qwen3 tokenizer about special tokens and can be ignored.diff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.pynew file mode 100644index 000000000..927da7b5c--- /dev/null+++ b/funasr/models/fun_asr_nano/inference_vllm.py@@ -0,0 +1,650 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++"""+Fun-ASR-Nano vLLM inference engine.++Uses vLLM for high-throughput LLM decoding while keeping the audio encoder+and adaptor in PyTorch. Supports batch inference and tensor-parallel for+multi-GPU acceleration.++Usage:+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM++ engine = FunASRNanoVLLM.from_pretrained(+ model="FunAudioLLM/Fun-ASR-Nano-2512",+ tensor_parallel_size=2,+ )+ results = engine.generate(["audio1.wav", "audio2.wav"])+"""++import glob+import json+import logging+import os+import re+import shutil+import time+from typing import List, Optional, Union++import numpy as np+import torch+import torch.nn as nn++logger = logging.getLogger(__name__)++dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}+++def prepare_vllm_model_dir(model_dir: str, output_dir: str = None) -> str:+ """Extract LLM weights from Fun-ASR-Nano model.pt and save in HuggingFace format.++ Fun-ASR-Nano stores all weights (audio encoder + adaptor + LLM) in a single+ model.pt file. vLLM needs the LLM weights in standard HuggingFace format.+ This function extracts LLM weights and saves them alongside the config/tokenizer+ files from the Qwen3-0.6B subdirectory.++ Args:+ model_dir: Path to the Fun-ASR-Nano model directory.+ output_dir: Where to save the extracted LLM. Defaults to model_dir/Qwen3-0.6B-vllm.++ Returns:+ Path to the directory containing the vLLM-ready LLM model.+ """+ if output_dir is None:+ output_dir = os.path.join(model_dir, "Qwen3-0.6B-vllm")++ # Check if already prepared+ safetensors_files = glob.glob(os.path.join(output_dir, "*.safetensors"))+ bin_files = glob.glob(os.path.join(output_dir, "model*.bin"))+ if safetensors_files or bin_files:+ logger.info(f"vLLM model already prepared at {output_dir}")+ return output_dir++ os.makedirs(output_dir, exist_ok=True)++ # Copy config and tokenizer from Qwen3-0.6B+ qwen_dir = os.path.join(model_dir, "Qwen3-0.6B")+ if not os.path.isdir(qwen_dir):+ raise FileNotFoundError(f"Qwen3-0.6B config directory not found at {qwen_dir}")++ for fname in os.listdir(qwen_dir):+ src = os.path.join(qwen_dir, fname)+ dst = os.path.join(output_dir, fname)+ if os.path.isfile(src) and not os.path.exists(dst):+ shutil.copy2(src, dst)++ # Load model.pt and extract LLM weights+ model_pt = os.path.join(model_dir, "model.pt")+ if not os.path.exists(model_pt):+ raise FileNotFoundError(+ f"model.pt not found at {model_pt}. Make sure the model is fully downloaded."+ )++ logger.info(f"Loading model.pt from {model_pt}...")+ checkpoint = torch.load(model_pt, map_location="cpu")+ if "state_dict" in checkpoint:+ state_dict = checkpoint["state_dict"]+ else:+ state_dict = checkpoint++ # Extract LLM weights (prefixed with "llm.")+ llm_state = {}+ for key, value in state_dict.items():+ if key.startswith("llm."):+ new_key = key[len("llm."):]+ llm_state[new_key] = value++ if not llm_state:+ raise RuntimeError("No LLM weights found in model.pt (expected prefix 'llm.')")++ logger.info(f"Extracted {len(llm_state)} LLM weight tensors")++ # Save in safetensors format (preferred by vLLM)+ try:+ from safetensors.torch import save_file++ save_path = os.path.join(output_dir, "model.safetensors")+ save_file(llm_state, save_path)+ logger.info(f"Saved LLM weights to {save_path}")++ # Create model index+ index = {+ "metadata": {"total_size": sum(v.numel() * v.element_size() for v in llm_state.values())},+ "weight_map": {k: "model.safetensors" for k in llm_state.keys()},+ }+ with open(os.path.join(output_dir, "model.safetensors.index.json"), "w") as f:+ json.dump(index, f, indent=2)+ except ImportError:+ save_path = os.path.join(output_dir, "model.bin")+ torch.save(llm_state, save_path)+ logger.info(f"Saved LLM weights to {save_path} (install safetensors for faster loading)")++ return output_dir+++class FunASRNanoVLLM:+ """Fun-ASR-Nano with vLLM backend for high-throughput inference.++ Architecture:+ Audio -> WavFrontend -> SenseVoiceEncoder -> AudioAdaptor -> audio embeddings+ Text tokens -> LLM embedding layer -> text embeddings+ Combined embeddings -> vLLM (Qwen3-0.6B) -> generated text++ The audio encoder and adaptor run in PyTorch on a single GPU,+ while vLLM handles the LLM inference with optional tensor parallelism.++ Args:+ model_dir: Path to the Fun-ASR-Nano model directory.+ device: Device for audio encoder/adaptor (e.g. "cuda:0").+ dtype: Dtype for audio processing ("bf16", "fp16", "fp32").+ tensor_parallel_size: Number of GPUs for vLLM tensor parallelism.+ gpu_memory_utilization: Fraction of GPU memory for vLLM KV cache.+ max_model_len: Maximum sequence length for vLLM.+ enforce_eager: Disable CUDA graph for debugging.++ Example:+ >>> engine = FunASRNanoVLLM(+ ... model_dir="/path/to/Fun-ASR-Nano-2512",+ ... tensor_parallel_size=2,+ ... )+ >>> results = engine.generate(["audio1.wav", "audio2.wav"])+ >>> for r in results:+ ... print(r["text"])+ """++ def __init__(+ self,+ model_dir: str,+ device: str = "cuda:0",+ dtype: str = "bf16",+ tensor_parallel_size: int = 1,+ gpu_memory_utilization: float = 0.8,+ max_model_len: int = 2048,+ enforce_eager: bool = False,+ **kwargs,+ ):+ from vllm import LLM, SamplingParams+ from vllm.inputs.data import EmbedsPrompt++ self.device = device+ self.dtype = dtype+ self.torch_dtype = dtype_map.get(dtype, torch.bfloat16)+ self.model_dir = model_dir++ # Step 1: Prepare LLM weights for vLLM (extract from model.pt if needed)+ vllm_model_dir = prepare_vllm_model_dir(model_dir)++ # Step 2: Load audio components (encoder + adaptor + frontend)+ self._load_audio_components(model_dir, **kwargs)++ # Step 3: Initialize vLLM engine+ logger.info(f"Initializing vLLM with model: {vllm_model_dir}")+ logger.info(f" tensor_parallel_size={tensor_parallel_size}")+ logger.info(f" gpu_memory_utilization={gpu_memory_utilization}")++ vllm_kwargs = kwargs.get("vllm_kwargs", {})+ self.vllm_engine = LLM(+ enable_prompt_embeds=True,+ model=vllm_model_dir,+ tensor_parallel_size=tensor_parallel_size,+ gpu_memory_utilization=gpu_memory_utilization,+ max_model_len=max_model_len,+ enforce_eager=enforce_eager,+ dtype={"bf16": "bfloat16", "fp16": "float16", "fp32": "auto"}.get(dtype, dtype),+ trust_remote_code=True,+ **vllm_kwargs,+ )++ # Step 4: Get tokenizer and LLM embedding layer+ self.tokenizer = self.vllm_engine.get_tokenizer()+ self._load_embedding_layer(model_dir)++ def _load_audio_components(self, model_dir: str, **kwargs):+ """Load audio encoder, adaptor, frontend, and CTC from checkpoint."""+ from omegaconf import OmegaConf+ from funasr.register import tables++ config_path = os.path.join(model_dir, "config.yaml")+ config = OmegaConf.load(config_path)+ self._config = OmegaConf.to_container(config, resolve=True)++ # --- Frontend ---+ frontend_class = tables.frontend_classes.get(config["frontend"])+ frontend_conf = OmegaConf.to_container(config.get("frontend_conf", {}), resolve=True)+ cmvn_file = frontend_conf.get("cmvn_file")+ if cmvn_file and not os.path.isabs(cmvn_file):+ frontend_conf["cmvn_file"] = os.path.join(model_dir, cmvn_file)+ self.frontend = frontend_class(**frontend_conf)+ self.frontend.eval()++ # --- Audio Encoder ---+ encoder_conf = OmegaConf.to_container(config.get("audio_encoder_conf", {}), resolve=True)+ hub = encoder_conf.get("hub", None)+ if hub == "ms":+ from funasr import AutoModel as FunAutoModel++ enc_model = FunAutoModel(+ model=config["audio_encoder"], model_revision="master", disable_update=True+ )+ self.audio_encoder_output_size = (+ enc_model.model.encoder_output_size+ if hasattr(enc_model.model, "encoder_output_size")+ else -1+ )+ self.audio_encoder = (+ enc_model.model.model.encoder+ if hasattr(enc_model.model, "model")+ else enc_model.model.encoder+ )+ else:+ encoder_class = tables.encoder_classes.get(config["audio_encoder"])+ input_size = self.frontend.output_size()+ self.audio_encoder = encoder_class(input_size=input_size, **encoder_conf)+ self.audio_encoder_output_size = self.audio_encoder.output_size()++ self.audio_encoder.eval()+ for p in self.audio_encoder.parameters():+ p.requires_grad = False++ # --- Audio Adaptor ---+ adaptor_conf = OmegaConf.to_container(config.get("audio_adaptor_conf", {}), resolve=True)+ adaptor_class = tables.adaptor_classes.get(config["audio_adaptor"])+ if self.audio_encoder_output_size > 0:+ adaptor_conf["encoder_dim"] = self.audio_encoder_output_size+ self.audio_adaptor = adaptor_class(**adaptor_conf)+ self.audio_adaptor.eval()+ for p in self.audio_adaptor.parameters():+ p.requires_grad = False+ self.use_low_frame_rate = adaptor_conf.get("use_low_frame_rate", False)++ # --- CTC Decoder (optional, for timestamps) ---+ self.ctc_decoder = None+ self.ctc = None+ self.ctc_tokenizer = None+ self.blank_id = None++ ctc_decoder_name = self._config.get("ctc_decoder", None)+ if ctc_decoder_name:+ ctc_decoder_class = tables.adaptor_classes.get(ctc_decoder_name)+ ctc_decoder_conf = self._config.get("ctc_decoder_conf", {})+ if self.audio_encoder_output_size > 0:+ ctc_decoder_conf["encoder_dim"] = self.audio_encoder_output_size+ self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)+ self.ctc_decoder.eval()+ for p in self.ctc_decoder.parameters():+ p.requires_grad = False++ from funasr.models.fun_asr_nano.ctc import CTC++ ctc_conf = self._config.get("ctc_conf", {})+ ctc_vocab_size = self._config.get("ctc_vocab_size", 60515)+ self.blank_id = ctc_conf.get("blank_id", ctc_vocab_size - 1)+ self.ctc = CTC(+ odim=ctc_vocab_size,+ encoder_output_size=self.audio_encoder_output_size,+ blank_id=self.blank_id,+ **ctc_conf,+ )++ # CTC tokenizer+ ds_conf = self._config.get("dataset_conf", {})+ ctc_tokenizer_name = ds_conf.get("ctc_tokenizer", None)+ ctc_tokenizer_conf = ds_conf.get("ctc_tokenizer_conf", {})+ if ctc_tokenizer_name:+ ctc_tokenizer_class = tables.tokenizer_classes.get(ctc_tokenizer_name)+ vocab_path = ctc_tokenizer_conf.get("vocab_path")+ if vocab_path is None or not os.path.isabs(vocab_path):+ multilingual_path = os.path.join(model_dir, "multilingual.tiktoken")+ if os.path.exists(multilingual_path):+ ctc_tokenizer_conf["vocab_path"] = multilingual_path+ elif vocab_path and not os.path.isabs(vocab_path):+ ctc_tokenizer_conf["vocab_path"] = os.path.join(model_dir, vocab_path)+ self.ctc_tokenizer = ctc_tokenizer_class(**ctc_tokenizer_conf)++ # --- Load weights from model.pt ---+ model_pt = os.path.join(model_dir, "model.pt")+ if os.path.exists(model_pt):+ logger.info(f"Loading audio component weights from {model_pt}")+ checkpoint = torch.load(model_pt, map_location="cpu")+ state_dict = checkpoint.get("state_dict", checkpoint)++ # Audio encoder+ enc_state = {+ k[len("audio_encoder."):]: v+ for k, v in state_dict.items()+ if k.startswith("audio_encoder.")+ }+ if enc_state:+ self.audio_encoder.load_state_dict(enc_state, strict=False)+ logger.info(f" Loaded audio_encoder: {len(enc_state)} params")++ # Audio adaptor+ adp_state = {+ k[len("audio_adaptor."):]: v+ for k, v in state_dict.items()+ if k.startswith("audio_adaptor.")+ }+ if adp_state:+ self.audio_adaptor.load_state_dict(adp_state, strict=False)+ logger.info(f" Loaded audio_adaptor: {len(adp_state)} params")++ # CTC decoder+ if self.ctc_decoder is not None:+ ctc_dec_state = {+ k[len("ctc_decoder."):]: v+ for k, v in state_dict.items()+ if k.startswith("ctc_decoder.")+ }+ if ctc_dec_state:+ self.ctc_decoder.load_state_dict(ctc_dec_state, strict=False)+ ctc_state = {+ k[len("ctc."):]: v+ for k, v in state_dict.items()+ if k.startswith("ctc.") and not k.startswith("ctc_decoder.")+ }+ if ctc_state:+ self.ctc.load_state_dict(ctc_state, strict=False)++ # Move to device+ self.audio_encoder = self.audio_encoder.to(self.device, dtype=torch.float32)+ self.audio_adaptor = self.audio_adaptor.to(self.device, dtype=self.torch_dtype)+ if self.ctc_decoder is not None:+ self.ctc_decoder = self.ctc_decoder.to(self.device, dtype=torch.float32)+ self.ctc = self.ctc.to(self.device, dtype=torch.float32)++ def _load_embedding_layer(self, model_dir: str):+ """Load the LLM embedding layer for text token embedding computation."""+ model_pt = os.path.join(model_dir, "model.pt")+ checkpoint = torch.load(model_pt, map_location="cpu")+ state_dict = checkpoint.get("state_dict", checkpoint)++ # Look for embedding weights+ embed_key = None+ for key in state_dict.keys():+ if "embed_tokens.weight" in key and key.startswith("llm."):+ embed_key = key+ break++ if embed_key is None:+ raise RuntimeError("Could not find LLM embedding weights in model.pt")++ embed_weight = state_dict[embed_key]+ self.embed_tokens = nn.Embedding.from_pretrained(embed_weight, freeze=True)+ self.embed_tokens = self.embed_tokens.to(self.device, dtype=self.torch_dtype)+ logger.info(f"Loaded embedding layer: {embed_weight.shape}")++ @torch.no_grad()+ def _encode_audio(self, audio_input: Union[str, torch.Tensor, np.ndarray]):+ """Encode audio through frontend -> encoder -> adaptor.++ Returns:+ adaptor_out: (1, T', D_llm) audio embeddings for LLM input+ adaptor_out_lens: (1,) lengths+ encoder_out: (1, T, D_enc) encoder output for CTC+ encoder_out_lens: (1,) encoder output lengths+ """+ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank++ if isinstance(audio_input, str):+ data_src = load_audio_text_image_video(audio_input, fs=self.frontend.fs)+ elif isinstance(audio_input, np.ndarray):+ data_src = torch.from_numpy(audio_input).float()+ elif isinstance(audio_input, torch.Tensor):+ data_src = audio_input.float()+ else:+ raise ValueError(f"Unsupported audio input type: {type(audio_input)}")++ speech, speech_lengths = extract_fbank(+ data_src, data_type="sound", frontend=self.frontend, is_final=True+ )+ speech = speech.to(self.device, dtype=torch.float32)+ speech_lengths = speech_lengths.to(self.device)++ encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)+ encoder_out_for_adaptor = encoder_out.to(dtype=self.torch_dtype)+ adaptor_out, adaptor_out_lens = self.audio_adaptor(encoder_out_for_adaptor, encoder_out_lens)+ return adaptor_out, adaptor_out_lens, encoder_out, encoder_out_lens++ def _build_prompt_text(+ self,+ hotwords: List[str] = None,+ language: str = None,+ itn: bool = True,+ ) -> str:+ """Build the ASR prompt string."""+ hotwords = hotwords or []+ if len(hotwords) > 0:+ hotwords_str = ", ".join(hotwords)+ prompt = (+ "请结合上下文信息,更加准确地完成语音转写任务。"+ "如果没有相关信息,我们会留空。\n\n\n**上下文信息:**\n\n\n"+ )+ prompt += f"热词列表:[{hotwords_str}]\n"+ else:+ prompt = ""+ if language is None:+ prompt += "语音转写"+ else:+ prompt += f"语音转写成{language}"+ if not itn:+ prompt += ",不进行文本规整"+ return prompt + ":"++ @torch.no_grad()+ def _build_input_embeds(+ self,+ audio_embeds: torch.Tensor,+ audio_embed_lens: torch.Tensor,+ hotwords: List[str] = None,+ language: str = None,+ itn: bool = True,+ system_prompt: str = "You are a helpful assistant.",+ ) -> torch.Tensor:+ """Build the full input embedding sequence with audio inserted.++ Returns:+ Tensor of shape (seq_len, D_llm)+ """+ prompt = self._build_prompt_text(hotwords, language, itn)++ # ChatML format+ prefix_text = (+ f"<|im_start|>system\n{system_prompt}<|im_end|>\n"+ f"<|im_start|>user\n{prompt}<|startofspeech|>"+ )+ suffix_text = "<|endofspeech|><|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"++ # Tokenize+ prefix_ids = self.tokenizer.encode(prefix_text, add_special_tokens=False)+ suffix_ids = self.tokenizer.encode(suffix_text, add_special_tokens=False)++ # Embed text tokens+ prefix_tensor = torch.tensor(prefix_ids, dtype=torch.long, device=self.device)+ suffix_tensor = torch.tensor(suffix_ids, dtype=torch.long, device=self.device)+ prefix_embeds = self.embed_tokens(prefix_tensor)+ suffix_embeds = self.embed_tokens(suffix_tensor)++ # Audio embeddings+ audio_len = audio_embed_lens[0].item()+ audio_emb = audio_embeds[0, :audio_len, :]++ # Concat: [prefix_text_emb | audio_emb | suffix_text_emb]+ inputs_embeds = torch.cat([prefix_embeds, audio_emb, suffix_embeds], dim=0)+ return inputs_embeds++ def generate(+ self,+ inputs: Union[str, List[str], np.ndarray, torch.Tensor, List],+ hotwords: List[str] = None,+ language: str = None,+ itn: bool = True,+ max_new_tokens: int = 512,+ temperature: float = 0.0,+ top_p: float = 1.0,+ top_k: int = -1,+ repetition_penalty: float = 1.0,+ **kwargs,+ ) -> List[dict]:+ """Run batch ASR inference using vLLM.++ Args:+ inputs: Audio input(s). Accepts:+ - str: single file path+ - List[str]: batch of file paths+ - np.ndarray / torch.Tensor: raw audio samples (16kHz)+ hotwords: Keywords to boost recognition accuracy.+ language: Language hint (e.g. "中文", "英文", "日文").+ itn: Apply inverse text normalization (default True).+ max_new_tokens: Maximum tokens to generate per sample.+ temperature: Sampling temperature (0 = greedy decoding).+ top_p: Nucleus sampling parameter.+ top_k: Top-k sampling (-1 = disabled).+ repetition_penalty: Repetition penalty factor.++ Returns:+ List of result dicts: [{"key": str, "text": str, "timestamps": [...]}]+ """+ from vllm import SamplingParams+ from vllm.inputs.data import EmbedsPrompt++ if isinstance(inputs, (str, np.ndarray, torch.Tensor)):+ inputs = [inputs]++ sampling_params = SamplingParams(+ max_tokens=max_new_tokens,+ temperature=temperature,+ top_p=top_p,+ top_k=top_k if top_k > 0 else -1,+ repetition_penalty=repetition_penalty,+ skip_special_tokens=True,+ )++ # Encode all audio samples and build embedding prompts+ prompts = []+ encoder_outputs = []++ t0 = time.perf_counter()+ for audio_input in inputs:+ adaptor_out, adaptor_out_lens, encoder_out, encoder_out_lens = self._encode_audio(+ audio_input+ )+ encoder_outputs.append((encoder_out, encoder_out_lens))++ input_embeds = self._build_input_embeds(+ adaptor_out, adaptor_out_lens, hotwords, language, itn+ )+ # vLLM EmbedsPrompt expects float32 or the model's dtype+ prompts.append(EmbedsPrompt(prompt_embeds=input_embeds.float()))++ t1 = time.perf_counter()+ logger.info(f"Audio encoding: {len(inputs)} samples in {t1 - t0:.3f}s")++ # vLLM batch generation+ outputs = self.vllm_engine.generate(prompts, sampling_params, use_tqdm=len(inputs) > 1)++ t2 = time.perf_counter()+ logger.info(f"vLLM generation: {t2 - t1:.3f}s")++ # Process results+ results = []+ for i, output in enumerate(outputs):+ text = output.outputs[0].text+ if not text and output.outputs[0].token_ids:+ text = self.tokenizer.decode(list(output.outputs[0].token_ids), skip_special_tokens=True)+ text_clean = re.sub(r"\s+", " ", text.replace("/sil", " ")).strip()++ key = (+ os.path.splitext(os.path.basename(inputs[i]))[0]+ if isinstance(inputs[i], str)+ else f"sample_{i}"+ )+ result = {"key": key, "text": text_clean}++ # Timestamps via CTC forced alignment+ if self.ctc_decoder is not None and self.ctc_tokenizer is not None:+ try:+ timestamps = self._compute_timestamps(+ encoder_outputs[i][0], encoder_outputs[i][1], text_clean+ )+ if timestamps:+ result["timestamps"] = timestamps+ except Exception as e:+ logger.debug(f"Timestamp computation failed for {key}: {e}")++ results.append(result)++ return results++ @torch.no_grad()+ def _compute_timestamps(self, encoder_out, encoder_out_lens, text):+ """CTC forced alignment for character-level timestamps."""+ from funasr.models.fun_asr_nano.tools.utils import forced_align++ decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)+ ctc_logits = self.ctc.log_softmax(decoder_out)+ x = ctc_logits[0, : encoder_out_lens[0].item(), :]++ target_ids = torch.tensor(self.ctc_tokenizer.encode(text), dtype=torch.int64)+ if len(target_ids) == 0:+ return []++ timestamps = forced_align(x, target_ids, self.blank_id)+ for ts in timestamps:+ ts["token"] = self.ctc_tokenizer.decode([ts["token"]])+ ts["start_time"] = ts["start_time"] * 6 * 10 / 1000+ ts["end_time"] = ts["end_time"] * 6 * 10 / 1000+ return timestamps++ @classmethod+ def from_pretrained(+ cls,+ model: str = "FunAudioLLM/Fun-ASR-Nano-2512",+ hub: str = "ms",+ device: str = "cuda:0",+ dtype: str = "bf16",+ tensor_parallel_size: int = 1,+ gpu_memory_utilization: float = 0.8,+ max_model_len: int = 2048,+ **kwargs,+ ) -> "FunASRNanoVLLM":+ """Load model from hub or local path.++ Args:+ model: Model name or local directory path.+ hub: "ms" (ModelScope) or "hf" (HuggingFace).+ device: Device for audio encoder/adaptor.+ dtype: Compute dtype ("bf16", "fp16", "fp32").+ tensor_parallel_size: GPUs for vLLM tensor parallel.+ gpu_memory_utilization: GPU memory fraction for vLLM.+ max_model_len: Maximum sequence length.++ Returns:+ Initialized FunASRNanoVLLM engine.+ """+ if os.path.isdir(model):+ model_dir = model+ else:+ if hub in ("ms", "modelscope"):+ from modelscope.hub.snapshot_download import snapshot_download++ model_dir = snapshot_download(model, revision=kwargs.pop("revision", "master"))+ elif hub in ("hf", "huggingface"):+ from huggingface_hub import snapshot_download++ model_dir = snapshot_download(model)+ else:+ raise ValueError(f"Unsupported hub: {hub}. Use 'ms' or 'hf'.")++ logger.info(f"Model directory: {model_dir}")+ return cls(+ model_dir=model_dir,+ device=device,+ dtype=dtype,+ tensor_parallel_size=tensor_parallel_size,+ gpu_memory_utilization=gpu_memory_utilization,+ max_model_len=max_model_len,+ **kwargs,+ )