Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: disable timestamps when CTC weights are missing (#3211)
zhifu gaoaefddc1088ff
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/README.md b/examples/industrial_data_pretraining/fun_asr_nano/README.mdindex 8c3e83763..8f15e38da 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/README.md+++ b/examples/industrial_data_pretraining/fun_asr_nano/README.md@@ -64,6 +64,12 @@ pip install -r requirements.txt- [ ] Support speaker diarization- [x] Support model training+> [!NOTE]+> Character-level timestamps require a checkpoint with complete `ctc_decoder.*` and+> `ctc.*` weights. The current `Fun-ASR-MLT-Nano-2512` checkpoint does not include+> those weights, so FunASR logs a warning and returns text without `timestamps` or+> `ctc_timestamps` instead of aligning with uninitialized layers.+# Usage 🛠️## Inferencediff --git a/examples/industrial_data_pretraining/fun_asr_nano/README_zh.md b/examples/industrial_data_pretraining/fun_asr_nano/README_zh.mdindex 2c18530e9..a5def3845 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/README_zh.md+++ b/examples/industrial_data_pretraining/fun_asr_nano/README_zh.md@@ -64,6 +64,11 @@ pip install -r requirements.txt- [ ] 支持区分说话人识别- [x] 支持模型训练+> [!NOTE]+> 字符级时间戳要求 checkpoint 同时包含完整的 `ctc_decoder.*` 和 `ctc.*` 权重。+> 当前 `Fun-ASR-MLT-Nano-2512` checkpoint 未包含这些权重,因此 FunASR 会记录警告并仅返回文本,+> 不再使用未初始化的层生成 `timestamps` 或 `ctc_timestamps`。+# 用法 🛠️## 推理diff --git a/examples/industrial_data_pretraining/fun_asr_nano/model.py b/examples/industrial_data_pretraining/fun_asr_nano/model.pyindex 88c52e9ce..af3513ce4 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/model.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/model.py@@ -15,6 +15,10 @@ from funasr.register import tablesfrom funasr.train_utils.device_funcs import force_gatherable, to_devicefrom funasr.utils.datadir_writer import DatadirWriterfrom funasr.utils.load_utils import extract_fbank, load_audio_text_image_video+from funasr.models.fun_asr_nano.checkpoint_utils import (+ disable_incomplete_ctc,+ normalize_checkpoint_state,+)from funasr.models.fun_asr_nano.device_utils import resolve_autocast_device_type@@ -109,6 +113,7 @@ class FunASRNano(nn.Module):# ctc decoderself.ctc_decoder = None+ self._externally_loaded_ctc_keys = set()# TODO: fix table namectc_decoder_class = tables.adaptor_classes.get(kwargs.get("ctc_decoder", None))if ctc_decoder_class is not None:@@ -135,8 +140,15 @@ class FunASRNano(nn.Module):self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)init_param_path = ctc_decoder_conf.get("init_param_path", None)if init_param_path is not None:- src_state = torch.load(init_param_path, map_location="cpu")+ src_state = normalize_checkpoint_state(+ torch.load(init_param_path, map_location="cpu")+ )flag = self.ctc_decoder.load_state_dict(src_state, strict=False)+ self._externally_loaded_ctc_keys.update(+ f"ctc_decoder.{key}"+ for key in self.ctc_decoder.state_dict()+ if key in src_state+ )logging.info(f"Loading ctc_decoder ckpt: {init_param_path}, status: {flag}")freeze = ctc_decoder_conf.get("freeze", False)if freeze:@@ -160,6 +172,11 @@ class FunASRNano(nn.Module):rank = int(os.environ.get("RANK", 0))logging.info(f"rank: {rank}, model is builded.")+ def on_pretrained_model_loaded(self, loaded_keys):+ """Fail closed when a checkpoint configures CTC without trained weights."""+ loaded_keys = set(loaded_keys).union(getattr(self, "_externally_loaded_ctc_keys", ()))+ disable_incomplete_ctc(self, loaded_keys, log=logging)+def forward(self,speech: torch.Tensor = None,diff --git a/funasr/models/fun_asr_nano/checkpoint_utils.py b/funasr/models/fun_asr_nano/checkpoint_utils.pynew file mode 100644index 000000000..6759ed939--- /dev/null+++ b/funasr/models/fun_asr_nano/checkpoint_utils.py@@ -0,0 +1,47 @@+def normalize_checkpoint_state(state):+ """Unwrap common checkpoint containers and remove one DDP key prefix."""+ while isinstance(state, dict):+ wrapped = next(+ (+ state[key]+ for key in ("state_dict", "model_state_dict", "model")+ if isinstance(state.get(key), dict)+ ),+ None,+ )+ if wrapped is None or wrapped is state:+ break+ state = wrapped++ return {+ key[len("module.") :] if key.startswith("module.") else key: value+ for key, value in state.items()+ }+++def disable_incomplete_ctc(model, loaded_keys, *, log):+ """Disable timestamp inference unless every required CTC tensor was loaded."""+ if model.ctc_decoder is None or model.ctc is None:+ return []++ expected = {f"ctc_decoder.{key}" for key in model.ctc_decoder.state_dict()}+ expected.update(f"ctc.{key}" for key in model.ctc.state_dict())+ missing = sorted(expected.difference(loaded_keys))+ if not missing:+ return []++ preview = ", ".join(missing[:3])+ suffix = "" if len(missing) <= 3 else ", ..."+ log.warning(+ "Disabling CTC timestamps because the checkpoint did not initialize "+ "%d of %d required CTC tensors (%s%s). Text transcription remains available.",+ len(missing),+ len(expected),+ preview,+ suffix,+ )+ model.ctc_decoder = None+ model.ctc = None+ model.ctc_tokenizer = None+ model.blank_id = None+ return missingdiff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.pyindex 6c5763105..48a0ad7b8 100644--- a/funasr/models/fun_asr_nano/inference_vllm.py+++ b/funasr/models/fun_asr_nano/inference_vllm.py@@ -33,6 +33,8 @@ import numpy as npimport torchimport torch.nn as nn+from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_state+logger = logging.getLogger(__name__)dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}@@ -343,20 +345,7 @@ class FunASRNanoVLLM:# CTC decoderif self.ctc_decoder is not None:- ctc_dec_state = {- k[len("ctc_decoder."):]: v- for k, v in state_dict.items()- if k.startswith("ctc_decoder.")- }- if ctc_dec_state:- self.ctc_decoder.load_state_dict(ctc_dec_state, strict=False)- ctc_state = {- k[len("ctc."):]: v- for k, v in state_dict.items()- if k.startswith("ctc.") and not k.startswith("ctc_decoder.")- }- if ctc_state:- self.ctc.load_state_dict(ctc_state, strict=False)+ self._load_ctc_weights(state_dict)# Move to deviceself.audio_encoder = self.audio_encoder.to(self.device, dtype=torch.float32)@@ -365,6 +354,35 @@ class FunASRNanoVLLM:self.ctc_decoder = self.ctc_decoder.to(self.device, dtype=torch.float32)self.ctc = self.ctc.to(self.device, dtype=torch.float32)+ def _load_ctc_weights(self, state_dict):+ """Load timestamp modules and disable them when the checkpoint is incomplete."""+ if self.ctc_decoder is None or self.ctc is None:+ return++ state_dict = normalize_checkpoint_state(state_dict)++ def compatible_state(module, prefix):+ expected = module.state_dict()+ return {+ key[len(prefix) :]: value+ for key, value in state_dict.items()+ if key.startswith(prefix)+ and key[len(prefix) :] in expected+ and value.size() == expected[key[len(prefix) :]].size()+ }++ ctc_dec_state = compatible_state(self.ctc_decoder, "ctc_decoder.")+ if ctc_dec_state:+ self.ctc_decoder.load_state_dict(ctc_dec_state, strict=False)++ ctc_state = compatible_state(self.ctc, "ctc.")+ if ctc_state:+ self.ctc.load_state_dict(ctc_state, strict=False)++ loaded_keys = {f"ctc_decoder.{key}" for key in ctc_dec_state}+ loaded_keys.update(f"ctc.{key}" for key in ctc_state)+ disable_incomplete_ctc(self, loaded_keys, log=logger)+def _load_embedding_layer(self, model_dir: str):"""Load the LLM embedding layer for text token embedding computation."""model_pt = os.path.join(model_dir, "model.pt")diff --git a/funasr/models/fun_asr_nano/model.py b/funasr/models/fun_asr_nano/model.pyindex ca53eab74..8b78fed67 100644--- a/funasr/models/fun_asr_nano/model.py+++ b/funasr/models/fun_asr_nano/model.py@@ -22,6 +22,7 @@ except ImportError:AutoModelForCausalLM = Nonefrom .ctc import CTC+from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_statefrom .device_utils import resolve_autocast_device_typefrom .tools.utils import forced_align@@ -145,6 +146,7 @@ class FunASRNano(nn.Module):# ctc decoderself.ctc_decoder = None+ self._externally_loaded_ctc_keys = set()# TODO: fix table namectc_decoder_class = tables.adaptor_classes.get(kwargs.get("ctc_decoder", None))if ctc_decoder_class is not None:@@ -171,8 +173,15 @@ class FunASRNano(nn.Module):self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)init_param_path = ctc_decoder_conf.get("init_param_path", None)if init_param_path is not None:- src_state = torch.load(init_param_path, map_location="cpu")+ src_state = normalize_checkpoint_state(+ torch.load(init_param_path, map_location="cpu")+ )flag = self.ctc_decoder.load_state_dict(src_state, strict=False)+ self._externally_loaded_ctc_keys.update(+ f"ctc_decoder.{key}"+ for key in self.ctc_decoder.state_dict()+ if key in src_state+ )logging.info(f"Loading ctc_decoder ckpt: {init_param_path}, status: {flag}")freeze = ctc_decoder_conf.get("freeze", False)if freeze:@@ -196,6 +205,11 @@ class FunASRNano(nn.Module):rank = int(os.environ.get("RANK", 0))logging.info(f"rank: {rank}, model is builded.")+ def on_pretrained_model_loaded(self, loaded_keys):+ """Fail closed when a checkpoint configures CTC without trained weights."""+ loaded_keys = set(loaded_keys).union(getattr(self, "_externally_loaded_ctc_keys", ()))+ disable_incomplete_ctc(self, loaded_keys, log=logging)+def forward(self,speech: torch.Tensor = None,diff --git a/funasr/train_utils/load_pretrained_model.py b/funasr/train_utils/load_pretrained_model.pyindex da2eed54f..383c22a4b 100644--- a/funasr/train_utils/load_pretrained_model.py+++ b/funasr/train_utils/load_pretrained_model.py@@ -45,6 +45,7 @@ def load_pretrained_model(src_state = src_state["state_dict"] if "state_dict" in src_state else src_statesrc_state = src_state["model_state_dict"] if "model_state_dict" in src_state else src_statesrc_state = src_state["model"] if "model" in src_state else src_state+ loaded_keys = set()if isinstance(scope_map, str):scope_map = scope_map.split(",")@@ -96,8 +97,18 @@ def load_pretrained_model()else:dst_state[k] = src_state[k_src]+ loaded_keys.add(k)else:print(f"Warning, miss key in ckpt: {k}, {path}")flag = obj.load_state_dict(dst_state, strict=True)+ on_loaded = getattr(obj, "on_pretrained_model_loaded", None)+ if not callable(on_loaded) and hasattr(obj, "module"):+ on_loaded = getattr(obj.module, "on_pretrained_model_loaded", None)+ if callable(on_loaded):+ loaded_keys = {+ key[len("module.") :] if key.startswith("module.") else key for key in loaded_keys+ }+ if callable(on_loaded):+ on_loaded(frozenset(loaded_keys))logging.info(f"Loading ckpt: {path}, status: {flag}")diff --git a/tests/test_fun_asr_nano_missing_ctc_weights.py b/tests/test_fun_asr_nano_missing_ctc_weights.pynew file mode 100644index 000000000..baa409dc9--- /dev/null+++ b/tests/test_fun_asr_nano_missing_ctc_weights.py@@ -0,0 +1,188 @@+import pytest+import torch++from funasr.models.fun_asr_nano import checkpoint_utils+from funasr.models.fun_asr_nano import model as nano_model+from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM+from funasr.train_utils.load_pretrained_model import load_pretrained_model+++class _HookedModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.loaded = torch.nn.Parameter(torch.zeros(2))+ self.mismatched = torch.nn.Parameter(torch.zeros(2))+ self.absent = torch.nn.Parameter(torch.zeros(2))+ self.loaded_checkpoint_keys = None++ def on_pretrained_model_loaded(self, loaded_keys):+ self.loaded_checkpoint_keys = set(loaded_keys)+++class _WrappedModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.module = _HookedModel()+++def _attach_ctc_modules(instance):+ instance.ctc_decoder = torch.nn.Linear(2, 2)+ instance.ctc = torch.nn.Linear(2, 2)+ instance.ctc_tokenizer = object()+ instance.blank_id = 2+++def _complete_ctc_state(instance):+ state = {+ f"ctc_decoder.{key}": value.clone()+ for key, value in instance.ctc_decoder.state_dict().items()+ }+ state.update({f"ctc.{key}": value.clone() for key, value in instance.ctc.state_dict().items()})+ return state+++def test_pretrained_loader_reports_only_keys_loaded_from_checkpoint(tmp_path):+ checkpoint = tmp_path / "model.pt"+ torch.save(+ {"state_dict": {"loaded": torch.ones(2), "mismatched": torch.ones(1)}},+ checkpoint,+ )+ model = _HookedModel()++ load_pretrained_model(path=str(checkpoint), model=model)++ assert model.loaded_checkpoint_keys == {"loaded"}+++def test_pretrained_loader_calls_inner_hook_with_clean_ddp_keys(tmp_path):+ checkpoint = tmp_path / "model.pt"+ torch.save({"state_dict": {"module.loaded": torch.ones(2)}}, checkpoint)+ model = _WrappedModel()++ load_pretrained_model(path=str(checkpoint), model=model)++ assert model.module.loaded_checkpoint_keys == {"loaded"}+++def test_checkpoint_state_normalization_unwraps_and_removes_ddp_prefix():+ normalizer = getattr(checkpoint_utils, "normalize_checkpoint_state", None)+ assert callable(normalizer)+ tensor = torch.ones(2)++ state = normalizer({"state_dict": {"model": {"module.weight": tensor}}})++ assert state == {"weight": tensor}+++def test_native_model_disables_timestamps_when_ctc_weights_are_missing():+ model = object.__new__(nano_model.FunASRNano)+ torch.nn.Module.__init__(model)+ _attach_ctc_modules(model)++ model.on_pretrained_model_loaded(set())++ assert model.ctc_decoder is None+ assert model.ctc is None+ assert model.ctc_tokenizer is None+ assert model.blank_id is None+++def test_native_model_keeps_timestamps_when_ctc_weights_are_complete():+ model = object.__new__(nano_model.FunASRNano)+ torch.nn.Module.__init__(model)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)++ model.on_pretrained_model_loaded(state)++ assert model.ctc_decoder is not None+ assert model.ctc is not None+ assert model.ctc_tokenizer is not None+ assert model.blank_id == 2+++def test_native_model_disables_timestamps_when_ctc_weights_are_incomplete():+ model = object.__new__(nano_model.FunASRNano)+ torch.nn.Module.__init__(model)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)+ state.pop("ctc.bias")++ model.on_pretrained_model_loaded(state)++ assert model.ctc_decoder is None+ assert model.ctc is None+ assert model.ctc_tokenizer is None+ assert model.blank_id is None+++def test_vllm_model_disables_timestamps_when_ctc_weights_are_missing():+ model = object.__new__(FunASRNanoVLLM)+ _attach_ctc_modules(model)++ model._load_ctc_weights({})++ assert model.ctc_decoder is None+ assert model.ctc is None+ assert model.ctc_tokenizer is None+ assert model.blank_id is None+++def test_vllm_model_keeps_timestamps_when_ctc_weights_are_complete():+ model = object.__new__(FunASRNanoVLLM)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)++ model._load_ctc_weights(state)++ assert model.ctc_decoder is not None+ assert model.ctc is not None+ assert model.ctc_tokenizer is not None+ assert model.blank_id == 2++[email protected]("checkpoint_format", ["ddp", "wrapped"])+def test_vllm_model_keeps_timestamps_for_normalized_checkpoint_state(checkpoint_format):+ model = object.__new__(FunASRNanoVLLM)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)+ if checkpoint_format == "ddp":+ state = {f"module.{key}": value for key, value in state.items()}+ else:+ state = {"model_state_dict": state}++ model._load_ctc_weights(state)++ assert model.ctc_decoder is not None+ assert model.ctc is not None+ assert model.ctc_tokenizer is not None+ assert model.blank_id == 2+++def test_vllm_model_disables_timestamps_when_ctc_weights_are_incomplete():+ model = object.__new__(FunASRNanoVLLM)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)+ state.pop("ctc_decoder.bias")++ model._load_ctc_weights(state)++ assert model.ctc_decoder is None+ assert model.ctc is None+ assert model.ctc_tokenizer is None+ assert model.blank_id is None++[email protected]("key", ["ctc_decoder.weight", "ctc.bias"])+def test_vllm_model_disables_timestamps_when_ctc_weight_shape_mismatches(key):+ model = object.__new__(FunASRNanoVLLM)+ _attach_ctc_modules(model)+ state = _complete_ctc_state(model)+ state[key] = torch.zeros(1)++ model._load_ctc_weights(state)++ assert model.ctc_decoder is None+ assert model.ctc is None+ assert model.ctc_tokenizer is None+ assert model.blank_id is None