Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Dev gzf (#1428)
zhifu gao753d579531e1
diff --git a/examples/industrial_data_pretraining/qwen_audio/demo.py b/examples/industrial_data_pretraining/qwen_audio/demo.pynew file mode 100644index 000000000..5c1e2a0e2--- /dev/null+++ b/examples/industrial_data_pretraining/qwen_audio/demo.py@@ -0,0 +1,15 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# To install requirements: pip3 install -U "funasr[llm]"++from funasr import AutoModel++model = AutoModel(model="Qwen/Qwen-Audio",+ model_path=None,+ )++res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", language=None)+print(res)diff --git a/examples/industrial_data_pretraining/qwen_audio/demo_chat.py b/examples/industrial_data_pretraining/qwen_audio/demo_chat.pynew file mode 100644index 000000000..4fb5e5bbb--- /dev/null+++ b/examples/industrial_data_pretraining/qwen_audio/demo_chat.py@@ -0,0 +1,26 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# To install requirements: pip3 install -U "funasr[llm]"++from funasr import AutoModel++model = AutoModel(model="Qwen/Qwen-Audio-Chat",+ model_path=None,+ )++audio_in = "https://github.com/QwenLM/Qwen-Audio/raw/main/assets/audio/1272-128104-0000.flac"++# 1st dialogue turn+prompt = 'what does the person say?'+cache = {"history": None}+res = model.generate(input=audio_in, prompt=prompt, cache=cache)+print(res)++prompt = 'Find the start time and end time of the word "middle classes"'+# 2nd dialogue turn+res = model.generate(input=None, prompt=prompt, cache=cache)+print(res)+diff --git a/examples/industrial_data_pretraining/qwen_audio/demo_chat_from_local.py b/examples/industrial_data_pretraining/qwen_audio/demo_chat_from_local.pynew file mode 100644index 000000000..d48d9094f--- /dev/null+++ b/examples/industrial_data_pretraining/qwen_audio/demo_chat_from_local.py@@ -0,0 +1,26 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# To install requirements: pip3 install -U "funasr[llm]"++from funasr import AutoModel++model = AutoModel(model="Qwen/Qwen-Audio-Chat",+ model_path="/nfs/zhifu.gzf/init_model/qwen/Qwen-Audio-Chat",+ )++audio_in = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav"++# 1st dialogue turn+prompt = 'what does the person say?'+cache = {"history": None}+res = model.generate(input=audio_in, prompt=prompt, cache=cache)+print(res)++prompt = 'Find the start time and end time of the word "middle classes"'+# 2nd dialogue turn+res = model.generate(input=None, prompt=prompt, cache=cache)+print(res)+diff --git a/examples/industrial_data_pretraining/qwen_audio/demo_from_local.py b/examples/industrial_data_pretraining/qwen_audio/demo_from_local.pynew file mode 100644index 000000000..aa28dd24d--- /dev/null+++ b/examples/industrial_data_pretraining/qwen_audio/demo_from_local.py@@ -0,0 +1,15 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# To install requirements: pip3 install -U "funasr[llm]"++from funasr import AutoModel++model = AutoModel(model="Qwen/Qwen-Audio",+ model_path="/nfs/zhifu.gzf/init_model/qwen/Qwen-Audio",+ )++res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", language=None)+print(res)diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex 70d09dfc2..9ae9f180e 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -245,7 +245,10 @@ class AutoModel:time1 = time.perf_counter()with torch.no_grad():- results, meta_data = model.inference(**batch, **kwargs)+ res = model.inference(**batch, **kwargs)+ if isinstance(res, (list, tuple)):+ results = res[0]+ meta_data = res[1] if len(res) > 1 else {}time2 = time.perf_counter()asr_result_list.extend(results)diff --git a/funasr/download/download_from_hub.py b/funasr/download/download_from_hub.pyindex 4f0ae74c0..4d5a5346a 100644--- a/funasr/download/download_from_hub.py+++ b/funasr/download/download_from_hub.py@@ -13,10 +13,16 @@ def download_model(**kwargs):passelif hub == "openai":model_or_path = kwargs.get("model")- if model_or_path in name_maps_openai:- model_or_path = name_maps_openai[model_or_path]- kwargs["model_path"] = model_or_path-+ if os.path.exists(model_or_path):+ # local path+ kwargs["model_path"] = model_or_path+ kwargs["model"] = "WhisperWarp"+ else:+ # model name+ if model_or_path in name_maps_openai:+ model_or_path = name_maps_openai[model_or_path]+ kwargs["model_path"] = model_or_path+return kwargsdef download_from_ms(**kwargs):@@ -24,7 +30,7 @@ def download_from_ms(**kwargs):if model_or_path in name_maps_ms:model_or_path = name_maps_ms[model_or_path]model_revision = kwargs.get("model_revision")- if not os.path.exists(model_or_path):+ if not os.path.exists(model_or_path) and "model_path" not in kwargs:try:model_or_path = get_or_download_model_dir(model_or_path, model_revision,is_training=kwargs.get("is_training"),@@ -32,7 +38,7 @@ def download_from_ms(**kwargs):except Exception as e:print(f"Download: {model_or_path} failed!: {e}")- kwargs["model_path"] = model_or_path+ kwargs["model_path"] = model_or_path if "model_path" not in kwargs else kwargs["model_path"]if os.path.exists(os.path.join(model_or_path, "configuration.json")):with open(os.path.join(model_or_path, "configuration.json"), 'r', encoding='utf-8') as f:diff --git a/funasr/download/name_maps_from_hub.py b/funasr/download/name_maps_from_hub.pyindex 07cf6a095..5e252af6e 100644--- a/funasr/download/name_maps_from_hub.py+++ b/funasr/download/name_maps_from_hub.py@@ -10,6 +10,7 @@ name_maps_ms = {"cam++": "damo/speech_campplus_sv_zh-cn_16k-common","Whisper-large-v2": "iic/speech_whisper-large_asr_multilingual","Whisper-large-v3": "iic/Whisper-large-v3",+ "Qwen-Audio": "Qwen/Qwen-Audio",}name_maps_hf = {diff --git a/funasr/models/qwen_audio/model.py b/funasr/models/qwen_audio/model.pyindex 805234b2a..3eba026c2 100644--- a/funasr/models/qwen_audio/model.py+++ b/funasr/models/qwen_audio/model.py@@ -9,28 +9,32 @@ from torch import Tensorfrom torch import nnimport whisperfrom funasr.utils.load_utils import load_audio_text_image_video, extract_fbank+from transformers import AutoModelForCausalLM, AutoTokenizer+from transformers.generation import GenerationConfigfrom funasr.register import tables--[email protected]("model_classes", "Qwen/Qwen-Audio")[email protected]("model_classes", "Qwen-Audio")[email protected]("model_classes", "Qwen/QwenAudio")[email protected]("model_classes", "QwenAudio")@tables.register("model_classes", "QwenAudioWarp")-class WhisperWarp(nn.Module):- def __init__(self, whisper_dims: dict, **kwargs):+class QwenAudioWarp(nn.Module):+ def __init__(self, *args, **kwargs):super().__init__()- hub = kwargs.get("hub", "funasr")- if hub == "openai":- init_param_path = kwargs.get("init_param_path", "large-v3")- model = whisper.load_model(init_param_path)- else:- dims = whisper.model.ModelDimensions(**whisper_dims)- model = whisper.model.Whisper(dims=dims)++ model_or_path = kwargs.get("model_path", "QwenAudio")+ model = AutoModelForCausalLM.from_pretrained(model_or_path, device_map="cpu",+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_or_path, trust_remote_code=True)+self.model = model+ self.tokenizer = tokenizerdef forward(self, ):pass-+def inference(self,data_in,data_lengths=None,@@ -41,45 +45,84 @@ class WhisperWarp(nn.Module):):if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")+meta_data = {}- if isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank": # fbank- speech, speech_lengths = data_in, data_lengths- if len(speech.shape) < 3:- speech = speech[None, :, :]- if speech_lengths is None:- speech_lengths = speech.shape[1]- else:- # extract fbank feats- time1 = time.perf_counter()- audio_sample_list = load_audio_text_image_video(data_in, fs=frontend.fs, audio_fs=kwargs.get("fs", 16000),- data_type=kwargs.get("data_type", "sound"),- tokenizer=tokenizer)- time2 = time.perf_counter()- meta_data["load_data"] = f"{time2 - time1:0.3f}"- speech, speech_lengths = extract_fbank(audio_sample_list, data_type=kwargs.get("data_type", "sound"),- frontend=frontend)- time3 = time.perf_counter()- meta_data["extract_feat"] = f"{time3 - time2:0.3f}"- frame_shift = frontend.frame_shift if hasattr(frontend, "frame_shift") else 10- lfr_n = frontend.lfr_n if hasattr(frontend, "lfr_n") else 1- meta_data["batch_data_time"] = speech_lengths.sum().item() * frame_shift * lfr_n / 1000+ # meta_data["batch_data_time"] = -1- speech = speech.to(device=kwargs["device"])[0, :, :]- speech_lengths = speech_lengths.to(device=kwargs["device"])+ sp_prompt = "<|startoftranscription|><|en|><|transcribe|><|en|><|notimestamps|><|wo_itn|>"+ query = f"<audio>{data_in[0]}</audio>{sp_prompt}"+ audio_info = self.tokenizer.process_audio(query)+ inputs = self.tokenizer(query, return_tensors='pt', audio_info=audio_info)+ inputs = inputs.to(self.model.device)+ pred = self.model.generate(**inputs, audio_info=audio_info)+ response = tokenizer.decode(pred.cpu()[0], skip_special_tokens=False, audio_info=audio_info)- # detect the spoken language- _, probs = self.model.detect_language(speech)- print(f"Detected language: {max(probs, key=probs.get)}")+ results = []+ result_i = {"key": key[0], "text": response}++ results.append(result_i)++ return results, meta_data- # decode the audio- options = whisper.DecodingOptions(language=kwargs.get("language", None), fp16=False)- result = whisper.decode(self.model, speech, options)[email protected]("model_classes", "Qwen/Qwen-Audio-Chat")[email protected]("model_classes", "Qwen/QwenAudioChat")[email protected]("model_classes", "Qwen-Audio-Chat")[email protected]("model_classes", "QwenAudioChat")[email protected]("model_classes", "QwenAudioChatWarp")+class QwenAudioChatWarp(nn.Module):+ def __init__(self, *args, **kwargs):+ super().__init__()++ model_or_path = kwargs.get("model_path", "QwenAudio")+ bf16 = kwargs.get("bf16", False)+ fp16 = kwargs.get("fp16", False)+ model = AutoModelForCausalLM.from_pretrained(model_or_path,+ device_map="cpu",+ bf16=bf16,+ fp16=fp16,+ trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_or_path, trust_remote_code=True)++ self.model = model+ self.tokenizer = tokenizer++ def forward(self, ):+ pass++ def inference(self,+ data_in,+ data_lengths=None,+ key: list = None,+ tokenizer=None,+ frontend=None,+ **kwargs,+ ):+ if kwargs.get("batch_size", 1) > 1:+ raise NotImplementedError("batch decoding is not implemented")+++ meta_data = {}- results = []- result_i = {"key": key[0], "text": result.text}+ prompt = kwargs.get("prompt", "what does the person say?")+ cache = kwargs.get("cache", {})+ history = cache.get("history", None)+ if data_in[0] is not None:+ # 1st dialogue turn+ query = self.tokenizer.from_list_format([+ {'audio': data_in[0]}, # Either a local path or an url+ {'text': prompt},+ ])+ else:+ query = prompt+ response, history = self.model.chat(self.tokenizer, query=query, history=history)+ cache["history"] = history+ # print(response)+ # The person says: "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel".+ results = []+ result_i = {"key": key[0], "text": response}+results.append(result_i)-+return results, meta_data-\ No newline at end of filediff --git a/funasr/version.txt b/funasr/version.txtindex 2ac9634d3..5b09c67c2 100644--- a/funasr/version.txt+++ b/funasr/version.txt@@ -1 +1 @@-1.0.13+1.0.14diff --git a/setup.py b/setup.pyindex 4e76c80e1..e3d1c2ee5 100644--- a/setup.py+++ b/setup.py@@ -41,6 +41,7 @@ requirements = {"jaconv","hydra-core>=1.3.2","tensorboardX",+ "rotary_embedding_torch",],# train: The modules invoked when training only."train": [@@ -82,6 +83,17 @@ requirements = {"sphinx-markdown-tables>=0.0.12","configargparse>=1.2.1"],+ "llm":[+ "transformers>=4.32.0",+ "accelerate",+ "tiktoken",+ "einops",+ "transformers_stream_generator>=0.0.4",+ "scipy",+ "torchvision",+ "pillow",+ "matplotlib",+ ],}requirements["all"].extend(requirements["train"])requirements["test"].extend(requirements["train"])