Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
add fun-asr dataset (#2758)
彭震东7bfce74f291d
diff --git a/funasr/datasets/fun_asr_datasets/__init__.py b/funasr/datasets/fun_asr_datasets/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/funasr/datasets/fun_asr_datasets/datasets.py b/funasr/datasets/fun_asr_datasets/datasets.pynew file mode 100644index 000000000..09bbfa300--- /dev/null+++ b/funasr/datasets/fun_asr_datasets/datasets.py@@ -0,0 +1,508 @@+import json+import logging++import re+import torch+import random+import traceback+import numpy as np+from funasr.register import tables+from funasr.utils.load_utils import extract_fbank, load_audio_text_image_video++[email protected]("dataset_classes", "FunASR")+class FunASR(torch.utils.data.Dataset):+ """+ FunASR dataset+ """++ def __init__(+ self,+ path,+ index_ds: str = None,+ frontend=None,+ tokenizer=None,+ int_pad_value: int = -1,+ float_pad_value: float = 0.0,+ **kwargs,+ ):+ super().__init__()+ index_ds_class = tables.index_ds_classes.get(index_ds)+ self.index_ds = index_ds_class(path, **kwargs)+ preprocessor_speech = kwargs.get("preprocessor_speech", None)+ if preprocessor_speech:+ preprocessor_speech_class = tables.preprocessor_classes.get(preprocessor_speech)+ preprocessor_speech = preprocessor_speech_class(+ **kwargs.get("preprocessor_speech_conf")+ )+ self.preprocessor_speech = preprocessor_speech++ preprocessor_noise = kwargs.get("preprocessor_noise", None)+ if preprocessor_noise:+ preprocessor_noise_class = tables.preprocessor_classes.get(preprocessor_noise)+ preprocessor_noise = preprocessor_noise_class(**kwargs.get("preprocessor_noise_conf"))+ self.preprocessor_noise = preprocessor_noise++ prompt_classes_text = kwargs.get("prompt_classes", None)+ if prompt_classes_text is not None:+ prompt_classes = tables.prompt_classes.get(prompt_classes_text)+ prompt_classes = prompt_classes(**kwargs.get("prompt_conf"))+ else:+ prompt_classes = None+ self.prompt_classes = prompt_classes++ self.frontend = frontend+ self.fs = 16000 if frontend is None else frontend.fs+ self.data_type = "sound"+ self.tokenizer = tokenizer++ self.int_pad_value = int_pad_value+ self.float_pad_value = float_pad_value+ self.sos = kwargs.get("sos", "<|startoftranscript|>")+ self.eos = kwargs.get("eos", "<|endoftext|>")+ self.batch_size = kwargs.get("batch_size")+ self.batch_type = kwargs.get("batch_type")+ self.prompt_ids_len = 0+ self.retry = kwargs.get("retry", 100)++ self.pattern = re.compile(r"(<\|startofspeech\|>.*?<\|endofspeech\|>)")+ # self.kwargs = kwargs+ self.max_token_length = kwargs.get("max_token_length", 1500)+ self.batch_size_scale_ratio_max = kwargs.get("batch_size_scale_ratio_max", 1.5)+ self.batch_size_token_max = kwargs.get("batch_size_token_max", 2500)+ self.multiturn_num_max = kwargs.get("multiturn_num_max", 5)+ self.max_source_length = kwargs.get("max_source_length", 3000)+ self.max_target_length = kwargs.get("max_target_length", 1024)+ self.do_think = kwargs.get("do_think", True)+ self.sys_prompt = kwargs.get("sys_prompt", True)++ # used for dynamic output alignment+ self.use_dynamic_output_ratio = kwargs.get("use_dynamic_output_ratio", 0.0)+ self.min_output_mask_token_len = kwargs.get("min_mask_token_len", 1)+ self.min_output_non_mask_token_len = kwargs.get("min_non_mask_token_len", 6) # [eos]++ def get_source_len(self, index):+ item = self.index_ds[index]+ return self.index_ds.get_source_len(item)++ def get_target_len(self, index):+ item = self.index_ds[index]+ return self.index_ds.get_target_len(item)++ def get_random_user_prompt(self, item, user_prompt):+ tasks = ["语音转写:", "Speech transcription:"]+ language = item.get("language", None)+ # LID in distill data is fake+ language = None+ if language is not None:+ if language.lower() == "zh":+ tasks.append("语音转写成中文:")+ tasks.append("Transcribe speech into Chinese:")+ elif language.lower() == "en":+ tasks.append("语音转写成英文:")+ tasks.append("Transcribe speech into English:")+ if len(tasks) == 2:+ task = random.choice(tasks)+ elif len(tasks) == 4:+ task = random.choices(tasks, weights=[0.4, 0.4, 0.1, 0.1])[0]+ if "语音转写:<|startofspeech|>" in user_prompt:+ user_prompt = user_prompt.replace("语音转写:<|startofspeech|>", task + "<|startofspeech|>")+ elif "Speech transcription:<|startofspeech|>" in user_prompt:+ user_prompt = user_prompt.replace("Speech transcription:<|startofspeech|>", task + "<|startofspeech|>")+ return user_prompt++ def __len__(self):+ return len(self.index_ds)++ def __getitem__(self, index):+ output = None++ for idx in range(self.retry):+ if idx > 0:+ logging.info(f"retry: {idx}")+ badcase_flag = False+ if idx == 0:+ index_cur = index+ else:+ index_cur = torch.randint(0, len(self.index_ds), ()).item()++ item = self.index_ds[index_cur]++ system = item["system"]+ user = item["user"]+ assistant = item["assistant"]+ is_noised = item.get("noised", False)+ if len(user) < 1 or len(assistant) < 1:+ logging.warning(f"item is error: {item}")+ continue+ input_ids, labels, fbank, fbank_lens, fbank_mask, fbank_beg, fake_token_len = (+ [],+ [],+ [],+ [],+ [],+ [],+ [],+ )++ for i, (system_prompt, user_prompt, target_out) in enumerate(+ zip(system, user, assistant)+ ):+ if i >= self.multiturn_num_max:+ break+ if len(input_ids) > self.max_token_length:+ logging.info(+ f"input_ids > max_token_length: {len(input_ids)}>{self.max_token_length}, {item}"+ )+ break++ if self.prompt_classes is not None:+ asr_prompt = user_prompt.split("<|startofspeech|>")[0]+ language = self.prompt_classes.detect_language(asr_prompt)+ user_prompt_all_context = self.prompt_classes.get_prompt(item, language)+ else:+ user_prompt_all_context = ""++ if i == 0:+ source_input = f"<|im_start|>system\n{system_prompt}<|im_end|>\n<|im_start|>user\n{user_prompt_all_context}{user_prompt}<|im_end|>\n<|im_start|>assistant\n"+ if not self.sys_prompt:+ source_input = f"<|im_start|>user\n{user_prompt_all_context}{user_prompt}<|im_end|>\n<|im_start|>assistant\n"+ else:+ source_input = (+ f"<|im_start|>user\n{user_prompt}<|im_end|>\n<|im_start|>assistant\n"+ )+ if not self.do_think:+ source_input += "<think>\n\n</think>\n\n"+ splits = self.pattern.split(source_input)+ source_ids = []+ fbank_i = []+ fake_token_len_i = 0+ fbank_beg_i = -1+ fbank_lens_i = []+ speech = []+ speech_lengths = []+ for k, sub_str in enumerate(splits):+ if not sub_str.startswith("<|startofspeech|>"):+ sub_token = self.tokenizer.encode(sub_str)+ source_ids += sub_token+ else:+ sub_str = sub_str.replace("<|startofspeech|>", "").replace(+ "<|endofspeech|>", ""+ )+ if sub_str.startswith("!"):+ try:+ data_src = load_audio_text_image_video(sub_str[1:], fs=self.fs)+ if self.preprocessor_noise is not None and not is_noised:+ try:+ data_src = self.preprocessor_noise(data_src.numpy())+ except Exception as e:+ logging.error(f"Generate noise audio failed: {e}")++ speech, speech_lengths = extract_fbank(+ data_src,+ data_type=self.data_type,+ frontend=self.frontend,+ is_final=True,+ ) # speech: [b, T, d]+ if speech_lengths > self.max_source_length:+ logging.info(+ f"speech_lengths > max_source_length: {speech_lengths}>{self.max_source_length}, {item}"+ )+ badcase_flag = True+ except Exception as e:+ logging.warning(+ f"Loading wav failed! {str(e)}, {traceback.format_exc()}\n{item}"+ )+ badcase_flag = True+ continue+ if True:+ olens = 1 + (speech_lengths[0].item() - 3 + 2 * 1) // 2+ olens = 1 + (olens - 3 + 2 * 1) // 2+ fake_token_len_i = (olens - 1) // 2 + 1+ else:+ fake_token_len_i = speech_lengths[0].item()+ fake_token = [0] * fake_token_len_i+ fbank_beg_i = len(source_ids)+ source_ids += fake_token++ if badcase_flag:+ continue+ if fbank_beg_i > 0:+ fbank_beg += [fbank_beg_i + len(input_ids)]+ fake_token_len += [fake_token_len_i]+ else:+ fbank_beg += [-1]+ fake_token_len += [0]++ if target_out is not None and any(+ isinstance(item, dict) and "prev_content" in item for item in target_out+ ):+ prev_value = next(+ (+ item["prev_content"]+ for item in target_out+ if isinstance(item, dict) and "prev_content" in item+ ),+ None,+ )+ source_ids += self.tokenizer.encode(prev_value)+ source_mask = [-100] * len(source_ids)+ target_out = f"{target_out[0]}<|im_end|>"+ else:+ source_mask = [-100] * len(source_ids)+ target_out = f"{target_out}<|im_end|>"+ target_ids = self.tokenizer.encode(target_out)++ if len(target_ids) > self.max_target_length:+ logging.info(+ f"text_length: {len(target_ids)} > {self.max_target_length}, drop it: {item}"+ )+ # simulate prev-token fixed output+ if np.random.rand() < self.use_dynamic_output_ratio:+ max_len = len(target_ids)+ min_output_mask_token_len = min(self.min_output_mask_token_len, max_len)+ min_output_non_mask_token_len = min(self.min_output_non_mask_token_len, max_len)+ if max_len - min_output_non_mask_token_len > min_output_mask_token_len:+ end_index = np.random.randint(min_output_mask_token_len,+ max_len - min_output_non_mask_token_len)+ else:+ end_index = max_len - min_output_non_mask_token_len+ if end_index > 0:+ target_ids[:end_index] = [-100] * end_index++ input_ids += source_ids + target_ids+ labels += source_mask + target_ids+ if len(speech) > 0:+ fbank.append(speech[0, :, :])+ fbank_lens.append(speech_lengths)+ if badcase_flag:+ continue++ input_ids = torch.tensor(input_ids, dtype=torch.int64) # [: self.max_token_length]+ attention_mask = torch.tensor([1] * len(input_ids), dtype=torch.int32)+ labels = torch.tensor(labels, dtype=torch.int64) # [: self.max_token_length]++ fbank_beg = torch.tensor(fbank_beg, dtype=torch.int32)+ fake_token_len = torch.tensor(fake_token_len, dtype=torch.int32)++ output = {+ "fbank_beg": fbank_beg,+ "fake_token_len": fake_token_len,+ "input_ids": input_ids,+ "attention_mask": attention_mask,+ "labels_ids": labels,+ }+ output["item"] = item+ if len(fbank) > 0:+ output["speech"] = fbank+ output["speech_lengths"] = fbank_lens+ if len(input_ids) > self.max_token_length:+ logging.warning(+ f"len(input_ids): {len(input_ids)} > max_token_length: {self.max_token_length}, item: {item}"+ )+ continue++ break++ return output++ def collator(self, samples: list = None):++ for idx in range(self.retry):+ badcase_flag = False++ outputs = {}+ for sample in samples:+ if sample is None:+ continue+ for key in sample.keys():+ if key not in outputs:+ outputs[key] = []+ if isinstance(sample[key], (list, tuple)):+ outputs[key].extend(sample[key])+ else:+ outputs[key].append(sample[key])++ for key, data_list in outputs.items():+ if isinstance(data_list[0], torch.Tensor):+ if data_list[0].dtype == torch.int64 or data_list[0].dtype == torch.int32:++ pad_value = self.int_pad_value+ else:+ pad_value = self.float_pad_value++ outputs[key] = torch.nn.utils.rnn.pad_sequence(+ data_list, batch_first=True, padding_value=pad_value+ )++ if self.batch_type != "example":+ b, t = outputs["input_ids"].shape+ if b > 1 and b * t > self.batch_size_token_max:+ logging.info(+ f"Warning, {idx}th, b*t: {b}*{t}={b * t} > batch_size_sample_max: {self.batch_size_token_max}, drop last data"+ )+ samples = samples[:-1]+ continue++ break++ return outputs++[email protected]("index_ds_classes", "FunASR")+class FunASR(torch.utils.data.Dataset): # torch.utils.data.Dataset++ def __init__(self, path: str, **kwargs):+ super().__init__()++ self.max_source_length = kwargs.get("max_source_length", 8000)+ self.min_source_length = kwargs.get("min_source_length", 10)+ self.max_target_length = kwargs.get("max_target_length", 2048)+ self.min_target_length = kwargs.get("min_target_length", 0)+ # self.max_token_length = kwargs.get("max_token_length", 2200)++ audio_downsample_rate = int(kwargs.get("audio_downsample_rate", 8))++ is_training = kwargs.get("is_training", True)+ if not (path.endswith(".jsonl") or path.endswith(".json")):+ # jsonl list file+ data_split_num = kwargs.get("data_split_num", 1)+ data_split_i = kwargs.get("data_split_i", 0)++ if not is_training:+ data_split_num = 1+ data_split_i = 0+ with open(path, encoding="utf-8") as fin:+ file_list_all = fin.readlines()++ num_per_slice = (len(file_list_all) - 1) // data_split_num + 1 # 16+ file_list = file_list_all[+ data_split_i * num_per_slice: (data_split_i + 1) * num_per_slice+ ]+ logging.info(+ f"is_training: {is_training}, data_split_num: {data_split_num}, data_split_i: {data_split_i}, \nfile_list: {file_list}, \nfile_list_all: {file_list_all}"+ )++ else:+ file_list = [path]++ contents = []+ total_whrs = 0.0+ total_token_for_llm_B = 0.0+ for file_json in file_list:+ with open(file_json.strip(), encoding="utf-8") as fin:+ for line in fin:+ try:+ data_dict = json.loads(line.strip())+ except Exception as e:+ logging.error(+ f"drop it, json error: {e}, line: {line}, file_json: {file_json}"+ )+ continue++ data = data_dict["messages"]+ if isinstance(data_dict.get("speech_length", 0), (list, tuple)):+ speech_length = int(data_dict.get("speech_length", [0])[0])+ text_length = int(data_dict.get("text_length", 0)[0])+ else:+ speech_length = int(data_dict.get("speech_length", 0))+ text_length = int(data_dict.get("text_length", 0))+ speech_length = int(speech_length)+ text_length = int(text_length)+ if speech_length > 0 and speech_length < 1:+ continue+ if text_length < 1:+ logging.warning(+ f"speech_length: {speech_length}, text_length: {text_length}, data: {data}, file_json: {file_json}"+ )+ if len(data) > 2:+ text_length = len(data[2]['content'])+ continue+ if speech_length > self.max_source_length:+ continue+ if speech_length < self.min_source_length:+ continue+ if text_length > self.max_target_length:+ continue++ system, user, assistant = [], [], []+ for i, item in enumerate(data):+ try:+ role = item["role"]+ content = item["content"]+ except KeyError:+ logging.error(+ f"drop it, KeyError: {item}, file_json: {file_json}"+ )+ continue++ if role == "system":+ system.append(content)+ elif role == "user":+ user.append(content)+ elif role == "assistant":+ if "prev_content" in item:+ prev_content = item["prev_content"]+ assistant.append([content, {"prev_content": prev_content}])+ else:+ assistant.append(content)+ if len(system) == 0:+ system = ["You are a helpful assistant."]+ system = system * len(user)++ contents_i = {+ "system": system,+ "user": user,+ "assistant": assistant,+ "source_len": speech_length + text_length,+ }+ if "key" in data_dict:+ contents_i["key"] = data_dict["key"] if not isinstance(data_dict.get("key", "key_01234"),+ (list, tuple)) else data_dict["key"][0]++ if "hist_context" in data_dict:+ contents_i["hist_context"] = data_dict["hist_context"]+ if "hotwords" in data_dict:+ contents_i["hotwords"] = data_dict["hotwords"]+ if "asr_hotwords" in data_dict:+ contents_i["asr_hotwords"] = data_dict["asr_hotwords"]+ if "vad_segs" in data_dict:+ contents_i["vad_segs"] = data_dict["vad_segs"]+ if "word_list" in data_dict:+ contents_i["word_list"] = data_dict["word_list"]+ if "one_pass_result" in data_dict:+ contents_i["one_pass_result"] = data_dict["one_pass_result"]+ if "one_pass_wer" in data_dict:+ contents_i["one_pass_wer"] = data_dict["one_pass_wer"]+ if "noised" in data_dict:+ contents_i["noised"] = data_dict["noised"]++ if kwargs.get("save_meta", False):+ contents_i["meta"] = data_dict++ total_whrs += speech_length / 100.0 / 3600 / 10000 * audio_downsample_rate+ total_token_for_llm_B += (text_length + speech_length / 8) / 1000 / 1000 / 1000+ contents.append(contents_i)++ self.contents = contents++ logging.info(+ f"\n\ntotal_num of samplers: {len(self.contents)}, total_whrs: {total_whrs:.5f}, total_token_for_llm_B: {total_token_for_llm_B:.5g}, {path}, {file_list}\n\n")++ def __len__(self):+ return len(self.contents)++ def __getitem__(self, index):++ data = self.contents[index]++ return data++ def get_source_len(self, data_dict):+ source_len = data_dict.get("source_len", -1)+ if source_len < 0:+ source_len = len(data_dict["system"]) + len(data_dict["user"])+ return source_len++ def get_target_len(self, data_dict):++ return 0diff --git a/funasr/datasets/fun_asr_datasets/multicontext_prompt.py b/funasr/datasets/fun_asr_datasets/multicontext_prompt.pynew file mode 100644index 000000000..6d46dceb2--- /dev/null+++ b/funasr/datasets/fun_asr_datasets/multicontext_prompt.py@@ -0,0 +1,307 @@+import numpy as np+from funasr.register import tables+import logging+import random+import re++[email protected]("prompt_classes", "MultiContextPrompt")+class MultiContextPrompt:+ CONTEXT_TEMPLATES = {+ 'en': {+ 'header': "Please combine the context information provided below to complete the speech transcription task more accurately. If there is no relevant information, we will leave it blank.\n",+ 'fields': {+ 'hist_context': "Historical transcription: {hist_context}\n",+ 'one_pass_result': "One-pass result: {one_pass_result}\n",+ 'hotwords': "Hotword list: {hotwords}\n"+ }+ },+ 'zh': {+ 'header': "请结合下面提供的上下文信息,更加准确地完成语音转写任务。如果没有相关信息,我们会留空。\n",+ 'fields': {+ 'hist_context': "历史转写结果:{hist_context}\n",+ 'one_pass_result': "一遍解码结果:{one_pass_result}\n",+ 'hotwords': "热词列表:{hotwords}\n"+ }+ }+ }++ def __init__(self,+ use_hist=True,+ use_one_pass_result=True,+ use_hotwords=True,+ use_asr_hotwords=True,+ use_multi_lingual_prompt=True,+ **kwargs):+ self.use_hist = use_hist+ self.use_one_pass_result = use_one_pass_result+ self.use_hotwords = use_hotwords+ self.use_asr_hotwords = use_asr_hotwords+ self.use_multi_lingual_prompt = use_multi_lingual_prompt+ self.kwargs = kwargs++ chinese_hotwords_list = kwargs.get("chinese_hotwords_list", "")+ english_hotwords_list = kwargs.get("english_hotwords_list", "")+ if chinese_hotwords_list:+ self.chinese_hotwords_list, self.chinese_hotwords_num = self.get_hotwords_list(chinese_hotwords_list)+ else:+ self.chinese_hotwords_list = None+ self.chinese_hotwords_num = 0+ logging.info(f"chinese_hotwords_num: {self.chinese_hotwords_num}")++ if english_hotwords_list:+ self.english_hotwords_list, self.english_hotwords_num = self.get_hotwords_list(english_hotwords_list)+ else:+ self.english_hotwords_list = None+ self.english_hotwords_num = 0+ logging.info(f"english_hotwords_num: {self.english_hotwords_num}")++ self.max_neg_hotwords_num = kwargs.get("max_neg_hotwords_num", 900)+ self.min_neg_hotwords_num = kwargs.get("min_neg_hotwords_num", 0)++ def get_hotwords_list(self, hotwords_file):+ with open(hotwords_file, "r") as f:+ hotwords_list = f.read().strip().split("\n")+ return hotwords_list, len(hotwords_list)++ def detect_language(self, text):+ if isinstance(text, list):+ text = " ".join(text)++ chinese_pattern = re.compile(+ "["+ "\u4e00-\u9fff" # CJK Unified Ideographs+ "]+"+ )++ english_pattern = re.compile(r'[A-Za-z]+')++ chinese_matches = chinese_pattern.findall(text)+ english_matches = english_pattern.findall(text)++ chinese_length = sum(len(match) for match in chinese_matches)+ english_length = sum(len(match) for match in english_matches)++ total_length = len(text)++ if total_length == 0:+ return 'zh'++ if (chinese_length > english_length) and (chinese_length / total_length > 0.3):+ return 'zh'+ else:+ return 'en'++ def hotwords_sampling(self, hotwords):++ # hotwords_list = hotwords.split(", ")+ hotwords_list = hotwords+ selected_hotwords = []+ if self.max_neg_hotwords_num > -1:+ max_neg_hotwords_num = min(self.max_neg_hotwords_num, len(hotwords_list))+ else:+ max_neg_hotwords_num = len(hotwords_list)++ if self.min_neg_hotwords_num < max_neg_hotwords_num:+ selected_hotwords_num = np.random.randint(self.min_neg_hotwords_num, max_neg_hotwords_num + 1)+ else:+ selected_hotwords_num = max_neg_hotwords_num+ if selected_hotwords_num > 0:+ selected_hotwords = np.random.choice(hotwords_list, selected_hotwords_num, replace=False).tolist()++ return selected_hotwords, selected_hotwords_num++ def get_prompt(self, item, language):+ template = self.CONTEXT_TEMPLATES[language]++ prompt = template['header']++ context_lines = []++ if self.use_hist and item.get("hist_context"):+ context_lines.append(template['fields']['hist_context'].format(hist_context=item["hist_context"]))++ if self.use_one_pass_result and item.get("one_pass_result"):+ context_lines.append(template['fields']['one_pass_result'].format(one_pass_result=item["one_pass_result"]))++ hotwords = None+ if self.use_hotwords and item.get("hotwords"):+ hotwords = item["hotwords"]+ if self.use_asr_hotwords and item.get("asr_hotwords"):+ hotwords = item["asr_hotwords"]+ if hotwords is not None and hotwords != "":+ language = self.detect_language(hotwords)+ if language == 'en':+ neg_hotwords = self.english_hotwords_list+ else:+ neg_hotwords = self.chinese_hotwords_list+ if neg_hotwords is not None:+ selected_neg_hotwords, selected_neg_hotwords_num = self.hotwords_sampling(neg_hotwords)+ else:+ selected_neg_hotwords = []++ if not isinstance(hotwords, list):+ pos_hotwords = hotwords.split(", ")+ else:+ pos_hotwords = hotwords+ hotwords = pos_hotwords + selected_neg_hotwords+ random.shuffle(hotwords)+ hotwords = ", ".join(hotwords)+ context_lines.append(template['fields']['hotwords'].format(hotwords=hotwords))++ if context_lines:+ prompt += ''.join(context_lines)+ else:+ prompt += "\n\n\n"++ return prompt++ def get_inference_prompt(self, item, language="zh"):+ template = self.CONTEXT_TEMPLATES[language]++ prompt = template['header']++ context_lines = []++ if self.use_hist and item.get("hist_context"):+ context_lines.append(template['fields']['hist_context'].format(hist_context=item["hist_context"]))++ if self.use_one_pass_result and item.get("one_pass_result"):+ context_lines.append(template['fields']['one_pass_result'].format(one_pass_result=item["one_pass_result"]))++ hotwords = None+ if self.use_hotwords and item.get("hotwords"):+ hotwords = item["hotwords"]+ if self.use_asr_hotwords and item.get("asr_hotwords"):+ hotwords = item["asr_hotwords"]+ if hotwords is not None and hotwords != "":+ print(f"hotwords: {hotwords}")+ language = self.detect_language(hotwords)+ if language == 'en':+ neg_hotwords = self.english_hotwords_list+ else:+ neg_hotwords = self.chinese_hotwords_list+ if neg_hotwords is not None:+ selected_neg_hotwords, selected_neg_hotwords_num = self.hotwords_sampling(neg_hotwords)+ else:+ selected_neg_hotwords = []++ if not isinstance(hotwords, list):+ pos_hotwords = hotwords.split(", ")+ else:+ pos_hotwords = hotwords+ hotwords = pos_hotwords + selected_neg_hotwords+ print(f"selected_neg_hotwords_num: {selected_neg_hotwords_num}")+ random.shuffle(hotwords)+ hotwords = ", ".join(hotwords)+ context_lines.append(template['fields']['hotwords'].format(hotwords=hotwords))++ if context_lines:+ prompt += ''.join(context_lines)+ else:+ prompt += "\n\n\n"++ return prompt++[email protected]("prompt_classes", "MultiContextPromptNew")+class MultiContextPromptNew:+ CONTEXT_TEMPLATES = {+ 'en': {+ 'header': "Please combine the context information to complete the speech transcription task more accurately. If there is no relevant information, we will leave it blank.\n\n",+ 'context_header': "**Context:**\n",+ 'fields': {+ 'hist_context': "Historical transcription: {hist_context}\n",+ 'one_pass_result': "One-pass result: {one_pass_result}\n",+ 'hotwords': "Hotword list: {hotwords}\n"+ }+ },+ 'zh': {+ 'header': "请结合上下文信息,更加准确地完成语音转写任务。如果没有相关信息,我们会留空。\n\n",+ 'context_header': "**上下文信息:**\n",+ 'fields': {+ 'hist_context': "历史转写结果:{hist_context}\n",+ 'one_pass_result': "一遍解码结果:{one_pass_result}\n",+ 'hotwords': "热词列表:{hotwords}\n"+ }+ }+ }++ def __init__(self,+ use_hist=True,+ use_one_pass_result=True,+ use_hotwords=True,+ use_multi_lingual_prompt=True,+ **kwargs):+ self.use_hist = use_hist+ self.use_one_pass_result = use_one_pass_result+ self.use_hotwords = use_hotwords+ self.use_multi_lingual_prompt = use_multi_lingual_prompt++ self.use_full_hotwords_ratio = kwargs.get("use_full_hotwords_ratio", 0.2)+ self.max_hotwords_num = kwargs.get("max_hotwords_num", -1)+ self.min_hotwords_num = kwargs.get("min_hotwords_num", 15)++ def hotwords_sampling(self, hotwords):++ hotwords_list = hotwords.split(", ")+ if self.max_hotwords_num > 0:+ max_hotwords_num = min(self.max_hotwords_num, len(hotwords_list))+ else:+ max_hotwords_num = len(hotwords_list)++ if self.min_hotwords_num < max_hotwords_num:+ selected_hotwords_num = np.random.randint(self.min_hotwords_num, max_hotwords_num + 1)+ else:+ selected_hotwords_num = max_hotwords_num++ selected_hotwords = np.random.choice(hotwords_list, selected_hotwords_num, replace=False)+ hotwords_list = ", ".join(selected_hotwords)++ return hotwords_list, selected_hotwords_num++ def get_prompt(self, item, language):+ template = self.CONTEXT_TEMPLATES[language]++ prompt = template['header']++ context_lines = []++ if self.use_hist and item.get("hist_context"):+ context_lines.append(template['fields']['hist_context'].format(hist_context=item["hist_context"]))++ if self.use_one_pass_result and item.get("one_pass_result"):+ context_lines.append(template['fields']['one_pass_result'].format(one_pass_result=item["one_pass_result"]))++ if self.use_hotwords and item.get("hotwords"):+ hotwords = item["hotwords"]+ if np.random.rand() < self.use_full_hotwords_ratio:+ hotwords = hotwords+ else:+ hotwords, selected_hotwords_num = self.hotwords_sampling(hotwords)+ context_lines.append(template['fields']['hotwords'].format(hotwords=hotwords))++ if context_lines:+ prompt += template['context_header'] + ''.join(context_lines)++ return prompt++ def get_inference_prompt(self, hist_context="", one_pass_result="", hotwords=""):+ language = 'zh' if self.use_multi_lingual_prompt and np.random.rand() < 0.5 else 'en'+ template = self.CONTEXT_TEMPLATES[language]++ prompt = template['header']++ context_lines = []++ if hist_context:+ context_lines.append(template['fields']['hist_context'].format(hist_context=hist_context))+ if one_pass_result:+ context_lines.append(template['fields']['one_pass_result'].format(one_pass_result=one_pass_result))+ if hotwords:+ context_lines.append(template['fields']['hotwords'].format(hotwords=hotwords))++ if context_lines:+ prompt += template['context_header'] + ''.join(context_lines)++ return prompt