FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.
Python ai-toolsai-video-editingasrauto-subtitleschinesecontent-creationfunasrfunclipgradiollmparaformerspeech-recognitionspeech-to-textsubtitles-generatortranscriptionvideo-editingvideo-processingvideo-subtitlesvideo-transcriptionwhisper-alternative
update to funasr1.0 call and hotword supported
shixian.shi51f1f89bce97
diff --git a/ClipVideo/clipvideo/gradio_service.py b/ClipVideo/clipvideo/gradio_service.pyindex 7e7e893..a4302c7 100644--- a/ClipVideo/clipvideo/gradio_service.py+++ b/ClipVideo/clipvideo/gradio_service.py@@ -23,8 +23,8 @@ if __name__ == "__main__":def audio_clip(dest_text, audio_spk_input, start_ost, end_ost, state):return audio_clipper.clip(dest_text, start_ost, end_ost, state, dest_spk=audio_spk_input)- def video_recog(video_input, sd_switch):- return audio_clipper.video_recog(video_input, sd_switch)+ def video_recog(video_input, sd_switch, hotwords):+ return audio_clipper.video_recog(video_input, sd_switch, hotwords)def video_clip(dest_text, video_spk_input, start_ost, end_ost, state):return audio_clipper.video_clip(dest_text, start_ost, end_ost, state, dest_spk=video_spk_input)@@ -85,6 +85,7 @@ if __name__ == "__main__":[video_input],label='说话人切分示例 SD Demo')video_sd_switch = gr.Radio(["no", "yes"], label="👥是否区分说话人 Recognize Speakers", value='no')+ hotwords_input = gr.Textbox(label="🚒热词 Hotwords")recog_button2 = gr.Button("👂识别 Recognize")video_text_output = gr.Textbox(label="✏️识别结果 Recognition Result")video_srt_output = gr.Textbox(label="📖SRT字幕内容 RST Subtitles")@@ -134,7 +135,7 @@ if __name__ == "__main__":outputs=[audio_output, audio_mess_output, audio_srt_clip_output])recog_button2.click(video_recog,- inputs=[video_input, video_sd_switch],+ inputs=[video_input, video_sd_switch, hotwords_input],outputs=[video_text_output, video_srt_output, video_state])clip_button2.click(video_clip,inputs=[video_text_input, video_spk_input, video_start_ost, video_end_ost, video_state],diff --git a/ClipVideo/clipvideo/trans_utils.py b/ClipVideo/clipvideo/trans_utils.pyindex 4cf6347..2485583 100644--- a/ClipVideo/clipvideo/trans_utils.py+++ b/ClipVideo/clipvideo/trans_utils.py@@ -39,7 +39,7 @@ def proc_spk(dest_spk, sd_sentences):for d in sd_sentences:d_start = d['timestamp'][0][0]d_end = d['timestamp'][-1][1]- spkid=dest_spk+ spkid=dest_spk[3:]# import pdb; pdb.set_trace()if str(d['spk']) == spkid and d_end-d_start>999:ts.append([d['start']*16, d['end']*16])diff --git a/ClipVideo/clipvideo/videoclipper.py b/ClipVideo/clipvideo/videoclipper.pyindex 4275d88..da412c5 100644--- a/ClipVideo/clipvideo/videoclipper.py+++ b/ClipVideo/clipvideo/videoclipper.py@@ -1,3 +1,4 @@+import reimport sysimport copyimport librosa@@ -6,22 +7,20 @@ import argparseimport numpy as npimport soundfile as sfimport moviepy.editor as mpy-# from modelscope.pipelines import pipeline-# from modelscope.utils.constant import Tasks-from subtitle_utils import generate_srt, generate_srt_clip# distribute_spk-from trans_utils import pre_proc, proc, write_state, load_state, proc_spk, generate_vad_data-from argparse_tools import ArgumentParser, get_commandline_args-from moviepy.editor import *from moviepy.video.tools.subtitles import SubtitlesClip+from subtitle_utils import generate_srt, generate_srt_clip+from argparse_tools import ArgumentParser, get_commandline_args+from trans_utils import pre_proc, proc, write_state, load_state, proc_spk, generate_vad_dataclass VideoClipper():def __init__(self, funasr_model):logging.warning("Initializing VideoClipper.")self.funasr_model = funasr_model+ self.GLOBAL_COUNT = 0- def recog(self, audio_input, sd_switch='no', state=None):+ def recog(self, audio_input, sd_switch='no', state=None, hotwords=""):if state is None:state = {}sr, data = audio_input@@ -32,18 +31,16 @@ class VideoClipper():state['audio_input'] = (sr, data)data = data.astype(np.float64)if sd_switch == 'yes':- rec_result = self.funasr_model.generate(data)+ rec_result = self.funasr_model.generate(data, return_raw_text=True, is_final=True, hotword=hotwords)res_srt = generate_srt(rec_result[0]['sentence_info'])state['sd_sentences'] = rec_result[0]['sentence_info']- '''- vad_data = generate_vad_data(data.astype(np.float32), rec_result['sentences'], sr)- sd_result = self.sd_pipeline(audio=vad_data, batch_size=1)- rec_result['sd_sentences'] = distribute_spk(rec_result['sentences'], sd_result['text'])- res_srt = generate_srt(rec_result['sd_sentences'])- state['sd_sentences'] = rec_result['sd_sentences']- '''else:- rec_result = self.funasr_model.generate(data, return_spk_res=False, sentence_timestamp=True)+ rec_result = self.funasr_model.generate(data,+ return_spk_res=False,+ sentence_timestamp=True,+ return_raw_text=True,+ is_final=True,+ hotword=hotwords)res_srt = generate_srt(rec_result[0]['sentence_info'])state['recog_res_raw'] = rec_result[0]['raw_text']state['timestamp'] = rec_result[0]['timestamp']@@ -63,6 +60,15 @@ class VideoClipper():all_ts = []if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:for _dest_text in dest_text.split('#'):+ if '[' in _dest_text:+ match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)+ if match:+ offset_b, offset_e = map(int, match.groups())+ log_append = ""+ else:+ offset_b, offset_e = 0, 0+ log_append = "(Bracket detected in dest_text but offset time matching failed)"+ _dest_text = _dest_text[:_dest_text.find('[')]_dest_text = pre_proc(_dest_text)ts = proc(recog_res_raw, timestamp, _dest_text)for _ts in ts: all_ts.append(_ts)@@ -91,13 +97,13 @@ class VideoClipper():srt_clip, _, srt_index = generate_srt_clip(sentences, start/16000.0, end/16000.0, begin_index=srt_index-1)clip_srt += srt_clipif len(ts):- message = "{} periods found in the speech: ".format(len(ts)) + start_end_info+ message = "{} periods found in the speech: ".format(len(ts)) + start_end_info + log_appendelse:message = "No period found in the speech, return raw speech. You may check the recognition result and try other destination text."res_audio = datareturn (sr, res_audio), message, clip_srt- def video_recog(self, vedio_filename, sd_switch='no'):+ def video_recog(self, vedio_filename, sd_switch='no', hotwords=""):vedio_filename = vedio_filenameclip_video_file = vedio_filename[:-4] + '_clip.mp4'video = mpy.VideoFileClip(vedio_filename)@@ -110,7 +116,7 @@ class VideoClipper():'video': video,}# res_text, res_srt = self.recog((16000, wav), state)- return self.recog((16000, wav), sd_switch, state)+ return self.recog((16000, wav), sd_switch, state, hotwords)def video_clip(self, dest_text, start_ost, end_ost, state, font_size=32, font_color='white', add_sub=False, dest_spk=None):# get from state@@ -125,9 +131,24 @@ class VideoClipper():srt_index = 0if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:for _dest_text in dest_text.split('#'):+ if '[' in _dest_text:+ match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)+ if match:+ offset_b, offset_e = map(int, match.groups())+ log_append = ""+ else:+ offset_b, offset_e = 0, 0+ log_append = "(Bracket detected in dest_text but offset time matching failed)"+ _dest_text = _dest_text[:_dest_text.find('[')]+ else:+ offset_b, offset_e = 0, 0+ log_append = ""_dest_text = pre_proc(_dest_text)ts = proc(recog_res_raw, timestamp, _dest_text)- for _ts in ts: all_ts.append(_ts)+ for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])+ if len(ts) > 1 and match:+ log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \+ offsets are applied to all periods)'else:for _dest_spk in dest_spk.split('#'):ts = proc_spk(_dest_spk, state['sd_sentences'])@@ -167,11 +188,13 @@ class VideoClipper():# _video_clip.write_videofile("debug.mp4", audio_codec="aac")concate_clip.append(copy.copy(_video_clip))time_acc_ost += end+end_ost/1000.0 - (start+start_ost/1000.0)- message = "{} periods found in the audio: ".format(len(ts)) + start_end_info+ message = "{} periods found in the audio: ".format(len(ts)) + start_end_info + log_appendlogging.warning("Concating...")if len(concate_clip) > 1:video_clip = concatenate_videoclips(concate_clip)- video_clip.write_videofile(clip_video_file, audio_codec="aac")+ clip_video_file = clip_video_file[:-4] + '_no{}.mp4'.format(self.GLOBAL_COUNT)+ video_clip.write_videofile(clip_video_file, audio_codec="aac", temp_audiofile="video_no{}.mp4".format(self.GLOBAL_COUNT))+ self.GLOBAL_COUNT += 1else:clip_video_file = vedio_filenamemessage = "No period found in the audio, return raw speech. You may check the recognition result and try other destination text."@@ -256,23 +279,19 @@ def runner(stage, file, sd_switch, output_dir, dest_text, dest_spk, start_ost, ewhile output_dir.endswith('/'):output_dir = output_dir[:-1]if stage == 1:- from modelscope.pipelines import pipeline- from modelscope.utils.constant import Tasks- # initialize modelscope asr pipeline+ from funasr import AutoModel+ # initialize funasr automodellogging.warning("Initializing modelscope asr pipeline.")- inference_pipeline = pipeline(- task=Tasks.auto_speech_recognition,- model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',- vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',- punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',- output_dir=output_dir,- )- sd_pipeline = pipeline(- task='speaker-diarization',- model='damo/speech_campplus_speaker-diarization_common',- model_revision='v1.0.0'- )- audio_clipper = VideoClipper(inference_pipeline, sd_pipeline)+ funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+ model_revision="v2.0.4",+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ vad_model_revision="v2.0.4",+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ punc_model_revision="v2.0.4",+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ spk_model_revision="v2.0.2",+ )+ audio_clipper = VideoClipper(funasr_model)if mode == 'audio':logging.warning("Recognizing audio file: {}".format(file))wav, sr = librosa.load(file, sr=16000)diff --git a/README.md b/README.mdindex 9e4df0b..5f86273 100644--- a/README.md+++ b/README.md@@ -4,8 +4,9 @@## What's New-- 10/17 Bug fix for multiple periods chosen, used to return video with wrong length.-- 10/10 ClipVideo now supports recognizing with speaker diarization ability, choose 'yes' button in 'Recognize Speakers' and you will get recognition results with speaker id for each sentence. And then you can clip out the periods of one or some speakers (e.g. 'spk0' or 'spk0#spk3') using ClipVideo.+- 2024/02/28 Update call function of funasr into funasr1.0, use SeACo_Paraformer thus ASR now supports hotword. Also support destination transcription like 'abcd[-100,150]#efgh[200,200]' to adjust offset time for every sub-sentence (TODO: use ClipVideo with command line is not updated currently).+- 2023/10/17 Bug fix for multiple periods chosen, used to return video with wrong length.+- 2023/10/10 ClipVideo now supports recognizing with speaker diarization ability, choose 'yes' button in 'Recognize Speakers' and you will get recognition results with speaker id for each sentence. And then you can clip out the periods of one or some speakers (e.g. 'spk0' or 'spk0#spk3') using ClipVideo.## ClipVideo