FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.
Python ai-toolsai-video-editingasrauto-subtitleschinesecontent-creationfunasrfunclipgradiollmparaformerspeech-recognitionspeech-to-textsubtitles-generatortranscriptionvideo-editingvideo-processingvideo-subtitlesvideo-transcriptionwhisper-alternative
update for funasr1.0
shixian.shid2f9ff173957
diff --git a/ClipVideo/clipvideo/gradio_service.py b/ClipVideo/clipvideo/gradio_service.pyindex 021f0f9..7e7e893 100644--- a/ClipVideo/clipvideo/gradio_service.py+++ b/ClipVideo/clipvideo/gradio_service.py@@ -1,23 +1,20 @@import gradio as gr-from modelscope.pipelines import pipeline-from modelscope.utils.constant import Tasks+from funasr import AutoModelfrom videoclipper import VideoClipper-from video import Video+# from video import Video+if __name__ == "__main__":- inference_pipeline = pipeline(- task=Tasks.auto_speech_recognition,- model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',- vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',- punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',- ncpu=16,- )- sd_pipeline = pipeline(- task='speaker-diarization',- model='damo/speech_campplus_speaker-diarization_common',- model_revision='v1.0.0'- )- audio_clipper = VideoClipper(inference_pipeline, sd_pipeline)+ funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+ model_revision="v2.0.4",+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ vad_model_revision="v2.0.4",+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ punc_model_revision="v2.0.4",+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ spk_model_revision="v2.0.2",+ )+ audio_clipper = VideoClipper(funasr_model)def audio_recog(audio_input, sd_switch):print(audio_input)@@ -77,7 +74,7 @@ if __name__ == "__main__":with gr.Tab("🎥✂️视频裁剪 Video Clipping"):with gr.Row():with gr.Column():- video_input = Video(label="🎥视频输入 Video Input")+ video_input = gr.Video(label="🎥视频输入 Video Input")gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E5%A4%9A%E8%AF%BB%E4%B9%A6%EF%BC%9F%E8%BF%99%E6%98%AF%E6%88%91%E5%90%AC%E8%BF%87%E6%9C%80%E5%A5%BD%E7%9A%84%E7%AD%94%E6%A1%88-%E7%89%87%E6%AE%B5.mp4','https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B5.mp4','https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B52.mp4',@@ -147,4 +144,4 @@ if __name__ == "__main__":outputs=[video_output, video_mess_output, video_srt_clip_output])# start gradio service in local- demo.queue(concurrency_count=3).launch()+ demo.launch()diff --git a/ClipVideo/clipvideo/subtitle_utils.py b/ClipVideo/clipvideo/subtitle_utils.pyindex 705f13e..ebcba4b 100644--- a/ClipVideo/clipvideo/subtitle_utils.py+++ b/ClipVideo/clipvideo/subtitle_utils.py@@ -17,10 +17,10 @@ def time_convert(ms):class Text2SRT():- def __init__(self, text_seg, ts_list, offset=0):- self.token_list = [i for i in text_seg.split() if len(i)]- self.ts_list = ts_list- start, end = ts_list[0][0] - offset, ts_list[-1][1] - offset+ def __init__(self, text, timestamp, offset=0):+ self.token_list = [i for i in text.split() if len(i)]+ self.timestamp = timestamp+ start, end = timestamp[0][0] - offset, timestamp[-1][1] - offsetself.start_sec, self.end_sec = start, endself.start_time = time_convert(start)self.end_time = time_convert(end)@@ -42,30 +42,11 @@ class Text2SRT():def time(self, acc_ost=0.0):return (self.start_sec/1000+acc_ost, self.end_sec/1000+acc_ost)-def distribute_spk(sentence_list, sd_time_list):- sd_sentence_list = []- for d in sentence_list:- sentence_start = d['ts_list'][0][0]- sentence_end = d['ts_list'][-1][1]- sentence_spk = 0- max_overlap = 0- for sd_time in sd_time_list:- spk_st, spk_ed, spk = sd_time- spk_st = spk_st*1000- spk_ed = spk_ed*1000- overlap = max(- min(sentence_end, spk_ed) - max(sentence_start, spk_st), 0)- if overlap > max_overlap:- max_overlap = overlap- sentence_spk = spk- d['spk'] = sentence_spk- sd_sentence_list.append(d)- return sd_sentence_listdef generate_srt(sentence_list):srt_total = ''for i, d in enumerate(sentence_list):- t2s = Text2SRT(d['text_seg'], d['ts_list'])+ t2s = Text2SRT(d['text'], d['timestamp'])if 'spk' in d:srt_total += "{} spk{}\n{}".format(i, d['spk'], t2s.srt())else:@@ -78,47 +59,47 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0cc = 1 + begin_indexsubs = []for i, d in enumerate(sentence_list):- if d['ts_list'][-1][1] <= start:+ if d['timestamp'][-1][1] <= start:continue- if d['ts_list'][0][0] >= end:+ if d['timestamp'][0][0] >= end:break# parts in between- if (d['ts_list'][-1][1] <= end and d['ts_list'][0][0] > start) or (d['ts_list'][-1][1] == end and d['ts_list'][0][0] == start):- t2s = Text2SRT(d['text_seg'], d['ts_list'], offset=start)+ if (d['timestamp'][-1][1] <= end and d['timestamp'][0][0] > start) or (d['timestamp'][-1][1] == end and d['timestamp'][0][0] == start):+ t2s = Text2SRT(d['text'], d['timestamp'], offset=start)srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))subs.append((t2s.time(time_acc_ost), t2s.text()))cc += 1continue- if d['ts_list'][0][0] <= start:- if not d['ts_list'][-1][1] > end:- for j, ts in enumerate(d['ts_list']):+ if d['timestamp'][0][0] <= start:+ if not d['timestamp'][-1][1] > end:+ for j, ts in enumerate(d['timestamp']):if ts[1] > start:break- _text = " ".join(d['text_seg'].split()[j:])- _ts = d['ts_list'][j:]+ _text = " ".join(d['text'].split()[j:])+ _ts = d['timestamp'][j:]else:- for j, ts in enumerate(d['ts_list']):+ for j, ts in enumerate(d['timestamp']):if ts[1] > start:_start = jbreak- for j, ts in enumerate(d['ts_list']):+ for j, ts in enumerate(d['timestamp']):if ts[1] > end:_end = jbreak- _text = " ".join(d['text_seg'].split()[_start:_end])- _ts = d['ts_list'][_start:_end]+ _text = " ".join(d['text'].split()[_start:_end])+ _ts = d['timestamp'][_start:_end]if len(ts):t2s = Text2SRT(_text, _ts, offset=start)srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))subs.append((t2s.time(time_acc_ost), t2s.text()))cc += 1continue- if d['ts_list'][-1][1] > end:- for j, ts in enumerate(d['ts_list']):+ if d['timestamp'][-1][1] > end:+ for j, ts in enumerate(d['timestamp']):if ts[1] > end:break- _text = " ".join(d['text_seg'].split()[:j])- _ts = d['ts_list'][:j]+ _text = " ".join(d['text'].split()[:j])+ _ts = d['timestamp'][:j]if len(_ts):t2s = Text2SRT(_text, _ts, offset=start)srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))diff --git a/ClipVideo/clipvideo/trans_utils.py b/ClipVideo/clipvideo/trans_utils.pyindex a630249..4cf6347 100644--- a/ClipVideo/clipvideo/trans_utils.py+++ b/ClipVideo/clipvideo/trans_utils.py@@ -37,9 +37,10 @@ def proc(raw_text, timestamp, dest_text):def proc_spk(dest_spk, sd_sentences):ts = []for d in sd_sentences:- d_start = d['ts_list'][0][0]- d_end = d['ts_list'][-1][1]- spkid=dest_spk[3:]+ d_start = d['timestamp'][0][0]+ d_end = d['timestamp'][-1][1]+ spkid=dest_spk+ # import pdb; pdb.set_trace()if str(d['spk']) == spkid and d_end-d_start>999:ts.append([d['start']*16, d['end']*16])return tsdiff --git a/ClipVideo/clipvideo/video.pyi b/ClipVideo/clipvideo/video.pyinew file mode 100644index 0000000..ab346c5--- /dev/null+++ b/ClipVideo/clipvideo/video.pyi@@ -0,0 +1,73 @@+import gradio+from pathlib import Path+from ffmpy import FFmpeg+from gradio import utils+from gradio_client.documentation import document, set_documentation_group+set_documentation_group("component")+from gradio.events import Dependency++@document()+class Video(gradio.Video):+ """+ This is adapted from gradio.Video. Only `def preprocess()` is modified for auto recoginzing input x is local data or url+ """+ def preprocess(self, x):+ """+ Parameters:+ x: A tuple of (video file data, subtitle file data) or just video file data.+ Returns:+ A string file path or URL to the preprocessed video. Subtitle file data is ignored.+ """+ if x is None:+ return None+ elif isinstance(x, dict):+ video = x+ else:+ video = x[0]++ file_name, file_data, is_file = (+ video.get("name"),+ video["data"],+ video.get("is_file", False),+ )++ if is_file:+ assert file_name is not None, "Received file data without a file name."+ if utils.validate_url(file_name): # URL path+ file_name = Path(self.download_temp_copy_if_needed(file_name))+ else: # Local path+ file_name = Path(self.make_temp_copy_if_needed(file_name))+ else:+ assert file_data is not None, "Received empty file data."+ file_name = Path(self.base64_to_temp_file_if_needed(file_data, file_name))++ uploaded_format = file_name.suffix.replace(".", "")+ needs_formatting = self.format is not None and uploaded_format != self.format+ flip = self.source == "webcam" and self.mirror_webcam++ if needs_formatting or flip:+ format = f".{self.format if needs_formatting else uploaded_format}"+ output_options = ["-vf", "hflip", "-c:a", "copy"] if flip else []+ output_options += ["-an"] if not self.include_audio else []+ flip_suffix = "_flip" if flip else ""+ output_file_name = str(+ file_name.with_name(f"{file_name.stem}{flip_suffix}{format}")+ )+ if Path(output_file_name).exists():+ return output_file_name+ ff = FFmpeg(+ inputs={str(file_name): None},+ outputs={output_file_name: output_options},+ )+ ff.run()+ return output_file_name+ elif not self.include_audio:+ output_file_name = str(file_name.with_name(f"muted_{file_name.name}"))+ ff = FFmpeg(+ inputs={str(file_name): None},+ outputs={output_file_name: ["-an"]},+ )+ ff.run()+ return output_file_name+ else:+ return str(file_name)\ No newline at end of filediff --git a/ClipVideo/clipvideo/videoclipper.py b/ClipVideo/clipvideo/videoclipper.pyindex 66cc79c..4275d88 100644--- a/ClipVideo/clipvideo/videoclipper.py+++ b/ClipVideo/clipvideo/videoclipper.py@@ -8,7 +8,7 @@ import soundfile as sfimport moviepy.editor as mpy# from modelscope.pipelines import pipeline# from modelscope.utils.constant import Tasks-from subtitle_utils import generate_srt, generate_srt_clip, distribute_spk+from subtitle_utils import generate_srt, generate_srt_clip# distribute_spkfrom trans_utils import pre_proc, proc, write_state, load_state, proc_spk, generate_vad_datafrom argparse_tools import ArgumentParser, get_commandline_args@@ -17,10 +17,9 @@ from moviepy.video.tools.subtitles import SubtitlesClipclass VideoClipper():- def __init__(self, asr_pipeline, sd_pipeline=None):+ def __init__(self, funasr_model):logging.warning("Initializing VideoClipper.")- self.asr_pipeline = asr_pipeline- self.sd_pipeline = sd_pipeline+ self.funasr_model = funasr_modeldef recog(self, audio_input, sd_switch='no', state=None):if state is None:@@ -32,19 +31,24 @@ class VideoClipper():data = data[:,0]state['audio_input'] = (sr, data)data = data.astype(np.float64)- rec_result = self.asr_pipeline(audio_in=data)if sd_switch == 'yes':+ rec_result = self.funasr_model.generate(data)+ res_srt = generate_srt(rec_result[0]['sentence_info'])+ state['sd_sentences'] = rec_result[0]['sentence_info']+ '''vad_data = generate_vad_data(data.astype(np.float32), rec_result['sentences'], sr)sd_result = self.sd_pipeline(audio=vad_data, batch_size=1)rec_result['sd_sentences'] = distribute_spk(rec_result['sentences'], sd_result['text'])res_srt = generate_srt(rec_result['sd_sentences'])state['sd_sentences'] = rec_result['sd_sentences']+ '''else:- res_srt = generate_srt(rec_result['sentences'])- state['recog_res_raw'] = rec_result['text_postprocessed']- state['timestamp'] = rec_result['time_stamp']- state['sentences'] = rec_result['sentences']- res_text = rec_result['text']+ rec_result = self.funasr_model.generate(data, return_spk_res=False, sentence_timestamp=True)+ res_srt = generate_srt(rec_result[0]['sentence_info'])+ state['recog_res_raw'] = rec_result[0]['raw_text']+ state['timestamp'] = rec_result[0]['timestamp']+ state['sentences'] = rec_result[0]['sentence_info']+ res_text = rec_result[0]['text']return res_text, res_srt, statedef clip(self, dest_text, start_ost, end_ost, state, dest_spk=None):diff --git a/README.md b/README.mdindex 144b3a9..9e4df0b 100644--- a/README.md+++ b/README.md@@ -20,11 +20,13 @@ Under the help of ClipVideo you can get the video clips easily with the followin### Usage```shell+# install funasr (necessary)+git clone https://github.com/alibaba-damo-academy/FunASR.git+cd FunASR+pip install -e ./+# python environmentsgit clone https://github.com/alibaba-damo-academy/FunASR-APP.gitcd FunASR-APP-# install modelscope-pip install "modelscope[audio_asr]" -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html-# python environmentspip install -r ClipVideo/requirments.txt```(Optional) If you want to clip video file with embedded subtitles