FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.
Python ai-toolsai-video-editingasrauto-subtitleschinesecontent-creationfunasrfunclipgradiollmparaformerspeech-recognitionspeech-to-textsubtitles-generatortranscriptionvideo-editingvideo-processingvideo-subtitlesvideo-transcriptionwhisper-alternative
update fold and name
维石79011f41ee82
diff --git a/.gitignore b/.gitignoreindex b7b6fd3..2f94d8d 100644--- a/.gitignore+++ b/.gitignore@@ -2,4 +2,5 @@.DS_Store*.DS_StoreClipVideo/clipvideo/output-*__pycache__\ No newline at end of file+*__pycache__+*.spec\ No newline at end of filediff --git a/README.md b/README.mdindex bdd3cdf..3b10b87 100644--- a/README.md+++ b/README.md@@ -85,7 +85,7 @@ wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/STHeitiMed### A.在本地启动Gradio服务```shell-python funclip/launch.py+python launch.py```随后在浏览器中访问```localhost:7860```即可看到如下图所示的界面,按如下步骤即可进行视频剪辑1. 上传你的视频(或使用下方的视频用例)diff --git a/README_en.md b/README_en.mdindex 096b6b4..fca0b0a 100644--- a/README_en.md+++ b/README_en.md@@ -88,7 +88,7 @@ wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/STHeitiMed### A. Use FunClip as local Gradio ServiceYou can establish your own FunClip service which is same as [Modelscope Space](https://modelscope.cn/studios/iic/funasr_app_clipvideo/summary) as follow:```shell-python funclip/launch.py+python launch.py```then visit ```localhost:7860``` you will get a Gradio service like below and you can use FunClip following the steps:<img src="docs/images/guide.jpg"/>diff --git a/funclip/launch.py b/funclip/launch.pyindex 0473770..8e05fd7 100644--- a/funclip/launch.py+++ b/funclip/launch.py@@ -6,8 +6,8 @@import osimport gradio as grfrom funasr import AutoModel-from videoclipper import VideoClipper-from introduction import top_md_1, top_md_2, top_md_3+from funclip.videoclipper import VideoClipper+from funclip.introduction import top_md_1, top_md_2, top_md_3if __name__ == "__main__":diff --git a/funclip/videoclipper.py b/funclip/videoclipper.pyindex c12816c..1b676bf 100644--- a/funclip/videoclipper.py+++ b/funclip/videoclipper.py@@ -15,9 +15,9 @@ import soundfile as sffrom moviepy.editor import *import moviepy.editor as mpyfrom moviepy.video.tools.subtitles import SubtitlesClip-from utils.subtitle_utils import generate_srt, generate_srt_clip-from utils.argparse_tools import ArgumentParser, get_commandline_args-from utils.trans_utils import pre_proc, proc, write_state, load_state, proc_spk, convert_pcm_to_float+from funclip.utils.subtitle_utils import generate_srt, generate_srt_clip+from funclip.utils.argparse_tools import ArgumentParser, get_commandline_args+from funclip.utils.trans_utils import pre_proc, proc, write_state, load_state, proc_spk, convert_pcm_to_floatclass VideoClipper():diff --git a/launch.py b/launch.pynew file mode 100644index 0000000..8e05fd7--- /dev/null+++ b/launch.py@@ -0,0 +1,176 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunClip). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import os+import gradio as gr+from funasr import AutoModel+from funclip.videoclipper import VideoClipper+from funclip.introduction import top_md_1, top_md_2, top_md_3+++if __name__ == "__main__":++ funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ )+ audio_clipper = VideoClipper(funasr_model)+ theme = gr.Theme.load("funclip/utils/theme.json")++ def audio_recog(audio_input, sd_switch, hotwords, output_dir):+ return audio_clipper.recog(audio_input, sd_switch, hotwords, output_dir=output_dir)++ def video_recog(video_input, sd_switch, hotwords, output_dir):+ return audio_clipper.video_recog(video_input, sd_switch, hotwords, output_dir=output_dir)++ def video_clip(dest_text, video_spk_input, start_ost, end_ost, state, output_dir):+ return audio_clipper.video_clip(+ dest_text, start_ost, end_ost, state, dest_spk=video_spk_input, output_dir=output_dir+ )++ def mix_recog(video_input, audio_input, hotwords, output_dir):+ output_dir = output_dir.strip()+ if not len(output_dir):+ output_dir = None+ else:+ output_dir = os.path.abspath(output_dir)+ audio_state, video_state = None, None+ if video_input is not None:+ res_text, res_srt, video_state = video_recog(+ video_input, 'No', hotwords, output_dir=output_dir)+ return res_text, res_srt, video_state, None+ if audio_input is not None:+ res_text, res_srt, audio_state = audio_recog(+ audio_input, 'No', hotwords, output_dir=output_dir)+ return res_text, res_srt, None, audio_state++ def mix_recog_speaker(video_input, audio_input, hotwords, output_dir):+ output_dir = output_dir.strip()+ if not len(output_dir):+ output_dir = None+ else:+ output_dir = os.path.abspath(output_dir)+ audio_state, video_state = None, None+ if video_input is not None:+ res_text, res_srt, video_state = video_recog(+ video_input, 'Yes', hotwords, output_dir=output_dir)+ return res_text, res_srt, video_state, None+ if audio_input is not None:+ res_text, res_srt, audio_state = audio_recog(+ audio_input, 'Yes', hotwords, output_dir=output_dir)+ return res_text, res_srt, None, audio_state++ def mix_clip(dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir):+ output_dir = output_dir.strip()+ if not len(output_dir):+ output_dir = None+ else:+ output_dir = os.path.abspath(output_dir)+ if video_state is not None:+ clip_video_file, message, clip_srt = audio_clipper.video_clip(+ dest_text, start_ost, end_ost, video_state, dest_spk=video_spk_input, output_dir=output_dir)+ return clip_video_file, None, message, clip_srt+ if audio_state is not None:+ (sr, res_audio), message, clip_srt = audio_clipper.clip(+ dest_text, start_ost, end_ost, audio_state, dest_spk=video_spk_input, output_dir=output_dir)+ return None, (sr, res_audio), message, clip_srt++ def video_clip_addsub(dest_text, video_spk_input, start_ost, end_ost, state, output_dir, font_size, font_color):+ return audio_clipper.video_clip(+ dest_text, start_ost, end_ost, state,+ font_size=font_size, font_color=font_color,+ add_sub=True, dest_spk=video_spk_input, output_dir=output_dir+ )++ # gradio interface+ with gr.Blocks(theme=theme) as funclip_service:+ gr.Markdown(top_md_1)+ gr.Markdown(top_md_2)+ gr.Markdown(top_md_3)+ video_state, audio_state = gr.State(), gr.State()+ with gr.Row():+ with gr.Column():+ with gr.Row():+ video_input = gr.Video(label="🎥视频输入 Video Input")+ audio_input = gr.Audio(label="🔊音频输入 Audio Input")+ with gr.Column():+ gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E5%A4%9A%E8%AF%BB%E4%B9%A6%EF%BC%9F%E8%BF%99%E6%98%AF%E6%88%91%E5%90%AC%E8%BF%87%E6%9C%80%E5%A5%BD%E7%9A%84%E7%AD%94%E6%A1%88-%E7%89%87%E6%AE%B5.mp4',+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B52.mp4',+ 'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%BD%BF%E7%94%A8chatgpt_%E7%89%87%E6%AE%B5.mp4'],+ [video_input],+ label='示例视频 Demo Video')+ gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E8%AE%BF%E8%B0%88.mp4'],+ [video_input],+ label='多说话人示例视频 Multi-speaker Demo Video')+ gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E9%B2%81%E8%82%83%E9%87%87%E8%AE%BF%E7%89%87%E6%AE%B51.wav'],+ [audio_input],+ label="示例音频 Demo Audio")+ with gr.Column():+ # with gr.Row():+ # video_sd_switch = gr.Radio(["No", "Yes"], label="👥区分说话人 Get Speakers", value='No')+ hotwords_input = gr.Textbox(label="🚒热词 Hotwords(多个热词使用空格分隔,仅支持中文热词)")+ output_dir = gr.Textbox(label="📁文件输出路径 File Output Dir", value=" ")+ with gr.Row():+ recog_button = gr.Button("👂识别 ASR", variant="primary")+ recog_button2 = gr.Button("👂识别+区分说话人 ASR+SD")+ video_text_output = gr.Textbox(label="✏️识别结果 Recognition Result")+ video_srt_output = gr.Textbox(label="📖SRT字幕内容 RST Subtitles")+ with gr.Column():+ video_text_input = gr.Textbox(label="✏️待裁剪文本 Text to Clip (多段文本使用'#'连接)")+ video_spk_input = gr.Textbox(label="✏️待裁剪说话人 Speaker to Clip (多个说话人使用'#'连接)")+ with gr.Row():+ video_start_ost = gr.Slider(minimum=-500, maximum=1000, value=0, step=50, label="⏪开始位置偏移 Start Offset (ms)")+ video_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩结束位置偏移 End Offset (ms)")+ with gr.Row():+ font_size = gr.Slider(minimum=10, maximum=100, value=32, step=2, label="🔠字幕字体大小 Subtitle Font Size")+ font_color = gr.Radio(["black", "white", "green", "red"], label="🌈字幕颜色 Subtitle Color", value='white')+ # font = gr.Radio(["黑体", "Alibaba Sans"], label="字体 Font")+ with gr.Row():+ clip_button = gr.Button("✂️裁剪\nClip", variant="primary")+ clip_subti_button = gr.Button("✂️裁剪+字幕\nClip+Subtitles")+ video_output = gr.Video(label="🎥裁剪结果 Video Clipped")+ audio_output = gr.Audio(label="🔊裁剪结果 Audio Clipped")+ clip_message = gr.Textbox(label="ℹ️裁剪信息 Clipping Log")+ srt_clipped = gr.Textbox(label="📖裁剪部分SRT字幕内容 Clipped RST Subtitles")++ recog_button.click(mix_recog,+ inputs=[video_input,+ audio_input,+ hotwords_input,+ output_dir,+ ],+ outputs=[video_text_output, video_srt_output, video_state, audio_state])+ recog_button2.click(mix_recog_speaker,+ inputs=[video_input,+ audio_input,+ hotwords_input,+ output_dir,+ ],+ outputs=[video_text_output, video_srt_output, video_state, audio_state])+ clip_button.click(mix_clip,+ inputs=[video_text_input,+ video_spk_input,+ video_start_ost,+ video_end_ost,+ video_state,+ audio_state,+ output_dir+ ],+ outputs=[video_output, audio_output, clip_message, srt_clipped])+ clip_subti_button.click(video_clip_addsub,+ inputs=[video_text_input,+ video_spk_input,+ video_start_ost,+ video_end_ost,+ video_state,+ output_dir,+ font_size,+ font_color,+ ],+ outputs=[video_output, clip_message, srt_clipped])++ # start gradio service in local+ funclip_service.launch()