Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

update llm clipping

维石94af28278b3d
diff --git a/funclip/introduction.py b/funclip/introduction.py
index 01d235d..a069e64 100644
--- a/funclip/introduction.py
+++ b/funclip/introduction.py
@@ -28,3 +28,12 @@ top_md_3 = ("""访问FunASR项目与论文能够帮助您深入了解ParaClipper
</div>
</div>
""")
+
+top_md_4 = ("""我们在「LLM智能裁剪」模块中提供两种LLM调用方式,
+ 1. 选择openai开头的模型即为调用openai官方api,此时需要您自备sk与网络环境;
+ 2. [gpt4free](https://github.com/xtekky/gpt4free?tab=readme-ov-file)项目也被集成进FunClip,可以通过它免费调用gpt模型;
+
+ 其中方式1中可以通过在界面中传入sk,或通过`export OPENAI_API_KEY=YOUR_SK`的方式在linux或mac中配置sk
+
+ 方式2而可能非常不稳定,返回时间可能很长或者结果获取失败,可以多多尝试或者自己准备sk使用方式1
+ """)
diff --git a/funclip/launch.py b/funclip/launch.py
index b163ed1..0a2512d 100644
--- a/funclip/launch.py
+++ b/funclip/launch.py
@@ -4,10 +4,14 @@
# MIT License (https://opensource.org/licenses/MIT)
import os
+import logging
import gradio as gr
from funasr import AutoModel
from videoclipper import VideoClipper
-from introduction import top_md_1, top_md_2, top_md_3
+from introduction import top_md_1, top_md_2, top_md_3, top_md_4
+from llm.openai_api import openai_call
+from llm.g4f_openai_api import g4f_openai_call
+from utils.trans_utils import extract_timestamps
if __name__ == "__main__":
@@ -18,7 +22,6 @@ if __name__ == "__main__":
spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
)
audio_clipper = VideoClipper(funasr_model)
- theme = gr.Theme.load("funclip/utils/theme.json")
def audio_recog(audio_input, sd_switch, hotwords, output_dir):
return audio_clipper.recog(audio_input, sd_switch, None, hotwords, output_dir=output_dir)
@@ -84,57 +87,110 @@ if __name__ == "__main__":
font_size=font_size, font_color=font_color,
add_sub=True, dest_spk=video_spk_input, output_dir=output_dir
)
-
+
+ def llm_inference(prompt_head, srt_text, model):
+ SUPPORT_LLM_PREFIX = ['openai', 'g4f']
+ prefix = model.split('-')[0]
+ model_wo_prefix = "-".join(model.split('-')[1:])
+ if prefix == 'openai':
+ return openai_call(prompt_head+'\n'+srt_text, model_wo_prefix)
+ elif prefix.startswith('g4f'):
+ return g4f_openai_call(prompt_head+'\n'+srt_text, model_wo_prefix)
+ else:
+ logging.error("LLM name error, only {} are supported as LLM name prefix."
+ .format(SUPPORT_LLM_PREFIX))
+
+ def AI_clip(LLM_res, dest_text, video_spk_input, start_ost, end_ost, video_state, audio_state, output_dir):
+ timestamp_list = extract_timestamps(LLM_res)
+ output_dir = output_dir.strip()
+ if not len(output_dir):
+ output_dir = None
+ else:
+ output_dir = os.path.abspath(output_dir)
+ if video_state is not None:
+ clip_video_file, message, clip_srt = audio_clipper.video_clip(
+ dest_text, start_ost, end_ost, video_state,
+ dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list)
+ return clip_video_file, None, message, clip_srt
+ if audio_state is not None:
+ (sr, res_audio), message, clip_srt = audio_clipper.clip(
+ dest_text, start_ost, end_ost, audio_state,
+ dest_spk=video_spk_input, output_dir=output_dir, timestamp_list=timestamp_list)
+ return None, (sr, res_audio), message, clip_srt
+
# gradio interface
+ theme = gr.Theme.load("funclip/utils/theme.json")
with gr.Blocks(theme=theme) as funclip_service:
gr.Markdown(top_md_1)
gr.Markdown(top_md_2)
gr.Markdown(top_md_3)
+ gr.Markdown(top_md_4)
video_state, audio_state = gr.State(), gr.State()
with gr.Row():
with gr.Column():
with gr.Row():
- video_input = gr.Video(label="🎥视频输入 Video Input")
- audio_input = gr.Audio(label="🔊音频输入 Audio Input")
+ video_input = gr.Video(label="视频输入 | Video Input")
+ audio_input = gr.Audio(label="音频输入 | Audio Input")
with gr.Column():
gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E5%A4%9A%E8%AF%BB%E4%B9%A6%EF%BC%9F%E8%BF%99%E6%98%AF%E6%88%91%E5%90%AC%E8%BF%87%E6%9C%80%E5%A5%BD%E7%9A%84%E7%AD%94%E6%A1%88-%E7%89%87%E6%AE%B5.mp4',
'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B52.mp4',
'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%BD%BF%E7%94%A8chatgpt_%E7%89%87%E6%AE%B5.mp4'],
[video_input],
- label='示例视频 Demo Video')
+ label='示例视频 | Demo Video')
gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E8%AE%BF%E8%B0%88.mp4'],
[video_input],
- label='多说话人示例视频 Multi-speaker Demo Video')
+ label='多说话人示例视频 | Multi-speaker Demo Video')
gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E9%B2%81%E8%82%83%E9%87%87%E8%AE%BF%E7%89%87%E6%AE%B51.wav'],
[audio_input],
- label="示例音频 Demo Audio")
+ label="示例音频 | Demo Audio")
with gr.Column():
# with gr.Row():
# video_sd_switch = gr.Radio(["No", "Yes"], label="👥区分说话人 Get Speakers", value='No')
- hotwords_input = gr.Textbox(label="🚒热词 Hotwords(多个热词使用空格分隔,仅支持中文热词)")
- output_dir = gr.Textbox(label="📁文件输出路径 File Output Dir", value=" ")
+ hotwords_input = gr.Textbox(label="🚒 热词 | Hotwords(多个热词使用空格分隔,仅支持中文热词)")
+ output_dir = gr.Textbox(label="📁 文件输出路径 | File Output Dir", value=" ")
with gr.Row():
- recog_button = gr.Button("👂识别 ASR", variant="primary")
- recog_button2 = gr.Button("👂识别+区分说话人 ASR+SD")
- video_text_output = gr.Textbox(label="✏️识别结果 Recognition Result")
- video_srt_output = gr.Textbox(label="📖SRT字幕内容 RST Subtitles")
+ recog_button = gr.Button("👂 识别 | ASR", variant="primary")
+ recog_button2 = gr.Button("👂👫 识别+区分说话人 | ASR+SD")
+ video_text_output = gr.Textbox(label="✏️ 识别结果 | Recognition Result")
+ video_srt_output = gr.Textbox(label="📖 SRT字幕内容 | RST Subtitles")
with gr.Column():
- video_text_input = gr.Textbox(label="✏️待裁剪文本 Text to Clip (多段文本使用'#'连接)")
- video_spk_input = gr.Textbox(label="✏️待裁剪说话人 Speaker to Clip (多个说话人使用'#'连接)")
- with gr.Row():
- video_start_ost = gr.Slider(minimum=-500, maximum=1000, value=0, step=50, label="⏪开始位置偏移 Start Offset (ms)")
- video_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩结束位置偏移 End Offset (ms)")
+ with gr.Tab("🧠 LLM智能裁剪 | LLM Clipping"):
+ with gr.Column():
+ prompt_head = gr.Textbox(label="Prompt Head", value=("你是一个视频srt字幕剪辑工具,"
+ "输入视频的srt字幕之后根据如下要求剪辑出最精彩的并且尽可能连续的片段,输出每个段落的开始与结束时间,"
+ "输出需严格按照如下格式:1. [开始时间-结束时间] 文本,\n"
+ "原始srt字幕如下:"))
+ with gr.Row():
+ llm_model = gr.Dropdown(
+ choices=["openai-gpt-3.5-turbo",
+ "openai-gpt-3.5-turbo-0125",
+ "openai-gpt-4-turbo",
+ "g4fopenai-gpt-3.5-turbo"],
+ value="openai-gpt-3.5-turbo",
+ label="LLM Model Name with Prefix",
+ allow_custom_value=True)
+ llm_button = gr.Button("LLM Clip", variant="primary")
+ llm_result = gr.Textbox(label="LLM Clipper Result")
+ with gr.Row():
+ llm_clip_button = gr.Button("🧠 LLM智能裁剪 | AI Clip", variant="primary")
+ llm_clip_subti_button = gr.Button("🧠 LLM智能裁剪+字幕 | AI Clip+Subtitles")
+ with gr.Tab("✂️ 根据文本\说话人裁剪 | Text\Speaker Clipping"):
+ video_text_input = gr.Textbox(label="✏️ 待裁剪文本 | Text to Clip (多段文本使用'#'连接)")
+ video_spk_input = gr.Textbox(label="✏️ 待裁剪说话人 | Speaker to Clip (多个说话人使用'#'连接)")
+ with gr.Row():
+ clip_button = gr.Button("✂️ 裁剪 | Clip", variant="primary")
+ clip_subti_button = gr.Button("✂️ 裁剪+字幕 | Clip+Subtitles")
+ with gr.Row():
+ video_start_ost = gr.Slider(minimum=-500, maximum=1000, value=0, step=50, label="⏪ 开始位置偏移 | Start Offset (ms)")
+ video_end_ost = gr.Slider(minimum=-500, maximum=1000, value=100, step=50, label="⏩ 结束位置偏移 | End Offset (ms)")
with gr.Row():
- font_size = gr.Slider(minimum=10, maximum=100, value=32, step=2, label="🔠字幕字体大小 Subtitle Font Size")
- font_color = gr.Radio(["black", "white", "green", "red"], label="🌈字幕颜色 Subtitle Color", value='white')
+ font_size = gr.Slider(minimum=10, maximum=100, value=32, step=2, label="🔠 字幕字体大小 | Subtitle Font Size")
+ font_color = gr.Radio(["black", "white", "green", "red"], label="🌈 字幕颜色 | Subtitle Color", value='white')
# font = gr.Radio(["黑体", "Alibaba Sans"], label="字体 Font")
- with gr.Row():
- clip_button = gr.Button("✂️裁剪\nClip", variant="primary")
- clip_subti_button = gr.Button("✂️裁剪+字幕\nClip+Subtitles")
- video_output = gr.Video(label="🎥裁剪结果 Video Clipped")
- audio_output = gr.Audio(label="🔊裁剪结果 Audio Clipped")
- clip_message = gr.Textbox(label="ℹ️裁剪信息 Clipping Log")
- srt_clipped = gr.Textbox(label="📖裁剪部分SRT字幕内容 Clipped RST Subtitles")
+ video_output = gr.Video(label="裁剪结果 | Video Clipped")
+ audio_output = gr.Audio(label="裁剪结果 | Audio Clipped")
+ clip_message = gr.Textbox(label="⚠️ 裁剪信息 | Clipping Log")
+ srt_clipped = gr.Textbox(label="📖 裁剪部分SRT字幕内容 | Clipped RST Subtitles")
recog_button.click(mix_recog,
inputs=[video_input,
@@ -171,6 +227,20 @@ if __name__ == "__main__":
font_color,
],
outputs=[video_output, clip_message, srt_clipped])
+ llm_button.click(llm_inference,
+ inputs=[prompt_head, video_srt_output, llm_model],
+ outputs=[llm_result])
+ llm_clip_button.click(AI_clip,
+ inputs=[llm_result,
+ video_text_input,
+ video_spk_input,
+ video_start_ost,
+ video_end_ost,
+ video_state,
+ audio_state,
+ output_dir
+ ],
+ outputs=[video_output, audio_output, clip_message, srt_clipped])
# start gradio service in local
funclip_service.launch()
diff --git a/funclip/llm/g4f_openai_api.py b/funclip/llm/g4f_openai_api.py
new file mode 100644
index 0000000..478af87
--- /dev/null
+++ b/funclip/llm/g4f_openai_api.py
@@ -0,0 +1,19 @@
+from g4f.client import Client
+
+if __name__ == '__main__':
+ from llm.demo_prompt import demo_prompt
+ client = Client()
+ response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "你好你的名字是什么"}],
+ )
+ print(response.choices[0].message.content)
+
+
+def g4f_openai_call(prompt, model="gpt-3.5-turbo"):
+ client = Client()
+ response = client.chat.completions.create(
+ model=model,
+ messages=[{"role": "user", "content": prompt}],
+ )
+ return(response.choices[0].message.content)
\ No newline at end of file
diff --git a/funclip/llm/openai_api.py b/funclip/llm/openai_api.py
index 0a986d0..5be31cb 100644
--- a/funclip/llm/openai_api.py
+++ b/funclip/llm/openai_api.py
@@ -1,8 +1,10 @@
import os
+import logging
from openai import OpenAI
-from demo_prompt import demo_prompt
+
if __name__ == '__main__':
+ from llm.demo_prompt import demo_prompt
client = OpenAI(
# This is the default and can be omitted
api_key=os.environ.get("OPENAI_API_KEY"),
@@ -35,4 +37,5 @@ def openai_call(prompt, model="gpt-3.5-turbo"):
],
model=model,
)
+ logging.info("Openai model inference done.")
return chat_completion.choices[0].message.content
\ No newline at end of file
diff --git a/funclip/imagemagick_test.py b/funclip/test/imagemagick_test.py
similarity index 100%
rename from funclip/imagemagick_test.py
rename to funclip/test/imagemagick_test.py
diff --git a/funclip/test.sh b/funclip/test/test.sh
similarity index 100%
rename from funclip/test.sh
rename to funclip/test/test.sh
diff --git a/funclip/utils/trans_utils.py b/funclip/utils/trans_utils.py
index 9573905..f479f54 100644
--- a/funclip/utils/trans_utils.py
+++ b/funclip/utils/trans_utils.py
@@ -4,6 +4,7 @@
# MIT License (https://opensource.org/licenses/MIT)
import os
+import re
import numpy as np
PUNC_LIST = [',', '。', '!', '?', '、']
@@ -106,5 +107,28 @@ def convert_pcm_to_float(data):
data = data - 128
return (data.astype(np.float64) / max_int_value)
+def convert_time_to_millis(time_str):
+ # 格式: [小时:分钟:秒,毫秒]
+ hours, minutes, seconds, milliseconds = map(int, re.split('[:,]', time_str))
+ return (hours * 3600 + minutes * 60 + seconds) * 1000 + milliseconds
-
\ No newline at end of file
+def extract_timestamps(input_text):
+ # 使用正则表达式查找所有时间戳
+ timestamps = re.findall(r'\[(\d{2}:\d{2}:\d{2},\d{2,3})-(\d{2}:\d{2}:\d{2},\d{2,3})\]', input_text)
+ times_list = []
+
+ # 循环遍历找到的所有时间戳,并转换为毫秒
+ for start_time, end_time in timestamps:
+ start_millis = convert_time_to_millis(start_time)
+ end_millis = convert_time_to_millis(end_time)
+ times_list.append([start_millis, end_millis])
+
+ return times_list
+
+
+if __name__ == '__main__':
+ text = ("1. [00:00:00,500-00:00:05,850] 在我们的设计普惠当中,有一个我经常津津乐道的项目叫寻找远方的美好。"
+ "2. [00:00:07,120-00:00:12,940] 啊,在这样一个我们叫寻美在这样的一个项目当中,我们把它跟乡村振兴去结合起来,利用我们的设计的能力。"
+ "3. [00:00:13,240-00:00:25,620] 问我们自身员工的设设计能力,我们设计生态伙伴的能力,帮助乡村振兴当中,要希望把他的产品推向市场,把他的农产品把他加工产品推向市场的这样的伙伴做一件事情,")
+
+ print(extract_timestamps(text))
\ No newline at end of file
diff --git a/funclip/videoclipper.py b/funclip/videoclipper.py
index ae66d5e..f3444ff 100644
--- a/funclip/videoclipper.py
+++ b/funclip/videoclipper.py
@@ -60,7 +60,7 @@ class VideoClipper():
res_text = rec_result[0]['text']
return res_text, res_srt, state
- def clip(self, dest_text, start_ost, end_ost, state, dest_spk=None, output_dir=None):
+ def clip(self, dest_text, start_ost, end_ost, state, dest_spk=None, output_dir=None, timestamp_list=None):
# get from state
audio_input = state['audio_input']
recog_res_raw = state['recog_res_raw']
@@ -69,32 +69,35 @@ class VideoClipper():
sr, data = audio_input
data = data.astype(np.float64)
- all_ts = []
- if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
- for _dest_text in dest_text.split('#'):
- if '[' in _dest_text:
- match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
- if match:
- offset_b, offset_e = map(int, match.groups())
- log_append = ""
+ if timestamp_list is not None:
+ all_ts = []
+ if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
+ for _dest_text in dest_text.split('#'):
+ if '[' in _dest_text:
+ match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
+ if match:
+ offset_b, offset_e = map(int, match.groups())
+ log_append = ""
+ else:
+ offset_b, offset_e = 0, 0
+ log_append = "(Bracket detected in dest_text but offset time matching failed)"
+ _dest_text = _dest_text[:_dest_text.find('[')]
else:
+ log_append = ""
offset_b, offset_e = 0, 0
- log_append = "(Bracket detected in dest_text but offset time matching failed)"
- _dest_text = _dest_text[:_dest_text.find('[')]
- else:
- log_append = ""
- offset_b, offset_e = 0, 0
- _dest_text = pre_proc(_dest_text)
- ts = proc(recog_res_raw, timestamp, _dest_text)
- for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
- if len(ts) > 1 and match:
- log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
- offsets are applied to all periods)'
+ _dest_text = pre_proc(_dest_text)
+ ts = proc(recog_res_raw, timestamp, _dest_text)
+ for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
+ if len(ts) > 1 and match:
+ log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
+ offsets are applied to all periods)'
+ else:
+ for _dest_spk in dest_spk.split('#'):
+ ts = proc_spk(_dest_spk, state['sd_sentences'])
+ for _ts in ts: all_ts.append(_ts)
+ log_append = ""
else:
- for _dest_spk in dest_spk.split('#'):
- ts = proc_spk(_dest_spk, state['sd_sentences'])
- for _ts in ts: all_ts.append(_ts)
- log_append = ""
+ all_ts = timestamp_list
ts = all_ts
# ts.sort()
srt_index = 0
@@ -158,7 +161,8 @@ class VideoClipper():
font_color='white',
add_sub=False,
dest_spk=None,
- output_dir=None):
+ output_dir=None,
+ timestamp_list=None):
# get from state
recog_res_raw = state['recog_res_raw']
timestamp = state['timestamp']
@@ -167,32 +171,36 @@ class VideoClipper():
clip_video_file = state['clip_video_file']
video_filename = state['video_filename']
- all_ts = []
- srt_index = 0
- if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
- for _dest_text in dest_text.split('#'):
- if '[' in _dest_text:
- match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
- if match:
- offset_b, offset_e = map(int, match.groups())
- log_append = ""
+ if timestamp_list is None:
+ all_ts = []
+ if dest_spk is None or dest_spk == '' or 'sd_sentences' not in state:
+ for _dest_text in dest_text.split('#'):
+ if '[' in _dest_text:
+ match = re.search(r'\[(\d+),\s*(\d+)\]', _dest_text)
+ if match:
+ offset_b, offset_e = map(int, match.groups())
+ log_append = ""
+ else:
+ offset_b, offset_e = 0, 0
+ log_append = "(Bracket detected in dest_text but offset time matching failed)"
+ _dest_text = _dest_text[:_dest_text.find('[')]
else:
offset_b, offset_e = 0, 0
- log_append = "(Bracket detected in dest_text but offset time matching failed)"
- _dest_text = _dest_text[:_dest_text.find('[')]
- else:
- offset_b, offset_e = 0, 0
- log_append = ""
- _dest_text = pre_proc(_dest_text)
- ts = proc(recog_res_raw, timestamp, _dest_text)
- for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
- if len(ts) > 1 and match:
- log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
- offsets are applied to all periods)'
- else:
- for _dest_spk in dest_spk.split('#'):
- ts = proc_spk(_dest_spk, state['sd_sentences'])
- for _ts in ts: all_ts.append(_ts)
+ log_append = ""
+ _dest_text = pre_proc(_dest_text)
+ ts = proc(recog_res_raw, timestamp, _dest_text)
+ for _ts in ts: all_ts.append([_ts[0]+offset_b*16, _ts[1]+offset_e*16])
+ if len(ts) > 1 and match:
+ log_append += '(offsets detected but No.{} sub-sentence matched to {} periods in audio, \
+ offsets are applied to all periods)'
+ else:
+ for _dest_spk in dest_spk.split('#'):
+ ts = proc_spk(_dest_spk, state['sd_sentences'])
+ for _ts in ts: all_ts.append(_ts)
+ else: # AI clip pass timestamp as input directly
+ all_ts = [[i[0]*16.0, i[1]*16.0] for i in timestamp_list]
+ import pdb; pdb.set_trace()
+ srt_index = 0
time_acc_ost = 0.0
ts = all_ts
# ts.sort()