Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

update english mode

维石e08ece35e4be
diff --git a/README.md b/README.md
index 275ee49..d2c5c5f 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,7 @@
<a name="What's New"></a>
## What's New🚀
+- 2024/06/12 FunClip supports recognize and clip English audio files now. Run `python funclip/launch.py -l en` to try.
- 🔥2024/05/13 FunClip v2.0.0 now supports smart clipping with large language models, integrating models from the qwen series, GPT series, etc., providing default prompts. You can also explore and share tips for setting prompts, the usage is as follows:
1. After the recognition, select the name of the large model and configure your own apikey;
2. Click on the 'LLM Inference' button, and FunClip will automatically combine two prompts with the video's srt subtitles;
@@ -106,6 +107,9 @@ wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/STHeitiMed
You can establish your own FunClip service which is same as [Modelscope Space](https://modelscope.cn/studios/iic/funasr_app_clipvideo/summary) as follow:
```shell
python funclip/launch.py
+# '-l en' for English audio recognize
+# '-p xxx' for setting port number
+# '-s True' for establishing service for public accessing
```
then visit ```localhost:7860``` you will get a Gradio service like below and you can use FunClip following the steps:
diff --git a/README_zh.md b/README_zh.md
index 477ae5f..4a789a6 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -36,6 +36,7 @@
<a name="近期更新"></a>
## 近期更新🚀
+- 2024/06/12 FunClip现在支持识别与裁剪英文视频,通过`python funclip/launch.py -l en`来启动英文版本服务。
- 🔥2024/05/13 FunClip v2.0.0加入大语言模型智能裁剪功能,集成qwen系列,gpt系列等模型,提供默认prompt,您也可以探索并分享prompt的设置技巧,使用方法如下:
1. 在进行识别之后,选择大模型名称,配置你自己的apikey;
2. 点击'LLM智能段落选择'按钮,FunClip将自动组合两个prompt与视频的srt字幕;
@@ -54,7 +55,7 @@
<a name="施工中"></a>
## 施工中🌵
-- [ ] FunClip将会集成Whisper模型,以提供英文视频剪辑能力。
+- [x] FunClip将会集成Whisper模型,以提供英文视频剪辑能力(Whisper模型的时间戳预测功能需要显存较大,我们在FunASR中添加了Paraformer英文模型的时间戳预测支持以允许FunClip支持英文识别裁剪)。
- [x] 集成大语言模型的能力,提供智能视频剪辑相关功能。大家可以基于FunClip探索使用大语言模型的视频剪辑~
- [ ] 给定文本段落,反向选取其他段落。
- [ ] 删除视频中无人说话的片段。
@@ -106,6 +107,9 @@ wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/STHeitiMed
```shell
python funclip/launch.py
+# '-l en' for English audio recognize
+# '-p xxx' for setting port number
+# '-s True' for establishing service for public accessing
```
随后在浏览器中访问```localhost:7860```即可看到如下图所示的界面,按如下步骤即可进行视频剪辑
1. 上传你的视频(或使用下方的视频用例)
diff --git a/funclip/launch.py b/funclip/launch.py
index ed3a64c..4209bca 100644
--- a/funclip/launch.py
+++ b/funclip/launch.py
@@ -5,24 +5,39 @@
import os
import logging
+import argparse
import gradio as gr
from funasr import AutoModel
from videoclipper import VideoClipper
-from introduction import top_md_1, top_md_3, top_md_4
from llm.openai_api import openai_call
-from llm.g4f_openai_api import g4f_openai_call
from llm.qwen_api import call_qwen_model
+from llm.g4f_openai_api import g4f_openai_call
from utils.trans_utils import extract_timestamps
+from introduction import top_md_1, top_md_3, top_md_4
if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='argparse testing')
+ parser.add_argument('--lang', '-l', type=str, default = "zh", help="language")
+ parser.add_argument('--share', '-s', type=bool, default = False, help="if to establish gradio share link")
+ parser.add_argument('--port', '-p', type=int, default=7860, help='port number')
+ args = parser.parse_args()
- funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
- vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
- punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
- spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
- )
+ if args.lang == 'zh':
+ funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
+ )
+ else:
+ funasr_model = AutoModel(model="iic/speech_paraformer_asr-en-16k-vocab4199-pytorch",
+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
+ )
audio_clipper = VideoClipper(funasr_model)
+ audio_clipper.lang = args.lang
+
def audio_recog(audio_input, sd_switch, hotwords, output_dir):
return audio_clipper.recog(audio_input, sd_switch, None, hotwords, output_dir=output_dir)
@@ -252,5 +267,8 @@ if __name__ == "__main__":
],
outputs=[video_output, audio_output, clip_message, srt_clipped])
- # start gradio service in local
- funclip_service.launch()
+ # start gradio service in local or share
+ if args.share:
+ funclip_service.launch(share=True, server_port=args.port)
+ else:
+ funclip_service.launch(server_port=args.port)
diff --git a/funclip/utils/subtitle_utils.py b/funclip/utils/subtitle_utils.py
index 745683d..c90cce6 100644
--- a/funclip/utils/subtitle_utils.py
+++ b/funclip/utils/subtitle_utils.py
@@ -2,6 +2,7 @@
# -*- encoding: utf-8 -*-
# Copyright FunASR (https://github.com/alibaba-damo-academy/FunClip). All Rights Reserved.
# MIT License (https://opensource.org/licenses/MIT)
+import re
def time_convert(ms):
ms = int(ms)
@@ -20,10 +21,14 @@ def time_convert(ms):
if len(s) == 1: s = '0' + s
return "{}:{}:{},{}".format(h, mi, s, tail)
+def str2list(text):
+ pattern = re.compile(r'[\u4e00-\u9fff]|[\w-]+', re.UNICODE)
+ elements = pattern.findall(text)
+ return elements
class Text2SRT():
def __init__(self, text, timestamp, offset=0):
- self.token_list = [i for i in text.split() if len(i)]
+ self.token_list = text
self.timestamp = timestamp
start, end = timestamp[0][0] - offset, timestamp[-1][1] - offset
self.start_sec, self.end_sec = start, end
@@ -64,6 +69,7 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0
cc = 1 + begin_index
subs = []
for _, sent in enumerate(sentence_list):
+ sent['text'] = str2list(sent['text'])
if sent['timestamp'][-1][1] <= start:
# print("CASE0")
continue
@@ -72,19 +78,19 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0
break
# parts in between
if (sent['timestamp'][-1][1] <= end and sent['timestamp'][0][0] > start) or (sent['timestamp'][-1][1] == end and sent['timestamp'][0][0] == start):
- # print("CASE1")
+ # print("CASE1"); import pdb; pdb.set_trace()
t2s = Text2SRT(sent['text'], sent['timestamp'], offset=start)
srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))
subs.append((t2s.time(time_acc_ost), t2s.text()))
cc += 1
continue
if sent['timestamp'][0][0] <= start:
- # print("CASE2")
+ # print("CASE2"); import pdb; pdb.set_trace()
if not sent['timestamp'][-1][1] > end:
for j, ts in enumerate(sent['timestamp']):
if ts[1] > start:
break
- _text = " ".join(sent['text'][j:])
+ _text = sent['text'][j:]
_ts = sent['timestamp'][j:]
else:
for j, ts in enumerate(sent['timestamp']):
@@ -95,7 +101,8 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0
if ts[1] > end:
_end = j
break
- _text = " ".join(sent['text'][_start:_end])
+ # _text = " ".join(sent['text'][_start:_end])
+ _text = sent['text'][_start:_end]
_ts = sent['timestamp'][_start:_end]
if len(ts):
t2s = Text2SRT(_text, _ts, offset=start)
@@ -104,11 +111,11 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0
cc += 1
continue
if sent['timestamp'][-1][1] > end:
- # print("CASE3")
+ # print("CASE3"); import pdb; pdb.set_trace()
for j, ts in enumerate(sent['timestamp']):
if ts[1] > end:
break
- _text = " ".join(sent['text'][:j])
+ _text = sent['text'][:j]
_ts = sent['timestamp'][:j]
if len(_ts):
t2s = Text2SRT(_text, _ts, offset=start)
diff --git a/funclip/utils/trans_utils.py b/funclip/utils/trans_utils.py
index a2cd03b..7798704 100644
--- a/funclip/utils/trans_utils.py
+++ b/funclip/utils/trans_utils.py
@@ -7,7 +7,7 @@ import os
import re
import numpy as np
-PUNC_LIST = [',', '。', '!', '?', '、']
+PUNC_LIST = [',', '。', '!', '?', '、', ',', '.', '?', '!']
def pre_proc(text):
res = ''
@@ -25,7 +25,7 @@ def pre_proc(text):
res = res[:-1]
return res
-def proc(raw_text, timestamp, dest_text):
+def proc(raw_text, timestamp, dest_text, lang='zh'):
# simple matching
ld = len(dest_text.split())
mi, ts = [], []
@@ -39,6 +39,7 @@ def proc(raw_text, timestamp, dest_text):
mi.append(fi)
ts.append([timestamp[ti][0]*16, timestamp[ti+ld-1][1]*16])
return ts
+
def proc_spk(dest_spk, sd_sentences):
ts = []
diff --git a/funclip/videoclipper.py b/funclip/videoclipper.py
index 4384a3a..ef0efa5 100644
--- a/funclip/videoclipper.py
+++ b/funclip/videoclipper.py
@@ -42,7 +42,13 @@ class VideoClipper():
data = data[:,0]
state['audio_input'] = (sr, data)
if sd_switch == 'Yes':
- rec_result = self.funasr_model.generate(data, return_raw_text=True, is_final=True, hotword=hotwords, cache={})
+ rec_result = self.funasr_model.generate(data,
+ return_raw_text=True,
+ is_final=True,
+ output_dir=output_dir,
+ hotword=hotwords,
+ pred_timestamp=self.lang=='en',
+ cache={})
res_srt = generate_srt(rec_result[0]['sentence_info'])
state['sd_sentences'] = rec_result[0]['sentence_info']
else:
@@ -53,6 +59,8 @@ class VideoClipper():
is_final=True,
hotword=hotwords,
output_dir=output_dir,
+ pred_timestamp=self.lang=='en',
+ en_post_proc=True,
cache={})
res_srt = generate_srt(rec_result[0]['sentence_info'])
state['recog_res_raw'] = rec_result[0]['raw_text']
@@ -207,6 +215,8 @@ class VideoClipper():
# ts.sort()
clip_srt = ""
if len(ts):
+ if self.lang == 'en':
+ sentences = sentences.split()
start, end = ts[0][0] / 16000, ts[0][1] / 16000
srt_clip, subs, srt_index = generate_srt_clip(sentences, start, end, begin_index=srt_index, time_acc_ost=time_acc_ost)
start, end = start+start_ost/1000.0, end+end_ost/1000.0