Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

update for funasr1.0

shixian.shid2f9ff173957
diff --git a/ClipVideo/clipvideo/gradio_service.py b/ClipVideo/clipvideo/gradio_service.py
index 021f0f9..7e7e893 100644
--- a/ClipVideo/clipvideo/gradio_service.py
+++ b/ClipVideo/clipvideo/gradio_service.py
@@ -1,23 +1,20 @@
import gradio as gr
-from modelscope.pipelines import pipeline
-from modelscope.utils.constant import Tasks
+from funasr import AutoModel
from videoclipper import VideoClipper
-from video import Video
+# from video import Video
+
if __name__ == "__main__":
- inference_pipeline = pipeline(
- task=Tasks.auto_speech_recognition,
- model='damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch',
- vad_model='damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',
- punc_model='damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',
- ncpu=16,
- )
- sd_pipeline = pipeline(
- task='speaker-diarization',
- model='damo/speech_campplus_speaker-diarization_common',
- model_revision='v1.0.0'
- )
- audio_clipper = VideoClipper(inference_pipeline, sd_pipeline)
+ funasr_model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
+ model_revision="v2.0.4",
+ vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ vad_model_revision="v2.0.4",
+ punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+ punc_model_revision="v2.0.4",
+ spk_model="damo/speech_campplus_sv_zh-cn_16k-common",
+ spk_model_revision="v2.0.2",
+ )
+ audio_clipper = VideoClipper(funasr_model)
def audio_recog(audio_input, sd_switch):
print(audio_input)
@@ -77,7 +74,7 @@ if __name__ == "__main__":
with gr.Tab("🎥✂️视频裁剪 Video Clipping"):
with gr.Row():
with gr.Column():
- video_input = Video(label="🎥视频输入 Video Input")
+ video_input = gr.Video(label="🎥视频输入 Video Input")
gr.Examples(['https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/%E4%B8%BA%E4%BB%80%E4%B9%88%E8%A6%81%E5%A4%9A%E8%AF%BB%E4%B9%A6%EF%BC%9F%E8%BF%99%E6%98%AF%E6%88%91%E5%90%AC%E8%BF%87%E6%9C%80%E5%A5%BD%E7%9A%84%E7%AD%94%E6%A1%88-%E7%89%87%E6%AE%B5.mp4',
'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B5.mp4',
'https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ClipVideo/2022%E4%BA%91%E6%A0%96%E5%A4%A7%E4%BC%9A_%E7%89%87%E6%AE%B52.mp4',
@@ -147,4 +144,4 @@ if __name__ == "__main__":
outputs=[video_output, video_mess_output, video_srt_clip_output])
# start gradio service in local
- demo.queue(concurrency_count=3).launch()
+ demo.launch()
diff --git a/ClipVideo/clipvideo/subtitle_utils.py b/ClipVideo/clipvideo/subtitle_utils.py
index 705f13e..ebcba4b 100644
--- a/ClipVideo/clipvideo/subtitle_utils.py
+++ b/ClipVideo/clipvideo/subtitle_utils.py
@@ -17,10 +17,10 @@ def time_convert(ms):
class Text2SRT():
- def __init__(self, text_seg, ts_list, offset=0):
- self.token_list = [i for i in text_seg.split() if len(i)]
- self.ts_list = ts_list
- start, end = ts_list[0][0] - offset, ts_list[-1][1] - offset
+ def __init__(self, text, timestamp, offset=0):
+ self.token_list = [i for i in text.split() if len(i)]
+ self.timestamp = timestamp
+ start, end = timestamp[0][0] - offset, timestamp[-1][1] - offset
self.start_sec, self.end_sec = start, end
self.start_time = time_convert(start)
self.end_time = time_convert(end)
@@ -42,30 +42,11 @@ class Text2SRT():
def time(self, acc_ost=0.0):
return (self.start_sec/1000+acc_ost, self.end_sec/1000+acc_ost)
-def distribute_spk(sentence_list, sd_time_list):
- sd_sentence_list = []
- for d in sentence_list:
- sentence_start = d['ts_list'][0][0]
- sentence_end = d['ts_list'][-1][1]
- sentence_spk = 0
- max_overlap = 0
- for sd_time in sd_time_list:
- spk_st, spk_ed, spk = sd_time
- spk_st = spk_st*1000
- spk_ed = spk_ed*1000
- overlap = max(
- min(sentence_end, spk_ed) - max(sentence_start, spk_st), 0)
- if overlap > max_overlap:
- max_overlap = overlap
- sentence_spk = spk
- d['spk'] = sentence_spk
- sd_sentence_list.append(d)
- return sd_sentence_list
def generate_srt(sentence_list):
srt_total = ''
for i, d in enumerate(sentence_list):
- t2s = Text2SRT(d['text_seg'], d['ts_list'])
+ t2s = Text2SRT(d['text'], d['timestamp'])
if 'spk' in d:
srt_total += "{} spk{}\n{}".format(i, d['spk'], t2s.srt())
else:
@@ -78,47 +59,47 @@ def generate_srt_clip(sentence_list, start, end, begin_index=0, time_acc_ost=0.0
cc = 1 + begin_index
subs = []
for i, d in enumerate(sentence_list):
- if d['ts_list'][-1][1] <= start:
+ if d['timestamp'][-1][1] <= start:
continue
- if d['ts_list'][0][0] >= end:
+ if d['timestamp'][0][0] >= end:
break
# parts in between
- if (d['ts_list'][-1][1] <= end and d['ts_list'][0][0] > start) or (d['ts_list'][-1][1] == end and d['ts_list'][0][0] == start):
- t2s = Text2SRT(d['text_seg'], d['ts_list'], offset=start)
+ if (d['timestamp'][-1][1] <= end and d['timestamp'][0][0] > start) or (d['timestamp'][-1][1] == end and d['timestamp'][0][0] == start):
+ t2s = Text2SRT(d['text'], d['timestamp'], offset=start)
srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))
subs.append((t2s.time(time_acc_ost), t2s.text()))
cc += 1
continue
- if d['ts_list'][0][0] <= start:
- if not d['ts_list'][-1][1] > end:
- for j, ts in enumerate(d['ts_list']):
+ if d['timestamp'][0][0] <= start:
+ if not d['timestamp'][-1][1] > end:
+ for j, ts in enumerate(d['timestamp']):
if ts[1] > start:
break
- _text = " ".join(d['text_seg'].split()[j:])
- _ts = d['ts_list'][j:]
+ _text = " ".join(d['text'].split()[j:])
+ _ts = d['timestamp'][j:]
else:
- for j, ts in enumerate(d['ts_list']):
+ for j, ts in enumerate(d['timestamp']):
if ts[1] > start:
_start = j
break
- for j, ts in enumerate(d['ts_list']):
+ for j, ts in enumerate(d['timestamp']):
if ts[1] > end:
_end = j
break
- _text = " ".join(d['text_seg'].split()[_start:_end])
- _ts = d['ts_list'][_start:_end]
+ _text = " ".join(d['text'].split()[_start:_end])
+ _ts = d['timestamp'][_start:_end]
if len(ts):
t2s = Text2SRT(_text, _ts, offset=start)
srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))
subs.append((t2s.time(time_acc_ost), t2s.text()))
cc += 1
continue
- if d['ts_list'][-1][1] > end:
- for j, ts in enumerate(d['ts_list']):
+ if d['timestamp'][-1][1] > end:
+ for j, ts in enumerate(d['timestamp']):
if ts[1] > end:
break
- _text = " ".join(d['text_seg'].split()[:j])
- _ts = d['ts_list'][:j]
+ _text = " ".join(d['text'].split()[:j])
+ _ts = d['timestamp'][:j]
if len(_ts):
t2s = Text2SRT(_text, _ts, offset=start)
srt_total += "{}\n{}".format(cc, t2s.srt(time_acc_ost))
diff --git a/ClipVideo/clipvideo/trans_utils.py b/ClipVideo/clipvideo/trans_utils.py
index a630249..4cf6347 100644
--- a/ClipVideo/clipvideo/trans_utils.py
+++ b/ClipVideo/clipvideo/trans_utils.py
@@ -37,9 +37,10 @@ def proc(raw_text, timestamp, dest_text):
def proc_spk(dest_spk, sd_sentences):
ts = []
for d in sd_sentences:
- d_start = d['ts_list'][0][0]
- d_end = d['ts_list'][-1][1]
- spkid=dest_spk[3:]
+ d_start = d['timestamp'][0][0]
+ d_end = d['timestamp'][-1][1]
+ spkid=dest_spk
+ # import pdb; pdb.set_trace()
if str(d['spk']) == spkid and d_end-d_start>999:
ts.append([d['start']*16, d['end']*16])
return ts
diff --git a/ClipVideo/clipvideo/video.pyi b/ClipVideo/clipvideo/video.pyi
new file mode 100644
index 0000000..ab346c5
--- /dev/null
+++ b/ClipVideo/clipvideo/video.pyi
@@ -0,0 +1,73 @@
+import gradio
+from pathlib import Path
+from ffmpy import FFmpeg
+from gradio import utils
+from gradio_client.documentation import document, set_documentation_group
+set_documentation_group("component")
+from gradio.events import Dependency
+
+@document()
+class Video(gradio.Video):
+ """
+ This is adapted from gradio.Video. Only `def preprocess()` is modified for auto recoginzing input x is local data or url
+ """
+ def preprocess(self, x):
+ """
+ Parameters:
+ x: A tuple of (video file data, subtitle file data) or just video file data.
+ Returns:
+ A string file path or URL to the preprocessed video. Subtitle file data is ignored.
+ """
+ if x is None:
+ return None
+ elif isinstance(x, dict):
+ video = x
+ else:
+ video = x[0]
+
+ file_name, file_data, is_file = (
+ video.get("name"),
+ video["data"],
+ video.get("is_file", False),
+ )
+
+ if is_file:
+ assert file_name is not None, "Received file data without a file name."
+ if utils.validate_url(file_name): # URL path
+ file_name = Path(self.download_temp_copy_if_needed(file_name))
+ else: # Local path
+ file_name = Path(self.make_temp_copy_if_needed(file_name))
+ else:
+ assert file_data is not None, "Received empty file data."
+ file_name = Path(self.base64_to_temp_file_if_needed(file_data, file_name))
+
+ uploaded_format = file_name.suffix.replace(".", "")
+ needs_formatting = self.format is not None and uploaded_format != self.format
+ flip = self.source == "webcam" and self.mirror_webcam
+
+ if needs_formatting or flip:
+ format = f".{self.format if needs_formatting else uploaded_format}"
+ output_options = ["-vf", "hflip", "-c:a", "copy"] if flip else []
+ output_options += ["-an"] if not self.include_audio else []
+ flip_suffix = "_flip" if flip else ""
+ output_file_name = str(
+ file_name.with_name(f"{file_name.stem}{flip_suffix}{format}")
+ )
+ if Path(output_file_name).exists():
+ return output_file_name
+ ff = FFmpeg(
+ inputs={str(file_name): None},
+ outputs={output_file_name: output_options},
+ )
+ ff.run()
+ return output_file_name
+ elif not self.include_audio:
+ output_file_name = str(file_name.with_name(f"muted_{file_name.name}"))
+ ff = FFmpeg(
+ inputs={str(file_name): None},
+ outputs={output_file_name: ["-an"]},
+ )
+ ff.run()
+ return output_file_name
+ else:
+ return str(file_name)
\ No newline at end of file
diff --git a/ClipVideo/clipvideo/videoclipper.py b/ClipVideo/clipvideo/videoclipper.py
index 66cc79c..4275d88 100644
--- a/ClipVideo/clipvideo/videoclipper.py
+++ b/ClipVideo/clipvideo/videoclipper.py
@@ -8,7 +8,7 @@ import soundfile as sf
import moviepy.editor as mpy
# from modelscope.pipelines import pipeline
# from modelscope.utils.constant import Tasks
-from subtitle_utils import generate_srt, generate_srt_clip, distribute_spk
+from subtitle_utils import generate_srt, generate_srt_clip# distribute_spk
from trans_utils import pre_proc, proc, write_state, load_state, proc_spk, generate_vad_data
from argparse_tools import ArgumentParser, get_commandline_args
@@ -17,10 +17,9 @@ from moviepy.video.tools.subtitles import SubtitlesClip
class VideoClipper():
- def __init__(self, asr_pipeline, sd_pipeline=None):
+ def __init__(self, funasr_model):
logging.warning("Initializing VideoClipper.")
- self.asr_pipeline = asr_pipeline
- self.sd_pipeline = sd_pipeline
+ self.funasr_model = funasr_model
def recog(self, audio_input, sd_switch='no', state=None):
if state is None:
@@ -32,19 +31,24 @@ class VideoClipper():
data = data[:,0]
state['audio_input'] = (sr, data)
data = data.astype(np.float64)
- rec_result = self.asr_pipeline(audio_in=data)
if sd_switch == 'yes':
+ rec_result = self.funasr_model.generate(data)
+ res_srt = generate_srt(rec_result[0]['sentence_info'])
+ state['sd_sentences'] = rec_result[0]['sentence_info']
+ '''
vad_data = generate_vad_data(data.astype(np.float32), rec_result['sentences'], sr)
sd_result = self.sd_pipeline(audio=vad_data, batch_size=1)
rec_result['sd_sentences'] = distribute_spk(rec_result['sentences'], sd_result['text'])
res_srt = generate_srt(rec_result['sd_sentences'])
state['sd_sentences'] = rec_result['sd_sentences']
+ '''
else:
- res_srt = generate_srt(rec_result['sentences'])
- state['recog_res_raw'] = rec_result['text_postprocessed']
- state['timestamp'] = rec_result['time_stamp']
- state['sentences'] = rec_result['sentences']
- res_text = rec_result['text']
+ rec_result = self.funasr_model.generate(data, return_spk_res=False, sentence_timestamp=True)
+ res_srt = generate_srt(rec_result[0]['sentence_info'])
+ state['recog_res_raw'] = rec_result[0]['raw_text']
+ state['timestamp'] = rec_result[0]['timestamp']
+ state['sentences'] = rec_result[0]['sentence_info']
+ res_text = rec_result[0]['text']
return res_text, res_srt, state
def clip(self, dest_text, start_ost, end_ost, state, dest_spk=None):
diff --git a/README.md b/README.md
index 144b3a9..9e4df0b 100644
--- a/README.md
+++ b/README.md
@@ -20,11 +20,13 @@ Under the help of ClipVideo you can get the video clips easily with the followin
### Usage
```shell
+# install funasr (necessary)
+git clone https://github.com/alibaba-damo-academy/FunASR.git
+cd FunASR
+pip install -e ./
+# python environments
git clone https://github.com/alibaba-damo-academy/FunASR-APP.git
cd FunASR-APP
-# install modelscope
-pip install "modelscope[audio_asr]" -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html
-# python environments
pip install -r ClipVideo/requirments.txt
```
(Optional) If you want to clip video file with embedded subtitles