FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.
Python ai-toolsai-video-editingasrauto-subtitleschinesecontent-creationfunasrfunclipgradiollmparaformerspeech-recognitionspeech-to-textsubtitles-generatortranscriptionvideo-editingvideo-processingvideo-subtitlesvideo-transcriptionwhisper-alternative
feat(llm): add TwelveLabs Pegasus as an optional content-aware clipping backend (#165)
mohitfe57e1db468a
diff --git a/README.md b/README.mdindex e16a68e..c85252a 100644--- a/README.md+++ b/README.md@@ -127,6 +127,10 @@ Follow the guide below to explore LLM based clipping:<img src="docs/images/LLM_guide.png" width=360/>+#### Content-aware clipping with TwelveLabs Pegasus (optional)++Besides the transcript-based LLMs above, FunClip can optionally use [TwelveLabs](https://twelvelabs.io) Pegasus, a video understanding model that reasons over the actual video (visuals + audio) rather than only the ASR transcript. This helps pick highlight segments even when the transcript alone is ambiguous (e.g. action, scene changes, on-screen events). To use it, select the `pegasus1.5` model name, paste your TwelveLabs API key, upload a video, and click 'LLM Inference' — Pegasus returns segments in the same `N. [start-end] text` format, so the existing 'AI Clip' button works unchanged. It needs `pip install twelvelabs`, and a free API key is available at https://twelvelabs.io.+### B. Experience FunClip in Modelscope[FunClip@Modelscope Space⭐](https://modelscope.cn/studios/iic/funasr_app_clipvideo/summary)diff --git a/funclip/launch.py b/funclip/launch.pyindex f27f164..76e77d7 100644--- a/funclip/launch.py+++ b/funclip/launch.py@@ -15,6 +15,7 @@ from videoclipper import VideoClipperfrom llm.openai_api import openai_callfrom llm.qwen_api import call_qwen_modelfrom llm.g4f_openai_api import g4f_openai_call+from llm.twelvelabs_api import call_twelvelabs_pegasusfrom utils.trans_utils import extract_timestampsfrom introduction import top_md_1, top_md_3, top_md_4@@ -158,8 +159,15 @@ if __name__ == "__main__":add_sub=True, dest_spk=video_spk_input, output_dir=output_dir)- def llm_inference(system_content, user_content, srt_text, model, apikey):- SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot', 'deepseek']+ def llm_inference(system_content, user_content, srt_text, model, apikey, video_input=None):+ SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot', 'deepseek', 'pegasus']+ if model.startswith('pegasus'):+ # TwelveLabs Pegasus reasons over the actual video (visuals + audio)+ # rather than the ASR transcript, so it needs the video source.+ if video_input is None:+ logging.error("Pegasus requires a video input; please upload a video first.")+ return "Please upload a video before running Pegasus inference."+ return call_twelvelabs_pegasus(apikey, video_input, model=model, prompt=system_content)if model.startswith('qwen'):return call_qwen_model(apikey, model, user_content+'\n'+srt_text, system_content)if model.startswith('gpt') or model.startswith('moonshot') or model.startswith('deepseek'):@@ -261,7 +269,8 @@ if __name__ == "__main__":"gpt-3.5-turbo","gpt-3.5-turbo-0125","gpt-4-turbo",- "g4f-gpt-3.5-turbo"],+ "g4f-gpt-3.5-turbo",+ "pegasus1.5"],value="deepseek-chat",label="LLM Model Name",allow_custom_value=True)@@ -325,7 +334,7 @@ if __name__ == "__main__":],outputs=[video_output, clip_message, srt_clipped])llm_button.click(llm_inference,- inputs=[prompt_head, prompt_head2, video_srt_output, llm_model, apikey_input],+ inputs=[prompt_head, prompt_head2, video_srt_output, llm_model, apikey_input, video_input],outputs=[llm_result])llm_clip_button.click(AI_clip,inputs=[llm_result,diff --git a/funclip/llm/twelvelabs_api.py b/funclip/llm/twelvelabs_api.pynew file mode 100644index 0000000..e3a9cc8--- /dev/null+++ b/funclip/llm/twelvelabs_api.py@@ -0,0 +1,91 @@+import os+import time+import logging+++# Default instruction asking Pegasus to return segments in the same+# "N. [start-end] text" format that FunClip's extract_timestamps() parses,+# so the existing AI_clip pipeline works unchanged.+PEGASUS_SYSTEM_PROMPT = (+ "You are a video clipping assistant. Watch the video and select up to four "+ "of the most engaging, self-contained highlight segments based on what "+ "actually happens on screen and what is said. Merge consecutive moments "+ "into a single segment. Reply strictly in this format, one segment per line:\n"+ "1. [start_time-end_time] description\n"+ "where start_time and end_time are seconds from the start of the video "+ "(e.g. 12.5), the connector is '-', and description is a short summary."+)+++def _resolve_video_context(client, video):+ """Return a VideoContext for a public URL or a local file path.++ Local files are uploaded as a TwelveLabs asset first; remote URLs are+ passed through directly.+ """+ from twelvelabs.types.video_context import VideoContext_Url, VideoContext_AssetId++ if video.startswith("http://") or video.startswith("https://"):+ return VideoContext_Url(url=video)++ if not os.path.isfile(video):+ raise FileNotFoundError("Video source not found: {}".format(video))++ with open(video, "rb") as f:+ asset = client.assets.create(method="direct", file=f)+ # Wait for the uploaded asset to finish processing before analysis.+ deadline = time.time() + 300+ while asset.status == "processing" and time.time() < deadline:+ time.sleep(5)+ asset = client.assets.retrieve(asset_id=asset.id)+ if asset.status != "ready":+ raise RuntimeError(+ "TwelveLabs asset {} not ready (status={}).".format(asset.id, asset.status)+ )+ return VideoContext_AssetId(asset_id=asset.id)+++def call_twelvelabs_pegasus(apikey,+ video,+ model="pegasus1.5",+ prompt=None,+ max_tokens=2048):+ """Run TwelveLabs Pegasus content-aware analysis over a video.++ Unlike the ASR-text based LLM backends, Pegasus reasons over the actual+ video (visuals + audio), which lets it pick highlight segments even when+ the transcript alone is ambiguous. Returns Pegasus' text response, which+ is expected to follow the "N. [start-end] text" segment format.++ Args:+ apikey: TwelveLabs API key (https://twelvelabs.io, free tier available).+ video: a public video URL or a local video file path.+ model: Pegasus model name, e.g. "pegasus1.5".+ prompt: optional extra instruction appended to the default prompt.+ max_tokens: max tokens for the Pegasus response.+ """+ from twelvelabs import TwelveLabs++ client = TwelveLabs(api_key=apikey or os.environ.get("TWELVELABS_API_KEY"))+ video_context = _resolve_video_context(client, video)++ full_prompt = PEGASUS_SYSTEM_PROMPT+ if prompt is not None and len(prompt.strip()):+ full_prompt = full_prompt + "\n\n" + prompt.strip()++ response = client.analyze(+ model_name=model,+ video=video_context,+ prompt=full_prompt,+ max_tokens=max_tokens,+ )+ logging.info("TwelveLabs Pegasus inference done.")+ return response.data+++if __name__ == '__main__':+ res = call_twelvelabs_pegasus(+ os.environ.get("TWELVELABS_API_KEY"),+ "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_1MB.mp4",+ )+ print(res)diff --git a/requirements.txt b/requirements.txtindex a5a66b5..c5aaa7b 100644--- a/requirements.txt+++ b/requirements.txt@@ -12,3 +12,4 @@ openaig4fdashscopecurl_cffi+twelvelabs>=1.2.8diff --git a/tests/test_twelvelabs_pegasus.py b/tests/test_twelvelabs_pegasus.pynew file mode 100644index 0000000..9406d14--- /dev/null+++ b/tests/test_twelvelabs_pegasus.py@@ -0,0 +1,48 @@+import os+import sys+import unittest+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT / "funclip"))++from llm.twelvelabs_api import (+ _resolve_video_context,+ call_twelvelabs_pegasus,+ PEGASUS_SYSTEM_PROMPT,+)+++class TestResolveVideoContext(unittest.TestCase):+ """No-network checks for the video-source routing logic."""++ def test_http_url_becomes_url_context_without_upload(self):+ # client is unused on the URL path; pass None to prove no upload happens.+ ctx = _resolve_video_context(None, "https://example.com/clip.mp4")+ self.assertEqual(ctx.type, "url")+ self.assertEqual(ctx.url, "https://example.com/clip.mp4")++ def test_missing_local_file_raises(self):+ with self.assertRaises(FileNotFoundError):+ _resolve_video_context(None, "/no/such/video.mp4")++ def test_default_prompt_requests_parseable_segment_format(self):+ self.assertIn("[start_time-end_time]", PEGASUS_SYSTEM_PROMPT)+++ os.environ.get("TWELVELABS_API_KEY"),+ "set TWELVELABS_API_KEY to run the live Pegasus smoke test",+)+class TestPegasusLive(unittest.TestCase):+ def test_analyze_public_url_returns_text(self):+ res = call_twelvelabs_pegasus(+ os.environ["TWELVELABS_API_KEY"],+ "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_1MB.mp4",+ )+ self.assertIsInstance(res, str)+ self.assertTrue(len(res) > 0)+++if __name__ == "__main__":+ unittest.main()