Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

feat(llm): add TwelveLabs Pegasus as an optional content-aware clipping backend (#165)

mohitfe57e1db468a
diff --git a/README.md b/README.md
index e16a68e..c85252a 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,10 @@ Follow the guide below to explore LLM based clipping:
<img src="docs/images/LLM_guide.png" width=360/>
+#### Content-aware clipping with TwelveLabs Pegasus (optional)
+
+Besides the transcript-based LLMs above, FunClip can optionally use [TwelveLabs](https://twelvelabs.io) Pegasus, a video understanding model that reasons over the actual video (visuals + audio) rather than only the ASR transcript. This helps pick highlight segments even when the transcript alone is ambiguous (e.g. action, scene changes, on-screen events). To use it, select the `pegasus1.5` model name, paste your TwelveLabs API key, upload a video, and click 'LLM Inference' — Pegasus returns segments in the same `N. [start-end] text` format, so the existing 'AI Clip' button works unchanged. It needs `pip install twelvelabs`, and a free API key is available at https://twelvelabs.io.
+
### B. Experience FunClip in Modelscope
[FunClip@Modelscope Space⭐](https://modelscope.cn/studios/iic/funasr_app_clipvideo/summary)
diff --git a/funclip/launch.py b/funclip/launch.py
index f27f164..76e77d7 100644
--- a/funclip/launch.py
+++ b/funclip/launch.py
@@ -15,6 +15,7 @@ from videoclipper import VideoClipper
from llm.openai_api import openai_call
from llm.qwen_api import call_qwen_model
from llm.g4f_openai_api import g4f_openai_call
+from llm.twelvelabs_api import call_twelvelabs_pegasus
from utils.trans_utils import extract_timestamps
from introduction import top_md_1, top_md_3, top_md_4
@@ -158,8 +159,15 @@ if __name__ == "__main__":
add_sub=True, dest_spk=video_spk_input, output_dir=output_dir
)
- def llm_inference(system_content, user_content, srt_text, model, apikey):
- SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot', 'deepseek']
+ def llm_inference(system_content, user_content, srt_text, model, apikey, video_input=None):
+ SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot', 'deepseek', 'pegasus']
+ if model.startswith('pegasus'):
+ # TwelveLabs Pegasus reasons over the actual video (visuals + audio)
+ # rather than the ASR transcript, so it needs the video source.
+ if video_input is None:
+ logging.error("Pegasus requires a video input; please upload a video first.")
+ return "Please upload a video before running Pegasus inference."
+ return call_twelvelabs_pegasus(apikey, video_input, model=model, prompt=system_content)
if model.startswith('qwen'):
return call_qwen_model(apikey, model, user_content+'\n'+srt_text, system_content)
if model.startswith('gpt') or model.startswith('moonshot') or model.startswith('deepseek'):
@@ -261,7 +269,8 @@ if __name__ == "__main__":
"gpt-3.5-turbo",
"gpt-3.5-turbo-0125",
"gpt-4-turbo",
- "g4f-gpt-3.5-turbo"],
+ "g4f-gpt-3.5-turbo",
+ "pegasus1.5"],
value="deepseek-chat",
label="LLM Model Name",
allow_custom_value=True)
@@ -325,7 +334,7 @@ if __name__ == "__main__":
],
outputs=[video_output, clip_message, srt_clipped])
llm_button.click(llm_inference,
- inputs=[prompt_head, prompt_head2, video_srt_output, llm_model, apikey_input],
+ inputs=[prompt_head, prompt_head2, video_srt_output, llm_model, apikey_input, video_input],
outputs=[llm_result])
llm_clip_button.click(AI_clip,
inputs=[llm_result,
diff --git a/funclip/llm/twelvelabs_api.py b/funclip/llm/twelvelabs_api.py
new file mode 100644
index 0000000..e3a9cc8
--- /dev/null
+++ b/funclip/llm/twelvelabs_api.py
@@ -0,0 +1,91 @@
+import os
+import time
+import logging
+
+
+# Default instruction asking Pegasus to return segments in the same
+# "N. [start-end] text" format that FunClip's extract_timestamps() parses,
+# so the existing AI_clip pipeline works unchanged.
+PEGASUS_SYSTEM_PROMPT = (
+ "You are a video clipping assistant. Watch the video and select up to four "
+ "of the most engaging, self-contained highlight segments based on what "
+ "actually happens on screen and what is said. Merge consecutive moments "
+ "into a single segment. Reply strictly in this format, one segment per line:\n"
+ "1. [start_time-end_time] description\n"
+ "where start_time and end_time are seconds from the start of the video "
+ "(e.g. 12.5), the connector is '-', and description is a short summary."
+)
+
+
+def _resolve_video_context(client, video):
+ """Return a VideoContext for a public URL or a local file path.
+
+ Local files are uploaded as a TwelveLabs asset first; remote URLs are
+ passed through directly.
+ """
+ from twelvelabs.types.video_context import VideoContext_Url, VideoContext_AssetId
+
+ if video.startswith("http://") or video.startswith("https://"):
+ return VideoContext_Url(url=video)
+
+ if not os.path.isfile(video):
+ raise FileNotFoundError("Video source not found: {}".format(video))
+
+ with open(video, "rb") as f:
+ asset = client.assets.create(method="direct", file=f)
+ # Wait for the uploaded asset to finish processing before analysis.
+ deadline = time.time() + 300
+ while asset.status == "processing" and time.time() < deadline:
+ time.sleep(5)
+ asset = client.assets.retrieve(asset_id=asset.id)
+ if asset.status != "ready":
+ raise RuntimeError(
+ "TwelveLabs asset {} not ready (status={}).".format(asset.id, asset.status)
+ )
+ return VideoContext_AssetId(asset_id=asset.id)
+
+
+def call_twelvelabs_pegasus(apikey,
+ video,
+ model="pegasus1.5",
+ prompt=None,
+ max_tokens=2048):
+ """Run TwelveLabs Pegasus content-aware analysis over a video.
+
+ Unlike the ASR-text based LLM backends, Pegasus reasons over the actual
+ video (visuals + audio), which lets it pick highlight segments even when
+ the transcript alone is ambiguous. Returns Pegasus' text response, which
+ is expected to follow the "N. [start-end] text" segment format.
+
+ Args:
+ apikey: TwelveLabs API key (https://twelvelabs.io, free tier available).
+ video: a public video URL or a local video file path.
+ model: Pegasus model name, e.g. "pegasus1.5".
+ prompt: optional extra instruction appended to the default prompt.
+ max_tokens: max tokens for the Pegasus response.
+ """
+ from twelvelabs import TwelveLabs
+
+ client = TwelveLabs(api_key=apikey or os.environ.get("TWELVELABS_API_KEY"))
+ video_context = _resolve_video_context(client, video)
+
+ full_prompt = PEGASUS_SYSTEM_PROMPT
+ if prompt is not None and len(prompt.strip()):
+ full_prompt = full_prompt + "\n\n" + prompt.strip()
+
+ response = client.analyze(
+ model_name=model,
+ video=video_context,
+ prompt=full_prompt,
+ max_tokens=max_tokens,
+ )
+ logging.info("TwelveLabs Pegasus inference done.")
+ return response.data
+
+
+if __name__ == '__main__':
+ res = call_twelvelabs_pegasus(
+ os.environ.get("TWELVELABS_API_KEY"),
+ "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_1MB.mp4",
+ )
+ print(res)
diff --git a/requirements.txt b/requirements.txt
index a5a66b5..c5aaa7b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,3 +12,4 @@ openai
g4f
dashscope
curl_cffi
+twelvelabs>=1.2.8
diff --git a/tests/test_twelvelabs_pegasus.py b/tests/test_twelvelabs_pegasus.py
new file mode 100644
index 0000000..9406d14
--- /dev/null
+++ b/tests/test_twelvelabs_pegasus.py
@@ -0,0 +1,48 @@
+import os
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "funclip"))
+
+from llm.twelvelabs_api import (
+ _resolve_video_context,
+ call_twelvelabs_pegasus,
+ PEGASUS_SYSTEM_PROMPT,
+)
+
+
+class TestResolveVideoContext(unittest.TestCase):
+ """No-network checks for the video-source routing logic."""
+
+ def test_http_url_becomes_url_context_without_upload(self):
+ # client is unused on the URL path; pass None to prove no upload happens.
+ ctx = _resolve_video_context(None, "https://example.com/clip.mp4")
+ self.assertEqual(ctx.type, "url")
+ self.assertEqual(ctx.url, "https://example.com/clip.mp4")
+
+ def test_missing_local_file_raises(self):
+ with self.assertRaises(FileNotFoundError):
+ _resolve_video_context(None, "/no/such/video.mp4")
+
+ def test_default_prompt_requests_parseable_segment_format(self):
+ self.assertIn("[start_time-end_time]", PEGASUS_SYSTEM_PROMPT)
+
+
+ os.environ.get("TWELVELABS_API_KEY"),
+ "set TWELVELABS_API_KEY to run the live Pegasus smoke test",
+)
+class TestPegasusLive(unittest.TestCase):
+ def test_analyze_public_url_returns_text(self):
+ res = call_twelvelabs_pegasus(
+ os.environ["TWELVELABS_API_KEY"],
+ "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/360/Big_Buck_Bunny_360_10s_1MB.mp4",
+ )
+ self.assertIsInstance(res, str)
+ self.assertTrue(len(res) > 0)
+
+
+if __name__ == "__main__":
+ unittest.main()