Skip to content

FunASR-powered video transcription, subtitle generation, and LLM-assisted clipping tool with a local Gradio UI.

Feat/add litellm provider (#178)

LiteLLM GTM50155e409735
diff --git a/funclip/launch.py b/funclip/launch.py
index e712318..51dbd71 100644
--- a/funclip/launch.py
+++ b/funclip/launch.py
@@ -15,6 +15,7 @@ from videoclipper import VideoClipper
from llm.openai_api import openai_call
from llm.qwen_api import call_qwen_model
from llm.g4f_openai_api import g4f_openai_call
+from llm.litellm_api import litellm_call
from llm.twelvelabs_api import call_twelvelabs_pegasus
from utils.trans_utils import extract_timestamps
from introduction import top_md_1, top_md_3, top_md_4
@@ -165,7 +166,9 @@ if __name__ == "__main__":
)
def llm_inference(system_content, user_content, srt_text, model, apikey, video_input=None):
- SUPPORT_LLM_PREFIX = ['qwen', 'gpt', 'g4f', 'moonshot', 'deepseek', 'pegasus']
+ SUPPORT_LLM_PREFIX = ['litellm', 'qwen', 'gpt', 'g4f', 'moonshot', 'deepseek', 'pegasus']
+ if model.startswith('litellm/'):
+ return litellm_call(apikey, model, user_content+'\n'+srt_text, system_content)
if model.startswith('pegasus'):
# TwelveLabs Pegasus reasons over the actual video (visuals + audio)
# rather than the ASR transcript, so it needs the video source.
@@ -271,10 +274,12 @@ if __name__ == "__main__":
choices=[
"deepseek-chat",
"qwen-plus",
- "gpt-3.5-turbo",
- "gpt-3.5-turbo-0125",
+ "gpt-3.5-turbo",
+ "gpt-3.5-turbo-0125",
"gpt-4-turbo",
"g4f-gpt-3.5-turbo",
+ "litellm/openai/gpt-4o",
+ "litellm/anthropic/claude-sonnet-4-6",
"pegasus1.5"],
value="deepseek-chat",
label="LLM Model Name",
diff --git a/funclip/llm/litellm_api.py b/funclip/llm/litellm_api.py
new file mode 100644
index 0000000..6513437
--- /dev/null
+++ b/funclip/llm/litellm_api.py
@@ -0,0 +1,74 @@
+import logging
+import os
+
+
+def _import_litellm():
+ try:
+ import litellm
+ return litellm
+ except ImportError:
+ raise ImportError(
+ "litellm is required for the litellm provider. "
+ "Install it with: pip install 'litellm>=1.83.0'"
+ )
+
+
+def litellm_call(apikey,
+ model="openai/gpt-3.5-turbo",
+ user_content="How to make tomato beef stew?",
+ system_content=None):
+ litellm = _import_litellm()
+ api_base = os.environ.get("LITELLM_API_BASE", "").strip()
+
+ if model.startswith("litellm/"):
+ model = model[len("litellm/"):]
+
+ if not model:
+ raise ValueError("Model name is empty after stripping litellm/ prefix")
+
+ messages = []
+ if system_content is not None and len(system_content.strip()):
+ messages.append({'role': 'system', 'content': system_content})
+ messages.append({'role': 'user', 'content': user_content})
+
+ kwargs = {
+ 'model': model,
+ 'messages': messages,
+ 'stream': False,
+ 'drop_params': True,
+ }
+ if apikey:
+ kwargs['api_key'] = apikey
+ if api_base:
+ kwargs['api_base'] = api_base
+
+ try:
+ response = litellm.completion(**kwargs)
+ except litellm.AuthenticationError as e:
+ logging.error(f"LiteLLM authentication failed: {e}")
+ raise
+ except litellm.NotFoundError as e:
+ logging.error(f"LiteLLM model not found: {e}")
+ raise
+ except litellm.RateLimitError as e:
+ logging.warning(f"LiteLLM rate limited: {e}")
+ raise
+ except litellm.Timeout as e:
+ logging.error(f"LiteLLM request timed out: {e}")
+ raise
+ except litellm.APIConnectionError as e:
+ logging.error(f"LiteLLM connection error: {e}")
+ raise
+
+ choices = getattr(response, 'choices', None)
+ if not choices:
+ logging.error("LiteLLM returned empty choices")
+ return ""
+
+ content = getattr(choices[0].message, 'content', None)
+ if content is None:
+ logging.warning("LiteLLM returned null content")
+ return ""
+
+ logging.info("LiteLLM model inference done.")
+ return content
diff --git a/tests/test_litellm_api.py b/tests/test_litellm_api.py
new file mode 100644
index 0000000..c8301e1
--- /dev/null
+++ b/tests/test_litellm_api.py
@@ -0,0 +1,137 @@
+"""Tests for the LiteLLM provider module."""
+
+import unittest
+from unittest.mock import MagicMock, patch
+
+
+def _mock_response(content="Test response"):
+ resp = MagicMock()
+ resp.choices = [MagicMock()]
+ resp.choices[0].message.content = content
+ return resp
+
+
+def _empty_choices_response():
+ resp = MagicMock()
+ resp.choices = []
+ return resp
+
+
+def _null_content_response():
+ resp = MagicMock()
+ resp.choices = [MagicMock()]
+ resp.choices[0].message.content = None
+ return resp
+
+
+class TestLiteLLMApi(unittest.TestCase):
+
+ def test_basic_call(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("Summary result")):
+ from funclip.llm.litellm_api import litellm_call
+ result = litellm_call("key", "litellm/openai/gpt-4o-mini", "article text", "Summarize")
+ self.assertEqual(result, "Summary result")
+ call_kwargs = litellm.completion.call_args[1]
+ self.assertEqual(call_kwargs["model"], "openai/gpt-4o-mini")
+
+ def test_prefix_stripped(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("key", "litellm/anthropic/claude-sonnet-4-6", "text")
+ self.assertEqual(litellm.completion.call_args[1]["model"], "anthropic/claude-sonnet-4-6")
+
+ def test_no_prefix_passes_through(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("key", "openai/gpt-4o", "text")
+ self.assertEqual(litellm.completion.call_args[1]["model"], "openai/gpt-4o")
+
+ def test_drop_params_true(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("key", "openai/gpt-4o-mini", "text")
+ self.assertTrue(litellm.completion.call_args[1]["drop_params"])
+
+ def test_api_key_forwarded(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("sk-test-123", "openai/gpt-4o-mini", "text")
+ self.assertEqual(litellm.completion.call_args[1]["api_key"], "sk-test-123")
+
+ def test_api_key_omitted_when_empty(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("", "openai/gpt-4o-mini", "text")
+ self.assertNotIn("api_key", litellm.completion.call_args[1])
+
+ def test_system_content_included(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("key", "openai/gpt-4o-mini", "user text", "system prompt")
+ messages = litellm.completion.call_args[1]["messages"]
+ self.assertEqual(len(messages), 2)
+ self.assertEqual(messages[0]["role"], "system")
+ self.assertEqual(messages[0]["content"], "system prompt")
+
+ def test_system_content_skipped_when_empty(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_mock_response("ok")):
+ from funclip.llm.litellm_api import litellm_call
+ litellm_call("key", "openai/gpt-4o-mini", "user text", " ")
+ messages = litellm.completion.call_args[1]["messages"]
+ self.assertEqual(len(messages), 1)
+ self.assertEqual(messages[0]["role"], "user")
+
+ def test_empty_choices_returns_empty(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_empty_choices_response()):
+ from funclip.llm.litellm_api import litellm_call
+ result = litellm_call("key", "openai/gpt-4o-mini", "text")
+ self.assertEqual(result, "")
+
+ def test_null_content_returns_empty(self):
+ import litellm
+ with patch.object(litellm, "completion", return_value=_null_content_response()):
+ from funclip.llm.litellm_api import litellm_call
+ result = litellm_call("key", "openai/gpt-4o-mini", "text")
+ self.assertEqual(result, "")
+
+ def test_auth_error_raises(self):
+ import litellm
+ with patch.object(litellm, "completion",
+ side_effect=litellm.AuthenticationError(message="Bad key", model="test", llm_provider="openai")):
+ from funclip.llm.litellm_api import litellm_call
+ with self.assertRaises(litellm.AuthenticationError):
+ litellm_call("bad-key", "openai/gpt-4o-mini", "text")
+
+ def test_rate_limit_raises(self):
+ import litellm
+ with patch.object(litellm, "completion",
+ side_effect=litellm.RateLimitError(message="Rate limited", model="test", llm_provider="openai")):
+ from funclip.llm.litellm_api import litellm_call
+ with self.assertRaises(litellm.RateLimitError):
+ litellm_call("key", "openai/gpt-4o-mini", "text")
+
+ def test_timeout_raises(self):
+ import litellm
+ with patch.object(litellm, "completion",
+ side_effect=litellm.Timeout(message="Timed out", model="test", llm_provider="openai")):
+ from funclip.llm.litellm_api import litellm_call
+ with self.assertRaises(litellm.Timeout):
+ litellm_call("key", "openai/gpt-4o-mini", "text")
+
+ def test_empty_model_after_strip_raises(self):
+ from funclip.llm.litellm_api import litellm_call
+ with self.assertRaises(ValueError):
+ litellm_call("key", "litellm/", "text")
+
+
+if __name__ == '__main__':
+ unittest.main()