Skip to content

A programming framework for agentic AI

Fix client mismatch in video surfer and its tools (#4431)

gagbb1e40017c31d
diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.toml
index a5c5a28d0..9da90d577 100644
--- a/python/packages/autogen-ext/pyproject.toml
+++ b/python/packages/autogen-ext/pyproject.toml
@@ -33,7 +33,6 @@ web-surfer = [
]
video-surfer = [
"autogen-agentchat==0.4.0.dev8",
- "openai",
"opencv-python>=4.5",
"ffmpeg-python",
"openai-whisper",
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py
index 8f9901133..8f8bb276e 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py
@@ -8,9 +8,9 @@ from .tools import (
extract_audio,
get_screenshot_at,
get_video_length,
- openai_transcribe_video_screenshot,
save_screenshot,
transcribe_audio_with_timestamps,
+ transcribe_video_screenshot,
)
@@ -27,7 +27,7 @@ class VideoSurferAgent(AssistantAgent):
- :func:`~autogen_ext.agents.video_surfer.tools.transcribe_audio_with_timestamps`
- :func:`~autogen_ext.agents.video_surfer.tools.get_screenshot_at`
- :func:`~autogen_ext.agents.video_surfer.tools.save_screenshot`
- - :func:`~autogen_ext.agents.video_surfer.tools.openai_transcribe_video_screenshot`
+ - :func:`~autogen_ext.agents.video_surfer.tools.transcribe_video_screenshot`
Example usage:
@@ -145,10 +145,23 @@ class VideoSurferAgent(AssistantAgent):
get_video_length,
get_screenshot_at,
save_screenshot,
- openai_transcribe_video_screenshot,
+ self.vs_transribe_video_screenshot,
extract_audio,
transcribe_audio_with_timestamps,
],
description=description or self.DEFAULT_DESCRIPTION,
system_message=system_message or self.DEFAULT_SYSTEM_MESSAGE,
)
+
+ async def vs_transribe_video_screenshot(self, video_path: str, timestamp: float) -> str:
+ """
+ Transcribes the video screenshot at a specific timestamp.
+
+ Args:
+ video_path (str): Path to the video file.
+ timestamp (float): Timestamp to take the screenshot.
+
+ Returns:
+ str: Transcription of the video screenshot.
+ """
+ return await transcribe_video_screenshot(video_path, timestamp, self._model_client)
diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py
index 076733e64..bbba4b79a 100644
--- a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py
+++ b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py
@@ -4,8 +4,12 @@ from typing import Any, Dict, List, Tuple
import cv2
import ffmpeg
import numpy as np
-import openai
import whisper
+from autogen_core.components import Image as AGImage
+from autogen_core.components.models import (
+ ChatCompletionClient,
+ UserMessage,
+)
def extract_audio(video_path: str, audio_output_path: str) -> str:
@@ -66,7 +70,7 @@ def save_screenshot(video_path: str, timestamp: float, output_path: str) -> None
:param video_path: Path to the video file.
:param timestamp: Timestamp in seconds.
- :param output_path: Path to save the screenshot.
+ :param output_path: Path to save the screenshot. The file format is determined by the extension in the path.
"""
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -82,12 +86,13 @@ def save_screenshot(video_path: str, timestamp: float, output_path: str) -> None
cap.release()
-def openai_transcribe_video_screenshot(video_path: str, timestamp: float) -> str:
+async def transcribe_video_screenshot(video_path: str, timestamp: float, model_client: ChatCompletionClient) -> str:
"""
Transcribes the content of a video screenshot captured at the specified timestamp using OpenAI API.
:param video_path: Path to the video file.
:param timestamp: Timestamp in seconds.
+ :param model_client: ChatCompletionClient instance.
:return: Description of the screenshot content.
"""
screenshots = get_screenshot_at(video_path, [timestamp])
@@ -99,31 +104,20 @@ def openai_transcribe_video_screenshot(video_path: str, timestamp: float) -> str
_, buffer = cv2.imencode(".jpg", frame)
frame_bytes = buffer.tobytes()
frame_base64 = base64.b64encode(frame_bytes).decode("utf-8")
-
- client = openai.Client()
-
- response = client.chat.completions.create(
- model="gpt-4o-mini",
- messages=[
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "Following is a screenshot from the video at {} seconds. Describe what you see here.".format(
- timestamp
- ),
- },
- {
- "type": "image_url",
- "image_url": {"url": f"data:image/jpeg;base64,{frame_base64}"},
- },
- ],
- }
- ],
- )
-
- return str(response.choices[0].message.content)
+ screenshot_uri = f"data:image/jpeg;base64,{frame_base64}"
+
+ messages = [
+ UserMessage(
+ content=[
+ "Following is a screenshot from the video at {} seconds. Describe what you see here.",
+ AGImage.from_uri(screenshot_uri),
+ ],
+ source="tool",
+ )
+ ]
+
+ result = await model_client.create(messages=messages)
+ return str(result.content)
def get_screenshot_at(video_path: str, timestamps: List[float]) -> List[Tuple[float, np.ndarray[Any, Any]]]:
diff --git a/python/uv.lock b/python/uv.lock
index 79850a649..3662a7900 100644
--- a/python/uv.lock
+++ b/python/uv.lock
@@ -508,7 +508,6 @@ openai = [
video-surfer = [
{ name = "autogen-agentchat" },
{ name = "ffmpeg-python" },
- { name = "openai" },
{ name = "openai-whisper" },
{ name = "opencv-python" },
]
@@ -530,7 +529,6 @@ requires-dist = [
{ name = "langchain-core", marker = "extra == 'langchain'", specifier = "~=0.3.3" },
{ name = "langchain-core", marker = "extra == 'langchain-tools'", specifier = "~=0.3.3" },
{ name = "openai", marker = "extra == 'openai'", specifier = ">=1.3" },
- { name = "openai", marker = "extra == 'video-surfer'" },
{ name = "openai-whisper", marker = "extra == 'video-surfer'" },
{ name = "opencv-python", marker = "extra == 'video-surfer'", specifier = ">=4.5" },
{ name = "pillow", marker = "extra == 'web-surfer'", specifier = ">=11.0.0" },