A programming framework for agentic AI
Fix client mismatch in video surfer and its tools (#4431)
gagbb1e40017c31d
diff --git a/python/packages/autogen-ext/pyproject.toml b/python/packages/autogen-ext/pyproject.tomlindex a5c5a28d0..9da90d577 100644--- a/python/packages/autogen-ext/pyproject.toml+++ b/python/packages/autogen-ext/pyproject.toml@@ -33,7 +33,6 @@ web-surfer = []video-surfer = ["autogen-agentchat==0.4.0.dev8",- "openai","opencv-python>=4.5","ffmpeg-python","openai-whisper",diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.pyindex 8f9901133..8f8bb276e 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/_video_surfer.py@@ -8,9 +8,9 @@ from .tools import (extract_audio,get_screenshot_at,get_video_length,- openai_transcribe_video_screenshot,save_screenshot,transcribe_audio_with_timestamps,+ transcribe_video_screenshot,)@@ -27,7 +27,7 @@ class VideoSurferAgent(AssistantAgent):- :func:`~autogen_ext.agents.video_surfer.tools.transcribe_audio_with_timestamps`- :func:`~autogen_ext.agents.video_surfer.tools.get_screenshot_at`- :func:`~autogen_ext.agents.video_surfer.tools.save_screenshot`- - :func:`~autogen_ext.agents.video_surfer.tools.openai_transcribe_video_screenshot`+ - :func:`~autogen_ext.agents.video_surfer.tools.transcribe_video_screenshot`Example usage:@@ -145,10 +145,23 @@ class VideoSurferAgent(AssistantAgent):get_video_length,get_screenshot_at,save_screenshot,- openai_transcribe_video_screenshot,+ self.vs_transribe_video_screenshot,extract_audio,transcribe_audio_with_timestamps,],description=description or self.DEFAULT_DESCRIPTION,system_message=system_message or self.DEFAULT_SYSTEM_MESSAGE,)++ async def vs_transribe_video_screenshot(self, video_path: str, timestamp: float) -> str:+ """+ Transcribes the video screenshot at a specific timestamp.++ Args:+ video_path (str): Path to the video file.+ timestamp (float): Timestamp to take the screenshot.++ Returns:+ str: Transcription of the video screenshot.+ """+ return await transcribe_video_screenshot(video_path, timestamp, self._model_client)diff --git a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.pyindex 076733e64..bbba4b79a 100644--- a/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py+++ b/python/packages/autogen-ext/src/autogen_ext/agents/video_surfer/tools.py@@ -4,8 +4,12 @@ from typing import Any, Dict, List, Tupleimport cv2import ffmpegimport numpy as np-import openaiimport whisper+from autogen_core.components import Image as AGImage+from autogen_core.components.models import (+ ChatCompletionClient,+ UserMessage,+)def extract_audio(video_path: str, audio_output_path: str) -> str:@@ -66,7 +70,7 @@ def save_screenshot(video_path: str, timestamp: float, output_path: str) -> None:param video_path: Path to the video file.:param timestamp: Timestamp in seconds.- :param output_path: Path to save the screenshot.+ :param output_path: Path to save the screenshot. The file format is determined by the extension in the path."""cap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -82,12 +86,13 @@ def save_screenshot(video_path: str, timestamp: float, output_path: str) -> Nonecap.release()-def openai_transcribe_video_screenshot(video_path: str, timestamp: float) -> str:+async def transcribe_video_screenshot(video_path: str, timestamp: float, model_client: ChatCompletionClient) -> str:"""Transcribes the content of a video screenshot captured at the specified timestamp using OpenAI API.:param video_path: Path to the video file.:param timestamp: Timestamp in seconds.+ :param model_client: ChatCompletionClient instance.:return: Description of the screenshot content."""screenshots = get_screenshot_at(video_path, [timestamp])@@ -99,31 +104,20 @@ def openai_transcribe_video_screenshot(video_path: str, timestamp: float) -> str_, buffer = cv2.imencode(".jpg", frame)frame_bytes = buffer.tobytes()frame_base64 = base64.b64encode(frame_bytes).decode("utf-8")-- client = openai.Client()-- response = client.chat.completions.create(- model="gpt-4o-mini",- messages=[- {- "role": "user",- "content": [- {- "type": "text",- "text": "Following is a screenshot from the video at {} seconds. Describe what you see here.".format(- timestamp- ),- },- {- "type": "image_url",- "image_url": {"url": f"data:image/jpeg;base64,{frame_base64}"},- },- ],- }- ],- )-- return str(response.choices[0].message.content)+ screenshot_uri = f"data:image/jpeg;base64,{frame_base64}"++ messages = [+ UserMessage(+ content=[+ "Following is a screenshot from the video at {} seconds. Describe what you see here.",+ AGImage.from_uri(screenshot_uri),+ ],+ source="tool",+ )+ ]++ result = await model_client.create(messages=messages)+ return str(result.content)def get_screenshot_at(video_path: str, timestamps: List[float]) -> List[Tuple[float, np.ndarray[Any, Any]]]:diff --git a/python/uv.lock b/python/uv.lockindex 79850a649..3662a7900 100644--- a/python/uv.lock+++ b/python/uv.lock@@ -508,7 +508,6 @@ openai = [video-surfer = [{ name = "autogen-agentchat" },{ name = "ffmpeg-python" },- { name = "openai" },{ name = "openai-whisper" },{ name = "opencv-python" },]@@ -530,7 +529,6 @@ requires-dist = [{ name = "langchain-core", marker = "extra == 'langchain'", specifier = "~=0.3.3" },{ name = "langchain-core", marker = "extra == 'langchain-tools'", specifier = "~=0.3.3" },{ name = "openai", marker = "extra == 'openai'", specifier = ">=1.3" },- { name = "openai", marker = "extra == 'video-surfer'" },{ name = "openai-whisper", marker = "extra == 'video-surfer'" },{ name = "opencv-python", marker = "extra == 'video-surfer'", specifier = ">=4.5" },{ name = "pillow", marker = "extra == 'web-surfer'", specifier = ">=11.0.0" },