🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor: video tookit (#1852)
Jacklee4ba85d8d0a93
diff --git a/camel/toolkits/video_analysis_toolkit.py b/camel/toolkits/video_analysis_toolkit.pyindex 2ae072311..828de4d16 100644--- a/camel/toolkits/video_analysis_toolkit.py+++ b/camel/toolkits/video_analysis_toolkit.py@@ -12,6 +12,7 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import ioimport osimport tempfilefrom pathlib import Path@@ -92,6 +93,10 @@ class VideoAnalysisToolkit(BaseToolkit):transcription using OpenAI's audio models. Requires a valid OpenAIAPI key. When disabled, video analysis will be based solely onvisual content. (default: :obj:`False`)+ frame_interval (float, optional): Interval in seconds between frames+ to extract from the video. (default: :obj:`4.0`)+ output_language (str, optional): The language for output responses.+ (default: :obj:`"English"`)timeout (Optional[float]): The timeout value for API requestsin seconds. If None, no timeout is applied.(default: :obj:`None`)@@ -103,12 +108,16 @@ class VideoAnalysisToolkit(BaseToolkit):download_directory: Optional[str] = None,model: Optional[BaseModelBackend] = None,use_audio_transcription: bool = False,+ frame_interval: float = 4.0,+ output_language: str = "English",timeout: Optional[float] = None,) -> None:super().__init__(timeout=timeout)self._cleanup = download_directory is Noneself._temp_files: list[str] = [] # Track temporary files for cleanupself._use_audio_transcription = use_audio_transcription+ self.output_language = output_language+ self.frame_interval = frame_intervalself._download_directory = Path(download_directory or tempfile.mkdtemp()@@ -137,13 +146,16 @@ class VideoAnalysisToolkit(BaseToolkit):# Import ChatAgent at runtime to avoid circular importsfrom camel.agents import ChatAgent- self.vl_agent = ChatAgent(model=self.vl_model)+ self.vl_model._timeout = timeout+ self.vl_agent = ChatAgent(+ model=self.vl_model, output_language=self.output_language+ )else:# If no model is provided, use default model in ChatAgent# Import ChatAgent at runtime to avoid circular importsfrom camel.agents import ChatAgent- self.vl_agent = ChatAgent()+ self.vl_agent = ChatAgent(output_language=self.output_language)logger.warning("No vision-language model provided. Using default model in"" ChatAgent."@@ -179,12 +191,18 @@ class VideoAnalysisToolkit(BaseToolkit):# Clean up temporary directory if neededif self._cleanup and os.path.exists(self._download_directory):try:- import shutil+ import sys- shutil.rmtree(self._download_directory)- logger.debug(- f"Removed temporary directory: {self._download_directory}"- )+ if getattr(sys, 'modules', None) is not None:+ import shutil++ shutil.rmtree(self._download_directory)+ logger.debug(+ f"Removed temp directory: {self._download_directory}"+ )+ except (ImportError, AttributeError):+ # Skip cleanup if interpreter is shutting down+ passexcept OSError as e:logger.warning(f"Failed to remove temporary directory"@@ -243,87 +261,212 @@ class VideoAnalysisToolkit(BaseToolkit):return "Audio transcription failed."def _extract_keyframes(- self, video_path: str, num_frames: int, threshold: float = 25.0+ self, video_path: str, threshold: float = 25.0) -> List[Image.Image]:- r"""Extract keyframes from a video based on scene changes- and return them as PIL.Image.Image objects.+ r"""Extract keyframes from a video based on scene changes and+ regular intervals,and return them as PIL.Image.Image objects.Args:video_path (str): Path to the video file.- num_frames (int): Number of keyframes to extract.threshold (float): The threshold value for scene change detection.+ This is kept for backward compatibility but not used in the+ current implementation.Returns:list: A list of PIL.Image.Image objects representingthe extracted keyframes."""+ import cv2+ import numpy as npfrom scenedetect import ( # type: ignore[import-untyped]SceneManager,- VideoManager,+ open_video,)from scenedetect.detectors import ( # type: ignore[import-untyped]ContentDetector,)- if num_frames <= 0:+ # Get video information+ cap = cv2.VideoCapture(video_path)+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ fps = cap.get(cv2.CAP_PROP_FPS)+ duration = total_frames / fps if fps > 0 else 0+ cap.release()++ frame_interval = self.frame_interval # seconds++ # Calculate the total number of frames to extract+ if duration <= 0 or fps <= 0:logger.warning(- f"Invalid num_frames: {num_frames}, using default of 1"+ "Invalid video duration or fps, using default frame count")- num_frames = 1+ num_frames = 10+ else:+ num_frames = max(int(duration / frame_interval), 1)- video_manager = VideoManager([video_path])+ max_frames = 100+ if num_frames > max_frames:+ frame_interval = duration / max_frames+ num_frames = max_frames++ logger.info(+ f"Video duration: {duration:.2f}s, target frames: {num_frames}"+ f"at {frame_interval:.2f}s intervals"+ )++ # Use scene detection to extract keyframes+ # Use open_video instead of VideoManager+ video = open_video(video_path)scene_manager = SceneManager()scene_manager.add_detector(ContentDetector(threshold=threshold))- video_manager.set_duration()- video_manager.start()- scene_manager.detect_scenes(video_manager)+ # Detect scenes using the modern API+ scene_manager.detect_scenes(video)scenes = scene_manager.get_scene_list()keyframes: List[Image.Image] = []- # Handle case where no scenes are detected- if not scenes:+ # If scene detection is successful, prioritize scene change points+ if scenes:+ logger.info(f"Detected {len(scenes)} scene changes")++ if len(scenes) > num_frames:+ scene_indices = np.linspace(+ 0, len(scenes) - 1, num_frames, dtype=int+ )+ selected_scenes = [scenes[i] for i in scene_indices]+ else:+ selected_scenes = scenes++ # Extract frames from scenes+ for scene in selected_scenes:+ try:+ # Get start time in seconds+ start_time = scene[0].get_seconds()+ frame = _capture_screenshot(video_path, start_time)+ keyframes.append(frame)+ except Exception as e:+ logger.warning(+ f"Failed to capture frame at scene change"+ f" {scene[0].get_seconds()}s: {e}"+ )++ if len(keyframes) < num_frames and duration > 0:+ logger.info(+ f"Scene detection provided {len(keyframes)} frames, "+ f"supplementing with regular interval frames"+ )++ existing_times = []+ if scenes:+ existing_times = [scene[0].get_seconds() for scene in scenes]++ regular_frames = []+ for i in range(num_frames):+ time_sec = i * frame_interval++ is_duplicate = False+ for existing_time in existing_times:+ if abs(existing_time - time_sec) < 1.0:+ is_duplicate = True+ break++ if not is_duplicate:+ try:+ frame = _capture_screenshot(video_path, time_sec)+ regular_frames.append(frame)+ except Exception as e:+ logger.warning(+ f"Failed to capture frame at {time_sec}s: {e}"+ )++ frames_needed = num_frames - len(keyframes)+ if frames_needed > 0 and regular_frames:+ if len(regular_frames) > frames_needed:+ indices = np.linspace(+ 0, len(regular_frames) - 1, frames_needed, dtype=int+ )+ selected_frames = [regular_frames[i] for i in indices]+ else:+ selected_frames = regular_frames++ keyframes.extend(selected_frames)++ if not keyframes:logger.warning(- "No scenes detected in video, capturing frames at "- "regular intervals"+ "No frames extracted, falling back to simple interval"+ "extraction")- import cv2-- cap = cv2.VideoCapture(video_path)- total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))- fps = cap.get(cv2.CAP_PROP_FPS)- duration = total_frames / fps if fps > 0 else 0-- if duration > 0 and total_frames > 0:- # Extract frames at regular intervals- interval = duration / min(num_frames, total_frames)- for i in range(min(num_frames, total_frames)):- time_sec = i * interval+ for i in range(+ min(num_frames, 10)+ ): # Limit to a maximum of 10 frames to avoid infinite loops+ time_sec = i * (duration / 10 if duration > 0 else 6.0)+ try:frame = _capture_screenshot(video_path, time_sec)keyframes.append(frame)-- cap.release()- else:- # Extract frames from detected scenes- for start_time, _ in scenes:- if len(keyframes) >= num_frames:- break- frame = _capture_screenshot(video_path, start_time)- keyframes.append(frame)+ except Exception as e:+ logger.warning(+ f"Failed to capture frame at {time_sec}s: {e}"+ )if not keyframes:logger.error("Failed to extract any keyframes from video")raise ValueError("Failed to extract keyframes from video")- logger.info(f"Extracted {len(keyframes)} keyframes")- return keyframes+ # Normalize image sizes+ normalized_keyframes = self._normalize_frames(keyframes)++ logger.info(+ f"Extracted and normalized {len(normalized_keyframes)} keyframes"+ )+ return normalized_keyframes++ def _normalize_frames(+ self, frames: List[Image.Image], target_width: int = 512+ ) -> List[Image.Image]:+ r"""Normalize the size of extracted frames.++ Args:+ frames (List[Image.Image]): List of frames to normalize.+ target_width (int): Target width for normalized frames.++ Returns:+ List[Image.Image]: List of normalized frames.+ """+ normalized_frames: List[Image.Image] = []++ for frame in frames:+ # Get original dimensions+ width, height = frame.size++ # Calculate new height, maintaining aspect ratio+ aspect_ratio = width / height+ new_height = int(target_width / aspect_ratio)++ # Resize image+ resized_frame = frame.resize(+ (target_width, new_height), Image.Resampling.LANCZOS+ )++ # Ensure the image has a proper format+ if resized_frame.mode != 'RGB':+ resized_frame = resized_frame.convert('RGB')++ # Create a new image with explicit format+ with io.BytesIO() as buffer:+ resized_frame.save(buffer, format='JPEG')+ buffer.seek(0)+ formatted_frame = Image.open(buffer)+ formatted_frame.load() # Load the image data++ normalized_frames.append(formatted_frame)++ return normalized_framesdef ask_question_about_video(self,video_path: str,question: str,- num_frames: int = 28,) -> str:r"""Ask a question about the video.@@ -331,9 +474,6 @@ class VideoAnalysisToolkit(BaseToolkit):video_path (str): The path to the video file.It can be a local file or a URL (such as Youtube website).question (str): The question to ask about the video.- num_frames (int): The number of frames to extract from the video.- To be adjusted based on the length of the video.- (default: :obj:`28`)Returns:str: The answer to the question.@@ -343,12 +483,6 @@ class VideoAnalysisToolkit(BaseToolkit):if not question:raise ValueError("Question cannot be empty")- if num_frames <= 0:- logger.warning(- f"Invalid num_frames: {num_frames}, using default of 28"- )- num_frames = 28-parsed_url = urlparse(video_path)is_url = all([parsed_url.scheme, parsed_url.netloc])@@ -374,7 +508,7 @@ class VideoAnalysisToolkit(BaseToolkit):audio_path = self._extract_audio_from_video(video_path)audio_transcript = self._transcribe_audio(audio_path)- video_frames = self._extract_keyframes(video_path, num_frames)+ video_frames = self._extract_keyframes(video_path)prompt = VIDEO_QA_PROMPT.format(audio_transcription=audio_transcript,question=question,diff --git a/camel/toolkits/video_download_toolkit.py b/camel/toolkits/video_download_toolkit.pyindex 637198873..e2d8d1e61 100644--- a/camel/toolkits/video_download_toolkit.py+++ b/camel/toolkits/video_download_toolkit.py@@ -102,10 +102,17 @@ class VideoDownloaderToolkit(BaseToolkit):Cleans up the downloaded video if they are stored in a temporarydirectory."""- import shutil-if self._cleanup:- shutil.rmtree(self._download_directory, ignore_errors=True)+ try:+ import sys++ if getattr(sys, 'modules', None) is not None:+ import shutil++ shutil.rmtree(self._download_directory, ignore_errors=True)+ except (ImportError, AttributeError):+ # Skip cleanup if interpreter is shutting down+ passdef download_video(self, url: str) -> str:r"""Download the video and optionally split it into chunks.diff --git a/examples/toolkits/video_analysis_toolkit.py b/examples/toolkits/video_analysis_toolkit.pyindex b441618d8..9fd1ce326 100644--- a/examples/toolkits/video_analysis_toolkit.py+++ b/examples/toolkits/video_analysis_toolkit.py@@ -58,7 +58,6 @@ print("Analyzing video...")result = video_toolkit.ask_question_about_video(video_path=video_url,question=question,- num_frames=5, # Extract only 5 frames for faster processing)print("Video Analysis Result:")