🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
Update video_analysis_toolkit.py (#2660)
EldenLordx91af15cbda72
diff --git a/camel/toolkits/video_analysis_toolkit.py b/camel/toolkits/video_analysis_toolkit.pyindex 3fc03f60e..3562ea3bb 100644--- a/camel/toolkits/video_analysis_toolkit.py+++ b/camel/toolkits/video_analysis_toolkit.py@@ -17,7 +17,11 @@ from __future__ import annotationsimport ioimport os+import reimport tempfile+import sys+import types+from pathlib import Pathfrom typing import List, Optional@@ -35,12 +39,39 @@ from .video_download_toolkit import (_capture_screenshot,)+class LazyLoader(types.ModuleType):+ def __init__(self, name, module_name):+ super().__init__(name)+ self._name = name+ self._module_name = module_name+ self._mod = None++ def __load__(self):+ if self._mod is None:+ self._mod = __import__(self._module_name, fromlist=[""])+ sys.modules[self._name] = self._mod+ return self._mod++ def __getattr__(self, name):+ return getattr(self.__load__(), name)++ def __dir__(self):+ return dir(self.__load__())++sys.modules["cv2"] = LazyLoader("cv2", "cv2")+sys.modules["numpy"] = LazyLoader("numpy", "numpy")+logger = get_logger(__name__)VIDEO_QA_PROMPT = """Analyze the provided video frames and corresponding audio transcription to \answer the given question(s) thoroughly and accurately.+The transcriptions may come from two sources:+ 1. **Audio Transcription**: The spoken words in the video.+ 2. **Visual Text (OCR)**: Text extracted from the video frames (like \+ captions, on-screen text, etc.).+Instructions:1. Visual Analysis:- Examine the video frames to identify visible entities.@@ -49,11 +80,13 @@ such as size, color, shape, texture, or behavior.- Note significant groupings, interactions, or contextual patterns \relevant to the analysis.- 2. Audio Integration:+ 2. Audio and Text Integration:- Use the audio transcription to complement or clarify your visual \observations.+ - Use the visual text (OCR) to get exact textual information that may \+not be accurately readable from the images alone.- Identify names, descriptions, or contextual hints in the \-transcription that help confirm or refine your visual analysis.+transcriptions that help confirm or refine your visual analysis.3. Detailed Reasoning and Justification:- Provide a brief explanation of how you identified and distinguished \@@ -65,7 +98,7 @@ your reasoning.- Specify the total number of distinct species or object types \identified in the video.- Describe the defining characteristics and any supporting evidence \-from the video and transcription.+from the video and transcription sources.5. Important Considerations:- Pay close attention to subtle differences that could distinguish \@@ -76,11 +109,13 @@ similar-looking species or objects**Audio Transcription:**{audio_transcription}+**Visual Text (OCR):**+{visual_text}+**Question:**{question}"""-@MCPServer()class VideoAnalysisToolkit(BaseToolkit):r"""A class for analysing videos with vision-language model.@@ -96,6 +131,8 @@ class VideoAnalysisToolkit(BaseToolkit):transcription using OpenAI's audio models. Requires a valid OpenAIAPI key. When disabled, video analysis will be based solely onvisual content. (default: :obj:`False`)+ use_ocr (bool, optional): Whether to enable OCR for extracting text+ from video frames. (default: :obj:`False`)frame_interval (float, optional): Interval in seconds between framesto extract from the video. (default: :obj:`4.0`)output_language (str, optional): The language for output responses.@@ -113,6 +150,7 @@ class VideoAnalysisToolkit(BaseToolkit):download_directory: Optional[str] = None,model: Optional[BaseModelBackend] = None,use_audio_transcription: bool = False,+ use_ocr: bool = False,frame_interval: float = 4.0,output_language: str = "English",cookies_path: Optional[str] = None,@@ -122,6 +160,7 @@ class VideoAnalysisToolkit(BaseToolkit):self._cleanup = download_directory is Noneself._temp_files: list[str] = [] # Track temporary files for cleanupself._use_audio_transcription = use_audio_transcription+ self._use_ocr = use_ocrself.output_language = output_languageself.frame_interval = frame_interval@@ -174,7 +213,9 @@ class VideoAnalysisToolkit(BaseToolkit):"Audio transcription will be disabled.")self._use_audio_transcription = False-+ if self._use_ocr:+ self._use_ocr = self._check_ocr_dependencies()+def __del__(self):r"""Clean up temporary directories and files when the object isdestroyed.@@ -211,6 +252,75 @@ class VideoAnalysisToolkit(BaseToolkit):f"{self._download_directory}: {e}")+ def _check_ocr_dependencies(self):+ if not self._use_ocr:+ return False++ try:+ import pytesseract+ pytesseract.get_tesseract_version()+ return True+ except ImportError:+ logger.error(+ "pytesseract not installed. OCR will be disabled. "+ "Install with: pip install pytesseract"+ )+ self._use_ocr = False+ return False+ except EnvironmentError:+ logger.error(+ "Tesseract OCR not found in PATH. OCR will be disabled. "+ "Install Tesseract: https://github.com/tesseract-ocr/tesseract"+ )+ self._use_ocr = False+ return False++ def _extract_text_from_frame(self, frame: Image.Image) -> str:+ r"""Extract text from a video frame using OCR.++ Args:+ frame (Image.Image): PIL image frame to process.++ Returns:+ str: Extracted text from the frame.+ """+ import cv2+ import numpy as np+ import pytesseract # type: ignore[import-untyped]+ try:++ # Convert to OpenCV format for preprocessing+ cv_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)++ # Preprocessing for better OCR results+ gray = cv2.cvtColor(cv_image, cv2.COLOR_BGR2GRAY)+ blur = cv2.GaussianBlur(gray, (3, 3), 0)+ _, threshold = cv2.threshold(+ blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU+ )++ # Convert back to PIL image for OCR+ preprocessed_frame = Image.fromarray(threshold)+ return pytesseract.image_to_string(preprocessed_frame).strip()+ except Exception as e:+ logger.error(f"OCR failed: {e}")+ return ""++ def _process_extracted_text(self, text: str) -> str:+ r"""Clean and format OCR-extracted text.++ Args:+ text (str): Raw extracted OCR text.++ Returns:+ str: Cleaned and formatted text.+ """+ # Filter irrelevant characters and noise+ text = re.sub(r'[^\w\s,.?!:;\'"()-]', '', text)+ # Remove excessive whitespace+ text = re.sub(r'\s+', ' ', text).strip()+ return text+def _extract_audio_from_video(self, video_path: str, output_format: str = "mp3") -> str:@@ -511,9 +621,24 @@ class VideoAnalysisToolkit(BaseToolkit):audio_path = self._extract_audio_from_video(video_path)audio_transcript = self._transcribe_audio(audio_path)+ # Extract visual text with OCR+ visual_text = ""video_frames = self._extract_keyframes(video_path)+ # Build visual text only if OCR is enabled+ if self._use_ocr:+ visual_text = "" # Initialize for OCR processing+ for frame in video_frames:+ text = self._extract_text_from_frame(frame)+ processed = self._process_extracted_text(text)+ if processed:+ visual_text += processed + "\n"+ visual_text = visual_text.strip() or "No visual text detected."+ else:+ visual_text = ""+prompt = VIDEO_QA_PROMPT.format(audio_transcription=audio_transcript,+ visual_text=visual_text,question=question,)