🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
enhance: Update OCR capability to video_analysis_toolkit.py PR2660 (#2701)
Wendong-Fan1828a8d9e6c4
diff --git a/camel/toolkits/video_analysis_toolkit.py b/camel/toolkits/video_analysis_toolkit.pyindex 3562ea3bb..8fedb8717 100644--- a/camel/toolkits/video_analysis_toolkit.py+++ b/camel/toolkits/video_analysis_toolkit.py@@ -19,9 +19,6 @@ import ioimport osimport reimport tempfile-import sys-import types-from pathlib import Pathfrom typing import List, Optional@@ -39,28 +36,6 @@ from .video_download_toolkit import (_capture_screenshot,)-class LazyLoader(types.ModuleType):- def __init__(self, name, module_name):- super().__init__(name)- self._name = name- self._module_name = module_name- self._mod = None-- def __load__(self):- if self._mod is None:- self._mod = __import__(self._module_name, fromlist=[""])- sys.modules[self._name] = self._mod- return self._mod-- def __getattr__(self, name):- return getattr(self.__load__(), name)-- def __dir__(self):- return dir(self.__load__())--sys.modules["cv2"] = LazyLoader("cv2", "cv2")-sys.modules["numpy"] = LazyLoader("numpy", "numpy")-logger = get_logger(__name__)VIDEO_QA_PROMPT = """@@ -116,6 +91,7 @@ similar-looking species or objects{question}"""+@MCPServer()class VideoAnalysisToolkit(BaseToolkit):r"""A class for analysing videos with vision-language model.@@ -213,9 +189,7 @@ class VideoAnalysisToolkit(BaseToolkit):"Audio transcription will be disabled.")self._use_audio_transcription = False- if self._use_ocr:- self._use_ocr = self._check_ocr_dependencies()-+def __del__(self):r"""Clean up temporary directories and files when the object isdestroyed.@@ -252,29 +226,7 @@ class VideoAnalysisToolkit(BaseToolkit):f"{self._download_directory}: {e}")- def _check_ocr_dependencies(self):- if not self._use_ocr:- return False-- try:- import pytesseract- pytesseract.get_tesseract_version()- return True- except ImportError:- logger.error(- "pytesseract not installed. OCR will be disabled. "- "Install with: pip install pytesseract"- )- self._use_ocr = False- return False- except EnvironmentError:- logger.error(- "Tesseract OCR not found in PATH. OCR will be disabled. "- "Install Tesseract: https://github.com/tesseract-ocr/tesseract"- )- self._use_ocr = False- return False-+ @dependencies_required("pytesseract", "cv2", "numpy")def _extract_text_from_frame(self, frame: Image.Image) -> str:r"""Extract text from a video frame using OCR.@@ -286,19 +238,19 @@ class VideoAnalysisToolkit(BaseToolkit):"""import cv2import numpy as np- import pytesseract # type: ignore[import-untyped]+ import pytesseract+try:-# Convert to OpenCV format for preprocessingcv_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)-+# Preprocessing for better OCR resultsgray = cv2.cvtColor(cv_image, cv2.COLOR_BGR2GRAY)blur = cv2.GaussianBlur(gray, (3, 3), 0)_, threshold = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)-+# Convert back to PIL image for OCRpreprocessed_frame = Image.fromarray(threshold)return pytesseract.image_to_string(preprocessed_frame).strip()@@ -320,7 +272,7 @@ class VideoAnalysisToolkit(BaseToolkit):# Remove excessive whitespacetext = re.sub(r'\s+', ' ', text).strip()return text-+def _extract_audio_from_video(self, video_path: str, output_format: str = "mp3") -> str:@@ -626,16 +578,13 @@ class VideoAnalysisToolkit(BaseToolkit):video_frames = self._extract_keyframes(video_path)# Build visual text only if OCR is enabledif self._use_ocr:- visual_text = "" # Initialize for OCR processingfor frame in video_frames:text = self._extract_text_from_frame(frame)processed = self._process_extracted_text(text)if processed:visual_text += processed + "\n"visual_text = visual_text.strip() or "No visual text detected."- else:- visual_text = ""-+prompt = VIDEO_QA_PROMPT.format(audio_transcription=audio_transcript,visual_text=visual_text,diff --git a/pyproject.toml b/pyproject.tomlindex 8ce5c2bd4..919fdaa29 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -135,6 +135,7 @@ media_tools = ["yt-dlp>=2024.11.4,<2025","ffmpeg-python>=0.2.0,<0.3","scenedetect>=0.6.5.2",+ "pytesseract>=0.3.13",]communication_tools = [@@ -256,6 +257,7 @@ owl = ["pylatex>=1.4.2","chunkr-ai>=0.0.50","python-pptx>=1.0.2",+ "pytesseract>=0.3.13",]all = ["numpy>=1.2,<=2.2",@@ -376,6 +378,7 @@ all = ["langfuse>=2.60.5","Flask>=2.0","rlcard>=1.0.0,<1.3.0",+ "pytesseract>=0.3.13",]@@ -579,6 +582,7 @@ module = ["sentence_transformers.*","cv2","rlcard.*",+ "pytesseract",]ignore_missing_imports = true@@ -590,4 +594,4 @@ skip = '.git*,*.pdf,*.svg,*.lock,*.css,examples/toolkits/semantic_scholar_toolkicheck-hidden = true# Ignore long lines as usually some kind of base64 encodings e.g. in ipynbignore-regex = '^\s*"image/\S+": ".*|^.{300,}|\b(NotIn|Unsupport)\b'-ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci'+ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,'diff --git a/uv.lock b/uv.lockindex de3f480d8..a6a96ea32 100644--- a/uv.lock+++ b/uv.lock@@ -656,6 +656,7 @@ all = [{ name = "pyobvector" },{ name = "pyowm" },{ name = "pytelegrambotapi" },+ { name = "pytesseract" },{ name = "pytest" },{ name = "pytest-asyncio" },{ name = "pytest-cov" },@@ -792,6 +793,7 @@ media-tools = [{ name = "ffmpeg-python" },{ name = "imageio", extra = ["pyav"] },{ name = "pydub" },+ { name = "pytesseract" },{ name = "scenedetect" },{ name = "yt-dlp" },]@@ -833,6 +835,7 @@ owl = [{ name = "pydub" },{ name = "pylatex" },{ name = "pymupdf" },+ { name = "pytesseract" },{ name = "python-dotenv" },{ name = "python-pptx" },{ name = "requests-oauthlib" },@@ -1131,6 +1134,9 @@ requires-dist = [{ name = "pyowm", marker = "extra == 'web-tools'", specifier = ">=3.3.0,<4" },{ name = "pytelegrambotapi", marker = "extra == 'all'", specifier = ">=4.18.0,<5" },{ name = "pytelegrambotapi", marker = "extra == 'communication-tools'", specifier = ">=4.18.0,<5" },+ { name = "pytesseract", marker = "extra == 'all'", specifier = ">=0.3.13" },+ { name = "pytesseract", marker = "extra == 'media-tools'", specifier = ">=0.3.13" },+ { name = "pytesseract", marker = "extra == 'owl'", specifier = ">=0.3.13" },{ name = "pytest", marker = "extra == 'all'", specifier = ">=7,<8" },{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7,<8" },{ name = "pytest", marker = "extra == 'test'", specifier = ">=7,<8" },@@ -5474,6 +5480,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/bd/62/d29612ca33b7844e7wheels = [{ url = "https://files.pythonhosted.org/packages/32/5a/3399a2caf51c91db650de57464465b830c2d4ea15b23d24a98182202b704/pymupdf-1.26.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:32296f12a7c7f36febd59cee77823a54490313bcaba9879b17def6518186f94e", size = 23054640 },{ url = "https://files.pythonhosted.org/packages/64/e0/cc3ec6a4d5ada8992b8610f134565ceb517243f12736b50d795cb3459315/pymupdf-1.26.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:aad7949eca62aca40854510cdb125cf873b181726dc9497a90834200f31faa63", size = 22402766 },+ { url = "https://files.pythonhosted.org/packages/e8/cf/d5b1cd775a17a7b83e25cbf4c46f64cf1352c962ca97646e3e01953cf0df/pymupdf-1.26.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3b62c4d443121ed9a2eb967c3a0e45f8dbabcc838db8604ece02c4e868808edc", size = 23448474 },{ url = "https://files.pythonhosted.org/packages/82/9f/e7101bd24a0f5cbfa0310c8e5c3a8ec0dd9a86986812ff86ac2fbd273c92/pymupdf-1.26.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a65c411eb1cbb79e40c307e10fbad23658f19e9d7334ac4de21d24b58009a7b9", size = 24056183 },{ url = "https://files.pythonhosted.org/packages/99/39/23ac15cf0edc2877ef366dc7ae041ac199d212433c2c3113661d1a1d5ad0/pymupdf-1.26.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:26cebdcc1b2b7a7445423599ce2e0000f2be0333cce0fa0e6846e5a7da46f965", size = 24258802 },{ url = "https://files.pythonhosted.org/packages/e1/8c/56bd5951128d5c5c0b64d2942090c2cd7bc44302bac991b941ac736e3d63/pymupdf-1.26.1-cp39-abi3-win32.whl", hash = "sha256:82ed9e106cf564fc959c0691c374ba68443086ba1a1c9f26128eebbc3e6df9e5", size = 16927933 },@@ -5672,6 +5679,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/a8/ec/7dc84f817b692a6e1e673a0c335724a0926118541efff86446a20f0fa1f1/pytelegrambotapi-4.27.0-py3-none-any.whl", hash = "sha256:f555cecea9dcc48274c2592f8158e0c336654d3c5a4b86ce44342ffa5f76bd86", size = 287399 },]+[[package]]+name = "pytesseract"+version = "0.3.13"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "packaging" },+ { name = "pillow" },+]+sdist = { url = "https://files.pythonhosted.org/packages/9f/a6/7d679b83c285974a7cb94d739b461fa7e7a9b17a3abfd7bf6cbc5c2394b0/pytesseract-0.3.13.tar.gz", hash = "sha256:4bf5f880c99406f52a3cfc2633e42d9dc67615e69d8a509d74867d3baddb5db9", size = 17689 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/7a/33/8312d7ce74670c9d39a532b2c246a853861120486be9443eebf048043637/pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:7a99c6c2ac598360693d83a416e36e0b33a67638bb9d77fdcac094a3589d4b34", size = 14705 },+]+[[package]]name = "pytest"version = "7.4.4"