Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

enhance: Update OCR capability to video_analysis_toolkit.py PR2660 (#2701)

Wendong-Fan1828a8d9e6c4
diff --git a/camel/toolkits/video_analysis_toolkit.py b/camel/toolkits/video_analysis_toolkit.py
index 3562ea3bb..8fedb8717 100644
--- a/camel/toolkits/video_analysis_toolkit.py
+++ b/camel/toolkits/video_analysis_toolkit.py
@@ -19,9 +19,6 @@ import io
import os
import re
import tempfile
-import sys
-import types
-
from pathlib import Path
from typing import List, Optional
@@ -39,28 +36,6 @@ from .video_download_toolkit import (
_capture_screenshot,
)
-class LazyLoader(types.ModuleType):
- def __init__(self, name, module_name):
- super().__init__(name)
- self._name = name
- self._module_name = module_name
- self._mod = None
-
- def __load__(self):
- if self._mod is None:
- self._mod = __import__(self._module_name, fromlist=[""])
- sys.modules[self._name] = self._mod
- return self._mod
-
- def __getattr__(self, name):
- return getattr(self.__load__(), name)
-
- def __dir__(self):
- return dir(self.__load__())
-
-sys.modules["cv2"] = LazyLoader("cv2", "cv2")
-sys.modules["numpy"] = LazyLoader("numpy", "numpy")
-
logger = get_logger(__name__)
VIDEO_QA_PROMPT = """
@@ -116,6 +91,7 @@ similar-looking species or objects
{question}
"""
+
@MCPServer()
class VideoAnalysisToolkit(BaseToolkit):
r"""A class for analysing videos with vision-language model.
@@ -213,9 +189,7 @@ class VideoAnalysisToolkit(BaseToolkit):
"Audio transcription will be disabled."
)
self._use_audio_transcription = False
- if self._use_ocr:
- self._use_ocr = self._check_ocr_dependencies()
-
+
def __del__(self):
r"""Clean up temporary directories and files when the object is
destroyed.
@@ -252,29 +226,7 @@ class VideoAnalysisToolkit(BaseToolkit):
f"{self._download_directory}: {e}"
)
- def _check_ocr_dependencies(self):
- if not self._use_ocr:
- return False
-
- try:
- import pytesseract
- pytesseract.get_tesseract_version()
- return True
- except ImportError:
- logger.error(
- "pytesseract not installed. OCR will be disabled. "
- "Install with: pip install pytesseract"
- )
- self._use_ocr = False
- return False
- except EnvironmentError:
- logger.error(
- "Tesseract OCR not found in PATH. OCR will be disabled. "
- "Install Tesseract: https://github.com/tesseract-ocr/tesseract"
- )
- self._use_ocr = False
- return False
-
+ @dependencies_required("pytesseract", "cv2", "numpy")
def _extract_text_from_frame(self, frame: Image.Image) -> str:
r"""Extract text from a video frame using OCR.
@@ -286,19 +238,19 @@ class VideoAnalysisToolkit(BaseToolkit):
"""
import cv2
import numpy as np
- import pytesseract # type: ignore[import-untyped]
+ import pytesseract
+
try:
-
# Convert to OpenCV format for preprocessing
cv_image = cv2.cvtColor(np.array(frame), cv2.COLOR_RGB2BGR)
-
+
# Preprocessing for better OCR results
gray = cv2.cvtColor(cv_image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (3, 3), 0)
_, threshold = cv2.threshold(
blur, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU
)
-
+
# Convert back to PIL image for OCR
preprocessed_frame = Image.fromarray(threshold)
return pytesseract.image_to_string(preprocessed_frame).strip()
@@ -320,7 +272,7 @@ class VideoAnalysisToolkit(BaseToolkit):
# Remove excessive whitespace
text = re.sub(r'\s+', ' ', text).strip()
return text
-
+
def _extract_audio_from_video(
self, video_path: str, output_format: str = "mp3"
) -> str:
@@ -626,16 +578,13 @@ class VideoAnalysisToolkit(BaseToolkit):
video_frames = self._extract_keyframes(video_path)
# Build visual text only if OCR is enabled
if self._use_ocr:
- visual_text = "" # Initialize for OCR processing
for frame in video_frames:
text = self._extract_text_from_frame(frame)
processed = self._process_extracted_text(text)
if processed:
visual_text += processed + "\n"
visual_text = visual_text.strip() or "No visual text detected."
- else:
- visual_text = ""
-
+
prompt = VIDEO_QA_PROMPT.format(
audio_transcription=audio_transcript,
visual_text=visual_text,
diff --git a/pyproject.toml b/pyproject.toml
index 8ce5c2bd4..919fdaa29 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -135,6 +135,7 @@ media_tools = [
"yt-dlp>=2024.11.4,<2025",
"ffmpeg-python>=0.2.0,<0.3",
"scenedetect>=0.6.5.2",
+ "pytesseract>=0.3.13",
]
communication_tools = [
@@ -256,6 +257,7 @@ owl = [
"pylatex>=1.4.2",
"chunkr-ai>=0.0.50",
"python-pptx>=1.0.2",
+ "pytesseract>=0.3.13",
]
all = [
"numpy>=1.2,<=2.2",
@@ -376,6 +378,7 @@ all = [
"langfuse>=2.60.5",
"Flask>=2.0",
"rlcard>=1.0.0,<1.3.0",
+ "pytesseract>=0.3.13",
]
@@ -579,6 +582,7 @@ module = [
"sentence_transformers.*",
"cv2",
"rlcard.*",
+ "pytesseract",
]
ignore_missing_imports = true
@@ -590,4 +594,4 @@ skip = '.git*,*.pdf,*.svg,*.lock,*.css,examples/toolkits/semantic_scholar_toolki
check-hidden = true
# Ignore long lines as usually some kind of base64 encodings e.g. in ipynb
ignore-regex = '^\s*"image/\S+": ".*|^.{300,}|\b(NotIn|Unsupport)\b'
-ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci'
+ignore-words-list = 'ans,datas,rouge,doubleClick,OCE,ACI,aci,Pinter,'
diff --git a/uv.lock b/uv.lock
index de3f480d8..a6a96ea32 100644
--- a/uv.lock
+++ b/uv.lock
@@ -656,6 +656,7 @@ all = [
{ name = "pyobvector" },
{ name = "pyowm" },
{ name = "pytelegrambotapi" },
+ { name = "pytesseract" },
{ name = "pytest" },
{ name = "pytest-asyncio" },
{ name = "pytest-cov" },
@@ -792,6 +793,7 @@ media-tools = [
{ name = "ffmpeg-python" },
{ name = "imageio", extra = ["pyav"] },
{ name = "pydub" },
+ { name = "pytesseract" },
{ name = "scenedetect" },
{ name = "yt-dlp" },
]
@@ -833,6 +835,7 @@ owl = [
{ name = "pydub" },
{ name = "pylatex" },
{ name = "pymupdf" },
+ { name = "pytesseract" },
{ name = "python-dotenv" },
{ name = "python-pptx" },
{ name = "requests-oauthlib" },
@@ -1131,6 +1134,9 @@ requires-dist = [
{ name = "pyowm", marker = "extra == 'web-tools'", specifier = ">=3.3.0,<4" },
{ name = "pytelegrambotapi", marker = "extra == 'all'", specifier = ">=4.18.0,<5" },
{ name = "pytelegrambotapi", marker = "extra == 'communication-tools'", specifier = ">=4.18.0,<5" },
+ { name = "pytesseract", marker = "extra == 'all'", specifier = ">=0.3.13" },
+ { name = "pytesseract", marker = "extra == 'media-tools'", specifier = ">=0.3.13" },
+ { name = "pytesseract", marker = "extra == 'owl'", specifier = ">=0.3.13" },
{ name = "pytest", marker = "extra == 'all'", specifier = ">=7,<8" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=7,<8" },
{ name = "pytest", marker = "extra == 'test'", specifier = ">=7,<8" },
@@ -5474,6 +5480,7 @@ sdist = { url = "https://files.pythonhosted.org/packages/bd/62/d29612ca33b7844e7
wheels = [
{ url = "https://files.pythonhosted.org/packages/32/5a/3399a2caf51c91db650de57464465b830c2d4ea15b23d24a98182202b704/pymupdf-1.26.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:32296f12a7c7f36febd59cee77823a54490313bcaba9879b17def6518186f94e", size = 23054640 },
{ url = "https://files.pythonhosted.org/packages/64/e0/cc3ec6a4d5ada8992b8610f134565ceb517243f12736b50d795cb3459315/pymupdf-1.26.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:aad7949eca62aca40854510cdb125cf873b181726dc9497a90834200f31faa63", size = 22402766 },
+ { url = "https://files.pythonhosted.org/packages/e8/cf/d5b1cd775a17a7b83e25cbf4c46f64cf1352c962ca97646e3e01953cf0df/pymupdf-1.26.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3b62c4d443121ed9a2eb967c3a0e45f8dbabcc838db8604ece02c4e868808edc", size = 23448474 },
{ url = "https://files.pythonhosted.org/packages/82/9f/e7101bd24a0f5cbfa0310c8e5c3a8ec0dd9a86986812ff86ac2fbd273c92/pymupdf-1.26.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a65c411eb1cbb79e40c307e10fbad23658f19e9d7334ac4de21d24b58009a7b9", size = 24056183 },
{ url = "https://files.pythonhosted.org/packages/99/39/23ac15cf0edc2877ef366dc7ae041ac199d212433c2c3113661d1a1d5ad0/pymupdf-1.26.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:26cebdcc1b2b7a7445423599ce2e0000f2be0333cce0fa0e6846e5a7da46f965", size = 24258802 },
{ url = "https://files.pythonhosted.org/packages/e1/8c/56bd5951128d5c5c0b64d2942090c2cd7bc44302bac991b941ac736e3d63/pymupdf-1.26.1-cp39-abi3-win32.whl", hash = "sha256:82ed9e106cf564fc959c0691c374ba68443086ba1a1c9f26128eebbc3e6df9e5", size = 16927933 },
@@ -5672,6 +5679,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a8/ec/7dc84f817b692a6e1e673a0c335724a0926118541efff86446a20f0fa1f1/pytelegrambotapi-4.27.0-py3-none-any.whl", hash = "sha256:f555cecea9dcc48274c2592f8158e0c336654d3c5a4b86ce44342ffa5f76bd86", size = 287399 },
]
+[[package]]
+name = "pytesseract"
+version = "0.3.13"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "packaging" },
+ { name = "pillow" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/a6/7d679b83c285974a7cb94d739b461fa7e7a9b17a3abfd7bf6cbc5c2394b0/pytesseract-0.3.13.tar.gz", hash = "sha256:4bf5f880c99406f52a3cfc2633e42d9dc67615e69d8a509d74867d3baddb5db9", size = 17689 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7a/33/8312d7ce74670c9d39a532b2c246a853861120486be9443eebf048043637/pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:7a99c6c2ac598360693d83a416e36e0b33a67638bb9d77fdcac094a3589d4b34", size = 14705 },
+]
+
[[package]]
name = "pytest"
version = "7.4.4"