Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Add audio toolkit from owl project (#1744)

Mengkang Hu67a06080d4ab
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c091d5d6b..02c5a7235 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
- id: mypy
name: Check mypy
entry: mypy --namespace-packages -p camel -p test -p apps
- language: system
+ language: python
types: [python]
pass_filenames: false
require_serial: true
diff --git a/camel/models/__init__.py b/camel/models/__init__.py
index cdee39cd4..8d4c241df 100644
--- a/camel/models/__init__.py
+++ b/camel/models/__init__.py
@@ -14,6 +14,7 @@
from .aiml_model import AIMLModel
from .anthropic_model import AnthropicModel
from .azure_openai_model import AzureOpenAIModel
+from .base_audio_model import BaseAudioModel
from .base_model import BaseModelBackend
from .cohere_model import CohereModel
from .deepseek_model import DeepSeekModel
@@ -74,4 +75,5 @@ __all__ = [
'InternLMModel',
'MoonshotModel',
'AIMLModel',
+ 'BaseAudioModel',
]
diff --git a/camel/models/base_audio_model.py b/camel/models/base_audio_model.py
new file mode 100644
index 000000000..67aa150af
--- /dev/null
+++ b/camel/models/base_audio_model.py
@@ -0,0 +1,92 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Optional
+
+
+class BaseAudioModel(ABC):
+ r"""Base class for audio models providing Text-to-Speech (TTS) and
+ Speech-to-Text (STT) functionality.
+ """
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ url: Optional[str] = None,
+ ) -> None:
+ r"""Initialize an instance of BaseAudioModel.
+
+ Args:
+ api_key (Optional[str]): API key for the audio service. If not
+ provided, will look for an environment variable specific to the
+ implementation.
+ url (Optional[str]): Base URL for the audio API. If not provided,
+ will use a default URL or look for an environment variable
+ specific to the implementation.
+ """
+ self._api_key = api_key
+ self._url = url
+
+ @abstractmethod
+ def text_to_speech(
+ self,
+ input: str,
+ *,
+ storage_path: str,
+ **kwargs: Any,
+ ) -> Any:
+ r"""Convert text to speech.
+
+ Args:
+ input (str): The text to be converted to speech.
+ storage_path (str): The local path to store the
+ generated speech file.
+ **kwargs (Any): Extra kwargs passed to the TTS API.
+
+ Returns:
+ Any: The response from the TTS API, which may vary by
+ implementation.
+ """
+ pass
+
+ @abstractmethod
+ def speech_to_text(
+ self,
+ audio_file_path: str,
+ **kwargs: Any,
+ ) -> str:
+ r"""Convert speech audio to text.
+
+ Args:
+ audio_file_path (str): The audio file path to transcribe.
+ **kwargs (Any): Extra keyword arguments passed to the
+ Speech-to-Text (STT) API.
+
+ Returns:
+ str: The transcribed text.
+ """
+ pass
+
+ def _ensure_directory_exists(self, file_path: str) -> None:
+ r"""Ensure the directory for the given file path exists.
+
+ Args:
+ file_path (str): The file path for which to ensure the directory
+ exists.
+ """
+ directory = os.path.dirname(file_path)
+ if directory and not os.path.exists(directory):
+ os.makedirs(directory)
diff --git a/camel/models/fish_audio_model.py b/camel/models/fish_audio_model.py
index 8c550dc43..0cc6d4b8b 100644
--- a/camel/models/fish_audio_model.py
+++ b/camel/models/fish_audio_model.py
@@ -15,8 +15,10 @@
import os
from typing import Any, Optional
+from camel.models.base_audio_model import BaseAudioModel
-class FishAudioModel:
+
+class FishAudioModel(BaseAudioModel):
r"""Provides access to FishAudio's Text-to-Speech (TTS) and Speech_to_Text
(STT) models.
"""
@@ -37,6 +39,7 @@ class FishAudioModel:
"""
from fish_audio_sdk import Session
+ super().__init__(api_key, url)
self._api_key = api_key or os.environ.get("FISHAUDIO_API_KEY")
self._url = url or os.environ.get(
"FISHAUDIO_API_BASE_URL", "https://api.fish.audio"
@@ -46,7 +49,8 @@ class FishAudioModel:
def text_to_speech(
self,
input: str,
- storage_path: str,
+ *,
+ storage_path: Optional[str] = None,
reference_id: Optional[str] = None,
reference_audio: Optional[str] = None,
reference_audio_text: Optional[str] = None,
@@ -55,9 +59,9 @@ class FishAudioModel:
r"""Convert text to speech and save the output to a file.
Args:
- input_text (str): The text to convert to speech.
- storage_path (str): The file path where the resulting speech will
- be saved.
+ input (str): The text to convert to speech.
+ storage_path (Optional[str]): The file path where the resulting
+ speech will be saved. (default: :obj:`None`)
reference_id (Optional[str]): An optional reference ID to
associate with the request. (default: :obj:`None`)
reference_audio (Optional[str]): Path to an audio file for
@@ -68,12 +72,18 @@ class FishAudioModel:
Raises:
FileNotFoundError: If the reference audio file cannot be found.
+ ValueError: If storage_path is not provided or if reference_audio
+ is provided without reference_audio_text.
"""
from fish_audio_sdk import ReferenceAudio, TTSRequest
- directory = os.path.dirname(storage_path)
- if directory and not os.path.exists(directory):
- os.makedirs(directory)
+ if storage_path is None:
+ raise ValueError(
+ "storage_path must be provided for "
+ "FishAudioModel.text_to_speech"
+ )
+
+ self._ensure_directory_exists(storage_path)
if not reference_audio:
with open(f"{storage_path}", "wb") as f:
diff --git a/camel/models/openai_audio_models.py b/camel/models/openai_audio_models.py
index 5ecc3e33c..fa28eb4bb 100644
--- a/camel/models/openai_audio_models.py
+++ b/camel/models/openai_audio_models.py
@@ -11,15 +11,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import base64
import os
from typing import Any, List, Optional, Union
from openai import AsyncOpenAI, OpenAI, _legacy_response
+from camel.models.base_audio_model import BaseAudioModel
from camel.types import AudioModelType, VoiceType
-class OpenAIAudioModels:
+class OpenAIAudioModels(BaseAudioModel):
r"""Provides access to OpenAI's Text-to-Speech (TTS) and Speech_to_Text
(STT) models."""
@@ -29,6 +31,7 @@ class OpenAIAudioModels:
url: Optional[str] = None,
) -> None:
r"""Initialize an instance of OpenAI."""
+ super().__init__(api_key, url)
self._url = url or os.environ.get("OPENAI_API_BASE_URL")
self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
self._client = OpenAI(
@@ -47,6 +50,7 @@ class OpenAIAudioModels:
def text_to_speech(
self,
input: str,
+ *,
model_type: AudioModelType = AudioModelType.TTS_1,
voice: VoiceType = VoiceType.ALLOY,
storage_path: Optional[str] = None,
@@ -111,6 +115,8 @@ class OpenAIAudioModels:
new_storage_path = (
f"{file_name}_{chunk_index}{file_extension}"
)
+ # Ensure directory exists
+ self._ensure_directory_exists(new_storage_path)
response.write_to_file(new_storage_path)
chunk_index += 1
except Exception as e:
@@ -131,6 +137,8 @@ class OpenAIAudioModels:
if storage_path:
try:
+ # Ensure directory exists
+ self._ensure_directory_exists(storage_path)
response.write_to_file(storage_path)
except Exception as e:
raise Exception("Error during write the file") from e
@@ -263,3 +271,74 @@ class OpenAIAudioModels:
return transcription.text
except Exception as e:
raise Exception("Error during STT API call") from e
+
+ def audio_question_answering(
+ self,
+ audio_file_path: str,
+ question: str,
+ model: str = "gpt-4o-mini-audio-preview",
+ **kwargs: Any,
+ ) -> str:
+ r"""Answer a question directly using the audio content.
+
+ Args:
+ audio_file_path (str): The path to the audio file.
+ question (str): The question to ask about the audio content.
+ model (str, optional): The model to use for audio question
+ answering. (default: :obj:`"gpt-4o-mini-audio-preview"`)
+ **kwargs (Any): Extra keyword arguments passed to the chat
+ completions API.
+
+ Returns:
+ str: The model's response to the question.
+
+ Raises:
+ Exception: If there's an error during the API call.
+ """
+ try:
+ # Read and encode the audio file
+ with open(audio_file_path, "rb") as audio_file:
+ audio_data = audio_file.read()
+
+ encoded_string = base64.b64encode(audio_data).decode('utf-8')
+
+ # Get file format
+ file_suffix = os.path.splitext(audio_file_path)[1]
+ file_format = file_suffix[1:].lower()
+
+ # Prepare the prompt
+ text_prompt = "Answer the following question based on the "
+ f"given audio information:\n\n{question}"
+
+ # Call the OpenAI API
+ completion = self._client.chat.completions.create(
+ model=model,
+ messages=[
+ {
+ "role": "system",
+ "content": "You are a helpful assistant "
+ "specializing in audio analysis.",
+ },
+ { # type: ignore[misc, list-item]
+ "role": "user",
+ "content": [
+ {"type": "text", "text": text_prompt},
+ {
+ "type": "input_audio",
+ "input_audio": {
+ "data": encoded_string,
+ "format": file_format,
+ },
+ },
+ ],
+ },
+ ],
+ **kwargs,
+ )
+
+ response = str(completion.choices[0].message.content)
+ return response
+ except Exception as e:
+ raise Exception(
+ "Error during audio question answering API call"
+ ) from e
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.py
index 920126e57..b78e4f257 100644
--- a/camel/toolkits/__init__.py
+++ b/camel/toolkits/__init__.py
@@ -50,6 +50,7 @@ from .semantic_scholar_toolkit import SemanticScholarToolkit
from .zapier_toolkit import ZapierToolkit
from .sympy_toolkit import SymPyToolkit
from .mineru_toolkit import MinerUToolkit
+from .audio_analysis_toolkit import AudioAnalysisToolkit
from .excel_toolkit import ExcelToolkit
from .video_analysis_toolkit import VideoAnalysisToolkit
from .image_analysis_toolkit import ImageAnalysisToolkit
@@ -91,6 +92,7 @@ __all__ = [
'ZapierToolkit',
'SymPyToolkit',
'MinerUToolkit',
+ 'AudioAnalysisToolkit',
'ExcelToolkit',
'VideoAnalysisToolkit',
'ImageAnalysisToolkit',
diff --git a/camel/toolkits/audio_analysis_toolkit.py b/camel/toolkits/audio_analysis_toolkit.py
new file mode 100644
index 000000000..aa51aae38
--- /dev/null
+++ b/camel/toolkits/audio_analysis_toolkit.py
@@ -0,0 +1,238 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+import uuid
+from typing import List, Optional
+from urllib.parse import urlparse
+
+import requests
+
+from camel.logger import get_logger
+from camel.messages import BaseMessage
+from camel.models import BaseAudioModel, BaseModelBackend, OpenAIAudioModels
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+def download_file(url: str, cache_dir: str) -> str:
+ r"""Download a file from a URL to a local cache directory.
+
+ Args:
+ url (str): The URL of the file to download.
+ cache_dir (str): The directory to save the downloaded file.
+
+ Returns:
+ str: The path to the downloaded file.
+
+ Raises:
+ Exception: If the download fails.
+ """
+ # Create cache directory if it doesn't exist
+ os.makedirs(cache_dir, exist_ok=True)
+
+ # Extract filename from URL or generate a unique one
+ parsed_url = urlparse(url)
+ filename = os.path.basename(parsed_url.path)
+ if not filename:
+ # Generate a unique filename if none is provided in the URL
+ file_ext = ".mp3" # Default extension
+ content_type = None
+
+ # Try to get the file extension from the content type
+ try:
+ response = requests.head(url)
+ content_type = response.headers.get('Content-Type', '')
+ if 'audio/wav' in content_type:
+ file_ext = '.wav'
+ elif 'audio/mpeg' in content_type:
+ file_ext = '.mp3'
+ elif 'audio/ogg' in content_type:
+ file_ext = '.ogg'
+ except Exception:
+ pass
+
+ filename = f"{uuid.uuid4()}{file_ext}"
+
+ local_path = os.path.join(cache_dir, filename)
+
+ # Download the file
+ response = requests.get(url, stream=True)
+ response.raise_for_status()
+
+ with open(local_path, 'wb') as f:
+ for chunk in response.iter_content(chunk_size=8192):
+ f.write(chunk)
+
+ logger.debug(f"Downloaded file from {url} to {local_path}")
+ return local_path
+
+
+class AudioAnalysisToolkit(BaseToolkit):
+ r"""A toolkit for audio processing and analysis.
+
+ This class provides methods for processing, transcribing, and extracting
+ information from audio data, including direct question answering about
+ audio content.
+
+ Args:
+ cache_dir (Optional[str]): Directory path for caching downloaded audio
+ files. If not provided, 'tmp/' will be used. (default: :obj:`None`)
+ transcribe_model (Optional[BaseAudioModel]): Model used for audio
+ transcription. If not provided, OpenAIAudioModels will be used.
+ (default: :obj:`None`)
+ audio_reasoning_model (Optional[BaseModelBackend]): Model used for
+ audio reasoning and question answering. If not provided, uses the
+ default model from ChatAgent. (default: :obj:`None`)
+ """
+
+ def __init__(
+ self,
+ cache_dir: Optional[str] = None,
+ transcribe_model: Optional[BaseAudioModel] = None,
+ audio_reasoning_model: Optional[BaseModelBackend] = None,
+ ):
+ self.cache_dir = 'tmp/'
+ if cache_dir:
+ self.cache_dir = cache_dir
+
+ if transcribe_model:
+ self.transcribe_model = transcribe_model
+ else:
+ self.transcribe_model = OpenAIAudioModels()
+ logger.warning(
+ "No audio transcription model provided. "
+ "Using OpenAIAudioModels."
+ )
+
+ from camel.agents import ChatAgent
+
+ if audio_reasoning_model:
+ self.audio_agent = ChatAgent(model=audio_reasoning_model)
+ else:
+ self.audio_agent = ChatAgent()
+ logger.warning(
+ "No audio reasoning model provided. Using default model in"
+ " ChatAgent."
+ )
+
+ def audio2text(self, audio_path: str) -> str:
+ r"""Transcribe audio to text.
+
+ Args:
+ audio_path (str): The path to the audio file or URL.
+
+ Returns:
+ str: The transcribed text.
+ """
+ logger.debug(
+ f"Calling transcribe_audio method for audio file `{audio_path}`."
+ )
+
+ try:
+ audio_transcript = self.transcribe_model.speech_to_text(audio_path)
+ if not audio_transcript:
+ logger.warning("Audio transcription returned empty result")
+ return "No audio transcription available."
+ return audio_transcript
+ except Exception as e:
+ logger.error(f"Audio transcription failed: {e}")
+ return "Audio transcription failed."
+
+ def ask_question_about_audio(self, audio_path: str, question: str) -> str:
+ r"""Ask any question about the audio and get the answer using
+ multimodal model.
+
+ Args:
+ audio_path (str): The path to the audio file.
+ question (str): The question to ask about the audio.
+
+ Returns:
+ str: The answer to the question.
+ """
+
+ logger.debug(
+ f"Calling ask_question_about_audio method for audio file \
+ `{audio_path}` and question `{question}`."
+ )
+
+ parsed_url = urlparse(audio_path)
+ is_url = all([parsed_url.scheme, parsed_url.netloc])
+ local_audio_path = audio_path
+
+ # If the audio is a URL, download it first
+ if is_url:
+ try:
+ local_audio_path = download_file(audio_path, self.cache_dir)
+ except Exception as e:
+ logger.error(f"Failed to download audio file: {e}")
+ return f"Failed to download audio file: {e!s}"
+
+ # Try direct audio question answering first
+ try:
+ # Check if the transcribe_model supports audio_question_answering
+ if hasattr(self.transcribe_model, 'audio_question_answering'):
+ logger.debug("Using direct audio question answering")
+ response = self.transcribe_model.audio_question_answering(
+ local_audio_path, question
+ )
+ return response
+ except Exception as e:
+ logger.warning(
+ f"Direct audio question answering failed: {e}. "
+ "Falling back to transcription-based approach."
+ )
+
+ # Fallback to transcription-based approach
+ try:
+ transcript = self.audio2text(local_audio_path)
+ reasoning_prompt = f"""
+ <speech_transcription_result>{transcript}</
+ speech_transcription_result>
+
+ Please answer the following question based on the speech
+ transcription result above:
+ <question>{question}</question>
+ """
+ msg = BaseMessage.make_user_message(
+ role_name="User", content=reasoning_prompt
+ )
+ response = self.audio_agent.step(msg)
+
+ if not response or not response.msgs:
+ logger.error("Model returned empty response")
+ return (
+ "Failed to generate an answer. "
+ "The model returned an empty response."
+ )
+
+ answer = response.msgs[0].content
+ return answer
+ except Exception as e:
+ logger.error(f"Audio question answering failed: {e}")
+ return f"Failed to answer question about audio: {e!s}"
+
+ def get_tools(self) -> List[FunctionTool]:
+ r"""Returns a list of FunctionTool objects representing the functions
+ in the toolkit.
+
+ Returns:
+ List[FunctionTool]: A list of FunctionTool objects representing the
+ functions in the toolkit.
+ """
+ return [
+ FunctionTool(self.ask_question_about_audio),
+ FunctionTool(self.audio2text),
+ ]
diff --git a/examples/toolkits/audio_analysis_toolkit.py b/examples/toolkits/audio_analysis_toolkit.py
new file mode 100644
index 000000000..fabee9ac2
--- /dev/null
+++ b/examples/toolkits/audio_analysis_toolkit.py
@@ -0,0 +1,104 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from camel.agents import ChatAgent
+from camel.configs import ChatGPTConfig
+from camel.models import ModelFactory, OpenAIAudioModels
+from camel.toolkits import AudioAnalysisToolkit
+from camel.types import ModelPlatformType, ModelType
+
+audio_models = OpenAIAudioModels()
+
+# Set example input
+input = """CAMEL-AI.org is an open-source community dedicated to the study of
+autonomous and communicative agents. We believe that studying these agents on
+a large scale offers valuable insights into their behaviors, capabilities, and
+potential risks. To facilitate research in this field, we provide, implement,
+and support various types of agents, tasks, prompts, models, datasets, and
+simulated environments.
+
+Join us via Slack, Discord, or WeChat in pushing the boundaries of building AI
+Society."""
+
+# Set example local path to store the file
+storage_path = "examples/openai_audio_models/example_audio.mp3"
+
+# Convert the example input into audio and store it locally
+audio_models.text_to_speech(input=input, storage_path=storage_path)
+
+model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=ChatGPTConfig(
+ temperature=0.0,
+ ).as_dict(),
+)
+
+
+audio_reason_model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI,
+ model_type=ModelType.GPT_4O_MINI,
+ model_config_dict=ChatGPTConfig(
+ temperature=0.0,
+ ).as_dict(),
+)
+
+# Create the AudioAnalysisToolkit with our reasoning model
+audio_toolkit = AudioAnalysisToolkit(audio_reasoning_model=audio_reason_model)
+
+# Create a ChatAgent with the audio toolkit tools
+agent = ChatAgent(
+ system_message="You are an assistant specialized in audio analysis.",
+ model=model,
+ tools=[*audio_toolkit.get_tools()],
+)
+
+question = "What content can you hear in this audio?"
+response = agent.step(
+ f"I have an audio file at {storage_path}. Can you analyze it and tell "
+ f"me {question}"
+)
+print(response.msgs[0].content)
+print("\n")
+
+response = agent.step(f"Please transcribe the audio file at {storage_path}")
+print(response.msgs[0].content)
+print("\n")
+
+"""
+==========================================================================
+2025-03-09 22:54:55,822 - camel.camel.toolkits.audio_analysis_toolkit -
+WARNING - No audio transcription model provided. Using OpenAIAudioModels.
+
+The audio content discusses Camel AI, an open-source community dedicated to
+the study of autonomous and communicative agents. It emphasizes the belief
+that large-scale research on these agents can yield valuable insights into
+their behaviors, capabilities, and potential risks. The community provides
+resources to support research, including various types of agents, tasks,
+prompts, models, datasets, and simulated environments. Additionally, it
+invites listeners to join the community through platforms like Slack, Discord,
+or WeChat to contribute to the development of AI society.
+
+
+Here is the transcription of the audio:
+
+"CamelAI.org is an open-source community dedicated to the study of autonomous
+and communicative agents. We believe that studying these agents on a large
+scale offers valuable insights into their behaviors, capabilities, and
+potential risks. To facilitate research in this field, we provide, implement,
+and support various types of agents, tasks, prompts, models, datasets, and
+simulated environments. Join us via Slack, Discord, or WeChat in pushing the
+boundaries of building AI society."
+==========================================================================
+"""
diff --git a/test/models/test_fish_audio_model.py b/test/models/test_fish_audio_model.py
index 4cb936792..9b28a7e29 100644
--- a/test/models/test_fish_audio_model.py
+++ b/test/models/test_fish_audio_model.py
@@ -48,7 +48,7 @@ def test_fish_audio_model_text_to_speech_no_reference_audio():
with patch("builtins.open") as mock_file:
mock_file.return_value.__enter__.return_value.write = MagicMock()
- model.text_to_speech("Hello", "output_path.wav")
+ model.text_to_speech(input="Hello", storage_path="output_path.wav")
mock_session.tts.assert_called_once()
mock_file.return_value.__enter__.return_value.write.assert_called_once_with(
@@ -76,8 +76,8 @@ def test_fish_audio_model_text_to_speech_with_reference_audio():
)
model.text_to_speech(
- "Hello",
- "output_path.wav",
+ input="Hello",
+ storage_path="output_path.wav",
reference_audio="reference_audio.wav",
reference_audio_text="Reference text",
)
diff --git a/test/toolkits/test_audio_analysis_toolkit.py b/test/toolkits/test_audio_analysis_toolkit.py
new file mode 100644
index 000000000..762ced4e4
--- /dev/null
+++ b/test/toolkits/test_audio_analysis_toolkit.py
@@ -0,0 +1,175 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+import tempfile
+from unittest.mock import MagicMock, patch
+
+from camel.messages import BaseMessage
+from camel.models import OpenAIAudioModels
+from camel.toolkits.audio_analysis_toolkit import (
+ AudioAnalysisToolkit,
+ download_file,
+)
+
+
+def test_download_file():
+ r"""Test the download_file function with a mock response."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ with (
+ patch('requests.get') as mock_get,
+ patch('requests.head') as mock_head,
+ ):
+ # Mock the head request to get content type
+ mock_head_response = MagicMock()
+ mock_head_response.headers = {'Content-Type': 'audio/mpeg'}
+ mock_head.return_value = mock_head_response
+
+ # Mock the get request for downloading
+ mock_response = MagicMock()
+ mock_response.iter_content.return_value = [b'test audio content']
+ mock_get.return_value = mock_response
+
+ # Test with URL that has a filename
+ url = 'http://example.com/audio.mp3'
+ result = download_file(url, temp_dir)
+
+ # Verify the file was created with expected path
+ assert os.path.exists(result)
+ assert os.path.basename(result) == 'audio.mp3'
+
+ # Test with URL that doesn't have a filename
+ url = 'http://example.com/audio'
+ result = download_file(url, temp_dir)
+
+ # Verify a file was created
+ assert os.path.exists(result)
+ # The file should exist in the temp directory
+ assert temp_dir in result
+
+
+def test_audio2text():
+ r"""Test the audio2text method of AudioAnalysisToolkit."""
+ # Create mock transcribe model
+ mock_model = MagicMock(spec=OpenAIAudioModels)
+ mock_model.speech_to_text.return_value = "This is a test transcription."
+
+ # Create toolkit with mock model
+ toolkit = AudioAnalysisToolkit(transcribe_model=mock_model)
+
+ # Test successful transcription
+ result = toolkit.audio2text("test_audio.mp3")
+ assert result == "This is a test transcription."
+ mock_model.speech_to_text.assert_called_once_with("test_audio.mp3")
+
+ # Test exception handling
+ mock_model.speech_to_text.side_effect = Exception("Test error")
+ result = toolkit.audio2text("test_audio.mp3")
+ assert result == "Audio transcription failed."
+
+
+def test_ask_question_about_audio_direct():
+ r"""Test ask_question_about_audio method with direct audio question
+ answering.
+ """
+ # Create mock transcribe model with audio_question_answering method
+ mock_model = MagicMock(spec=OpenAIAudioModels)
+ mock_model.audio_question_answering.return_value = (
+ "The answer to your question."
+ )
+
+ # Create toolkit with mock model
+ toolkit = AudioAnalysisToolkit(transcribe_model=mock_model)
+
+ # Test with local audio file
+ result = toolkit.ask_question_about_audio(
+ "test_audio.mp3", "What is in the audio?"
+ )
+ assert result == "The answer to your question."
+ mock_model.audio_question_answering.assert_called_once_with(
+ "test_audio.mp3", "What is in the audio?"
+ )
+
+
+def test_ask_question_about_audio_fallback():
+ r"""Test ask_question_about_audio method with fallback to transcription."""
+ # Create mock transcribe model without audio_question_answering
+ mock_model = MagicMock(spec=OpenAIAudioModels)
+ mock_model.audio_question_answering.side_effect = Exception(
+ "Not implemented"
+ )
+ mock_model.speech_to_text.return_value = "This is a test transcription."
+
+ # Create mock chat agent
+ mock_agent = MagicMock()
+ mock_response = MagicMock()
+ mock_response.msgs = [MagicMock(content="Answer based on transcription.")]
+ mock_agent.step.return_value = mock_response
+
+ # Create toolkit with mocks
+ with patch('camel.agents.ChatAgent') as MockChatAgent:
+ MockChatAgent.return_value = mock_agent
+ toolkit = AudioAnalysisToolkit(transcribe_model=mock_model)
+
+ # Test fallback to transcription
+ result = toolkit.ask_question_about_audio(
+ "test_audio.mp3", "What is in the audio?"
+ )
+ assert result == "Answer based on transcription."
+ mock_model.speech_to_text.assert_called_once_with("test_audio.mp3")
+ mock_agent.step.assert_called_once()
+ # Verify the message passed to the agent contains the transcript and
+ # question
+ msg = mock_agent.step.call_args[0][0]
+ assert isinstance(msg, BaseMessage)
+ assert "speech_transcription_result" in msg.content
+ assert "What is in the audio?" in msg.content
+
+
+def test_ask_question_about_audio_url():
+ r"""Test ask_question_about_audio method with URL input."""
+ # Create mock transcribe model
+ mock_model = MagicMock(spec=OpenAIAudioModels)
+ mock_model.audio_question_answering.return_value = (
+ "The answer to your question."
+ )
+
+ # Create toolkit with mock model
+ with patch(
+ 'camel.toolkits.audio_analysis_toolkit.download_file'
+ ) as mock_download:
+ mock_download.return_value = "/tmp/downloaded_audio.mp3"
+ toolkit = AudioAnalysisToolkit(transcribe_model=mock_model)
+
+ # Test with URL
+ url = "http://example.com/audio.mp3"
+ result = toolkit.ask_question_about_audio(url, "What is in the audio?")
+ assert result == "The answer to your question."
+ mock_download.assert_called_once_with(url, toolkit.cache_dir)
+ mock_model.audio_question_answering.assert_called_once_with(
+ "/tmp/downloaded_audio.mp3", "What is in the audio?"
+ )
+
+
+def test_get_tools():
+ r"""Test the get_tools method of AudioAnalysisToolkit."""
+ toolkit = AudioAnalysisToolkit()
+ tools = toolkit.get_tools()
+
+ # Verify we have the expected number of tools
+ assert len(tools) == 2
+
+ # Verify the tool functions
+ tool_functions = [tool.func for tool in tools]
+ assert toolkit.ask_question_about_audio in tool_functions
+ assert toolkit.audio2text in tool_functions
diff --git a/test/toolkits/test_open_api_function.py b/test/toolkits/test_open_api_function.py
index 74d41a88e..921a7c0fb 100644
--- a/test/toolkits/test_open_api_function.py
+++ b/test/toolkits/test_open_api_function.py
@@ -83,7 +83,7 @@ def test_klarna_productsUsingGET(get_function):
'url': '''https://www.klarna.com/us/shopping/pl/cl337/
3200177969/Shoes/Nike-Dunk-Low-Retro-M-Black-White/?
utm_source=openai&ref-site=openai_plugin''',
- 'price': '\$81.00',
+ 'price': r'$81.00',
'attributes': [
'Outsole:Rubber',
'Fastening:Laced',
@@ -103,7 +103,7 @@ def test_klarna_productsUsingGET(get_function):
'url': '''https://www.klarna.com/us/shopping/pl/cl337/3979297/
Shoes/Nike-Air-Force-1-07-M-White/?utm_source=openai&
ref-site=openai_plugin''',
- 'price': '\$80.00',
+ 'price': r'$80.00',
'attributes': [
'Outsole:Rubber',
'Fastening:Laced',