🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Jina reranker toolkit (#2170)
JINO ROHIT48687a609902
diff --git a/camel/toolkits/__init__.py b/camel/toolkits/__init__.pyindex 3ea41ac0a..7a8074f2d 100644--- a/camel/toolkits/__init__.py+++ b/camel/toolkits/__init__.py@@ -67,6 +67,7 @@ from .thinking_toolkit import ThinkingToolkitfrom .pyautogui_toolkit import PyAutoGUIToolkitfrom .openai_agent_toolkit import OpenAIAgentToolkitfrom .searxng_toolkit import SearxNGToolkit+from .jina_reranker_toolkit import JinaRerankerToolkit__all__ = [@@ -122,4 +123,5 @@ __all__ = ['PyAutoGUIToolkit','OpenAIAgentToolkit','SearxNGToolkit',+ 'JinaRerankerToolkit',]diff --git a/camel/toolkits/jina_reranker_toolkit.py b/camel/toolkits/jina_reranker_toolkit.pynew file mode 100644index 000000000..2e9b41fff--- /dev/null+++ b/camel/toolkits/jina_reranker_toolkit.py@@ -0,0 +1,223 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import List, Optional, Tuple++import torch+from transformers import AutoModel++from camel.toolkits import FunctionTool+from camel.toolkits.base import BaseToolkit+from camel.utils import MCPServer+++@MCPServer()+class JinaRerankerToolkit(BaseToolkit):+ r"""A class representing a toolkit for reranking documents+ using Jina Reranker.++ This class provides methods for reranking documents (text or images)+ based on their relevance to a given query using the Jina Reranker model.+ """++ def __init__(+ self,+ timeout: Optional[float] = None,+ device: Optional[str] = None,+ ) -> None:+ r"""Initializes a new instance of the JinaRerankerToolkit class.++ Args:+ timeout (Optional[float]): The timeout value for API requests+ in seconds. If None, no timeout is applied.+ (default: :obj:`None`)+ device (Optional[str]): Device to load the model on. If None,+ will use CUDA if available, otherwise CPU.+ (default: :obj:`None`)+ """+ super().__init__(timeout=timeout)++ self.model = AutoModel.from_pretrained(+ 'jinaai/jina-reranker-m0',+ torch_dtype="auto",+ trust_remote_code=True,+ )+ DEVICE = (+ device+ if device is not None+ else ("cuda" if torch.cuda.is_available() else "cpu")+ )+ self.model.to(DEVICE)+ self.model.eval()++ def _sort_documents(+ self, documents: List[str], scores: List[float]+ ) -> List[Tuple[str, float]]:+ r"""Sort documents by their scores in descending order.++ Args:+ documents (List[str]): List of documents to sort.+ scores (List[float]): Corresponding scores for each document.++ Returns:+ List[Tuple[str, float]]: Sorted list of (document, score) pairs.++ Raises:+ ValueError: If documents and scores have different lengths.+ """+ if len(documents) != len(scores):+ raise ValueError("Number of documents must match number of scores")+ doc_score_pairs = list(zip(documents, scores))+ doc_score_pairs.sort(key=lambda x: x[1], reverse=True)++ return doc_score_pairs++ def rerank_text_documents(+ self,+ query: str,+ documents: List[str],+ max_length: int = 1024,+ ) -> List[Tuple[str, float]]:+ r"""Reranks text documents based on their relevance to a text query.++ Args:+ query (str): The text query for reranking.+ documents (List[str]): List of text documents to be reranked.+ max_length (int): Maximum token length for processing.+ (default: :obj:`1024`)++ Returns:+ List[Tuple[str, float]]: A list of tuples containing+ the reranked documents and their relevance scores.+ """+ if self.model is None:+ raise ValueError(+ "Model has not been initialized or failed to initialize."+ )++ with torch.inference_mode():+ text_pairs = [[query, doc] for doc in documents]+ scores = self.model.compute_score(+ text_pairs, max_length=max_length, doc_type="text"+ )++ return self._sort_documents(documents, scores)++ def rerank_image_documents(+ self,+ query: str,+ documents: List[str],+ max_length: int = 2048,+ ) -> List[Tuple[str, float]]:+ r"""Reranks image documents based on their relevance to a text query.++ Args:+ query (str): The text query for reranking.+ documents (List[str]): List of image URLs or paths to be reranked.+ max_length (int): Maximum token length for processing.+ (default: :obj:`2048`)++ Returns:+ List[Tuple[str, float]]: A list of tuples containing+ the reranked image URLs/paths and their relevance scores.+ """+ if self.model is None:+ raise ValueError(+ "Model has not been initialized or failed to initialize."+ )++ with torch.inference_mode():+ image_pairs = [[query, doc] for doc in documents]+ scores = self.model.compute_score(+ image_pairs, max_length=max_length, doc_type="image"+ )++ return self._sort_documents(documents, scores)++ def image_query_text_documents(+ self,+ image_query: str,+ documents: List[str],+ max_length: int = 2048,+ ) -> List[Tuple[str, float]]:+ r"""Reranks text documents based on their relevance to an image query.++ Args:+ image_query (str): The image URL or path used as query.+ documents (List[str]): List of text documents to be reranked.+ max_length (int): Maximum token length for processing.+ (default: :obj:`2048`)++ Returns:+ List[Tuple[str, float]]: A list of tuples containing+ the reranked documents and their relevance scores.+ """+ if self.model is None:+ raise ValueError("Model has not been initialized.")+ with torch.inference_mode():+ image_pairs = [[image_query, doc] for doc in documents]+ scores = self.model.compute_score(+ image_pairs,+ max_length=max_length,+ query_type="image",+ doc_type="text",+ )++ return self._sort_documents(documents, scores)++ def image_query_image_documents(+ self,+ image_query: str,+ documents: List[str],+ max_length: int = 2048,+ ) -> List[Tuple[str, float]]:+ r"""Reranks image documents based on their relevance to an image query.++ Args:+ image_query (str): The image URL or path used as query.+ documents (List[str]): List of image URLs or paths to be reranked.+ max_length (int): Maximum token length for processing.+ (default: :obj:`2048`)++ Returns:+ List[Tuple[str, float]]: A list of tuples containing+ the reranked image URLs/paths and their relevance scores.+ """+ if self.model is None:+ raise ValueError("Model has not been initialized.")++ with torch.inference_mode():+ image_pairs = [[image_query, doc] for doc in documents]+ scores = self.model.compute_score(+ image_pairs,+ max_length=max_length,+ query_type="image",+ doc_type="image",+ )++ return self._sort_documents(documents, scores)++ def get_tools(self) -> List[FunctionTool]:+ r"""Returns a list of FunctionTool objects representing the+ functions in the toolkit.++ Returns:+ List[FunctionTool]: A list of FunctionTool objects+ representing the functions in the toolkit.+ """+ return [+ FunctionTool(self.rerank_text_documents),+ FunctionTool(self.rerank_image_documents),+ FunctionTool(self.image_query_text_documents),+ FunctionTool(self.image_query_image_documents),+ ]diff --git a/docs/key_modules/tools.md b/docs/key_modules/tools.mdindex bb41b8a24..1e383e69f 100644--- a/docs/key_modules/tools.md+++ b/docs/key_modules/tools.md@@ -166,6 +166,7 @@ CAMEL provides a variety of built-in toolkits that you can use right away. Here'| GoogleScholarToolkit | A toolkit for retrieving information about authors and their publications from Google Scholar. || HumanToolkit | A toolkit for facilitating human-in-the-loop interactions and feedback in AI systems. || ImageAnalysisToolkit | A toolkit for comprehensive image analysis and understanding using vision-capable language models. |+| JinaRerankerToolkit | A toolkit for reranking documents (text or images) based on their relevance to a given query using the Jina Reranker model. || LinkedInToolkit | A toolkit for LinkedIn operations including creating posts, deleting posts, and retrieving user profile information. || MathToolkit | A toolkit for performing basic mathematical operations such as addition, subtraction, and multiplication. || MCPToolkit | A toolkit for interacting with external tools using the Model Context Protocol (MCP). |diff --git a/examples/toolkits/jina_reranker_toolkit.py b/examples/toolkits/jina_reranker_toolkit.pynew file mode 100644index 000000000..fefc4ccf7--- /dev/null+++ b/examples/toolkits/jina_reranker_toolkit.py@@ -0,0 +1,63 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from camel.agents import ChatAgent+from camel.models import ModelFactory+from camel.toolkits import JinaRerankerToolkit+from camel.types import ModelPlatformType, ModelType++model = ModelFactory.create(+ model_platform=ModelPlatformType.DEFAULT,+ model_type=ModelType.DEFAULT,+)++reranker_toolkit = JinaRerankerToolkit(device="cpu")+reranker_tool = reranker_toolkit.get_tools()++agent = ChatAgent(model=model, tools=reranker_tool)++documents = [+ "Markdown is a lightweight markup language with plain-text "+ "formatting syntax.",+ "Python is a high-level, interpreted programming language known for "+ "its readability.",+ "SLM (Small Language Models) are compact AI models designed for "+ "specific tasks.",+ "JavaScript is a scripting language primarily used for "+ "creating interactive web pages.",+]++query = "How to use markdown with small language models"++response = agent.step(+ f"Can you rerank these documents {documents} against the query {query}"+)+print(str(response.info['tool_calls'])[:1000])+""""+===========================================================================+[ToolCallingRecord(tool_name='rerank_text_documents', args={'query': 'How to+use markdown with small language models', 'documents': ['Markdown is a+lightweight markup language with plain-text formatting syntax.', 'Python is a+high-level, interpreted programming language known for its readability.', 'SLM+(Small Language Models) are compact AI models designed for specific tasks.',+'JavaScript is a scripting language primarily used for creating interactive+web pages.'], 'max_length': 1024}, result=[('Markdown is a lightweight markup+language with plain-text formatting syntax.', 0.7915633916854858), ('SLM+(Small Language Models) are compact AI models designed for specific tasks.', 0.+7915633916854858), ('Python is a high-level, interpreted programming language+known for its readability.', 0.43936243653297424), ('JavaScript is a scripting+language primarily used for creating interactive web pages.', 0.+3716837763786316)], tool_call_id='call_JKnuvTO1fUQP7PWhyCSQCK7N')]+===========================================================================+"""diff --git a/test/toolkits/test_jina_reranker_toolkit.py b/test/toolkits/test_jina_reranker_toolkit.pynew file mode 100644index 000000000..418fd43f7--- /dev/null+++ b/test/toolkits/test_jina_reranker_toolkit.py@@ -0,0 +1,236 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from unittest.mock import MagicMock, patch++import pytest++from camel.toolkits.jina_reranker_toolkit import JinaRerankerToolkit+++def mock_model():+ r"""Fixture for mocking the Jina Reranker model."""+ model = MagicMock()+ model.compute_score = MagicMock()+ return model+++def reranker_toolkit(mock_model):+ r"""Fixture for creating a JinaRerankerToolkit with a mocked model."""+ with patch(+ 'transformers.AutoModel.from_pretrained', return_value=mock_model+ ):+ toolkit = JinaRerankerToolkit()+ # Replace the actual model with our mock+ toolkit.model = mock_model+ return toolkit+++def test_init_with_default_params():+ r"""Test initialization with default parameters."""+ with patch(+ 'transformers.AutoModel.from_pretrained'+ ) as mock_from_pretrained:+ with patch('torch.cuda.is_available', return_value=False):+ JinaRerankerToolkit()+ mock_from_pretrained.assert_called_once_with(+ 'jinaai/jina-reranker-m0',+ torch_dtype="auto",+ trust_remote_code=True,+ )+++def test_init_with_custom_device():+ r"""Test initialization with custom device."""+ with patch(+ 'transformers.AutoModel.from_pretrained'+ ) as mock_from_pretrained:+ with patch('torch.cuda.is_available', return_value=True):+ model_mock = MagicMock()+ mock_from_pretrained.return_value = model_mock++ JinaRerankerToolkit(device="cpu")++ model_mock.to.assert_called_once_with("cpu")+ model_mock.eval.assert_called_once()+++def test_sort_documents():+ r"""Test the _sort_documents method."""+ toolkit = JinaRerankerToolkit()+ documents = ["doc1", "doc2", "doc3"]+ scores = [0.5, 0.9, 0.1]++ result = toolkit._sort_documents(documents, scores)++ # Should be sorted by score in descending order+ assert result == [("doc2", 0.9), ("doc1", 0.5), ("doc3", 0.1)]+++def test_sort_documents_with_mismatched_lengths():+ r"""Test _sort_documents with mismatched document and score lists."""+ toolkit = JinaRerankerToolkit()+ documents = ["doc1", "doc2"]+ scores = [0.5, 0.9, 0.1]++ with pytest.raises(+ ValueError, match="Number of documents must match number of scores"+ ):+ toolkit._sort_documents(documents, scores)+++def test_rerank_text_documents(reranker_toolkit):+ r"""Test reranking text documents."""+ query = "test query"+ documents = ["doc1", "doc2", "doc3"]+ scores = [0.7, 0.3, 0.9]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.rerank_text_documents(query, documents)++ # Check that compute_score was called with the correct parameters+ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[query, doc] for doc in documents], max_length=1024, doc_type="text"+ )++ # Check that the result is sorted correctly+ assert result == [("doc3", 0.9), ("doc1", 0.7), ("doc2", 0.3)]+++def test_rerank_text_documents_with_custom_max_length(reranker_toolkit):+ r"""Test reranking text documents with custom max_length."""+ query = "test query"+ documents = ["doc1", "doc2"]+ scores = [0.6, 0.8]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.rerank_text_documents(+ query, documents, max_length=512+ )++ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[query, doc] for doc in documents], max_length=512, doc_type="text"+ )++ assert result == [("doc2", 0.8), ("doc1", 0.6)]+++def test_rerank_text_documents_model_not_initialized():+ r"""Test reranking text documents when model is not initialized."""+ toolkit = JinaRerankerToolkit()+ toolkit.model = None++ with pytest.raises(+ ValueError,+ match="Model has not been initialized or failed to initialize.",+ ):+ toolkit.rerank_text_documents("query", ["doc1"])+++def test_rerank_image_documents(reranker_toolkit):+ r"""Test reranking image documents."""+ query = "test query"+ documents = ["image1.jpg", "image2.jpg"]+ scores = [0.4, 0.8]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.rerank_image_documents(query, documents)++ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[query, doc] for doc in documents], max_length=2048, doc_type="image"+ )++ assert result == [("image2.jpg", 0.8), ("image1.jpg", 0.4)]+++def test_rerank_image_documents_custom_max_length(reranker_toolkit):+ r"""Test reranking image documents with custom max_length."""+ query = "test query"+ documents = ["image1.jpg", "image2.jpg"]+ scores = [0.4, 0.8]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.rerank_image_documents(+ query, documents, max_length=1024+ )++ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[query, doc] for doc in documents], max_length=1024, doc_type="image"+ )++ assert result == [("image2.jpg", 0.8), ("image1.jpg", 0.4)]+++def test_image_query_text_documents(reranker_toolkit):+ r"""Test reranking text documents with an image query."""+ image_query = "image_query.jpg"+ documents = ["doc1", "doc2"]+ scores = [0.3, 0.7]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.image_query_text_documents(+ image_query, documents+ )++ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[image_query, doc] for doc in documents],+ max_length=2048,+ query_type="image",+ doc_type="text",+ )++ assert result == [("doc2", 0.7), ("doc1", 0.3)]+++def test_image_query_image_documents(reranker_toolkit):+ r"""Test reranking image documents with an image query."""+ image_query = "image_query.jpg"+ documents = ["image1.jpg", "image2.jpg"]+ scores = [0.6, 0.2]++ reranker_toolkit.model.compute_score.return_value = scores++ result = reranker_toolkit.image_query_image_documents(+ image_query, documents+ )++ reranker_toolkit.model.compute_score.assert_called_once_with(+ [[image_query, doc] for doc in documents],+ max_length=2048,+ query_type="image",+ doc_type="image",+ )++ assert result == [("image1.jpg", 0.6), ("image2.jpg", 0.2)]+++def test_with_inference_mode(reranker_toolkit):+ r"""Test that torch.inference_mode is used correctly."""+ with patch('torch.inference_mode') as mock_inference_mode:+ mock_context = MagicMock()+ mock_inference_mode.return_value = mock_context++ reranker_toolkit.model.compute_score.return_value = [0.5]++ reranker_toolkit.rerank_text_documents("query", ["doc"])++ mock_inference_mode.assert_called_once()+ mock_context.__enter__.assert_called_once()+ mock_context.__exit__.assert_called_once()