Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: jina reranker refactor + api support (#2258)

JINO ROHIT885298b47cc9
diff --git a/camel/toolkits/jina_reranker_toolkit.py b/camel/toolkits/jina_reranker_toolkit.py
index 8187ecaa8..02e443340 100644
--- a/camel/toolkits/jina_reranker_toolkit.py
+++ b/camel/toolkits/jina_reranker_toolkit.py
@@ -11,7 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-from typing import List, Optional, Tuple
+import json
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
from camel.toolkits import FunctionTool
from camel.toolkits.base import BaseToolkit
@@ -30,7 +34,9 @@ class JinaRerankerToolkit(BaseToolkit):
def __init__(
self,
timeout: Optional[float] = None,
+ model_name: Optional[str] = "jinaai/jina-reranker-m0",
device: Optional[str] = None,
+ use_api: bool = True,
) -> None:
r"""Initializes a new instance of the JinaRerankerToolkit class.
@@ -38,31 +44,57 @@ class JinaRerankerToolkit(BaseToolkit):
timeout (Optional[float]): The timeout value for API requests
in seconds. If None, no timeout is applied.
(default: :obj:`None`)
+ model_name (Optional[str]): The reranker model name. If None,
+ will use the default model.
+ (default: :obj:`None`)
device (Optional[str]): Device to load the model on. If None,
will use CUDA if available, otherwise CPU.
+ Only effective when use_api=False.
(default: :obj:`None`)
+ use_api (bool): A flag to switch between local model and API.
+ (default: :obj:`True`)
"""
- import torch
- from transformers import AutoModel
super().__init__(timeout=timeout)
- self.model = AutoModel.from_pretrained(
- 'jinaai/jina-reranker-m0',
- torch_dtype="auto",
- trust_remote_code=True,
- )
- DEVICE = (
- device
- if device is not None
- else ("cuda" if torch.cuda.is_available() else "cpu")
- )
- self.model.to(DEVICE)
- self.model.eval()
+ self.use_api = use_api
+ self.model_name = model_name
+
+ if self.use_api:
+ self.model = None
+ self._api_key = os.environ.get("JINA_API_KEY", "None")
+ if self._api_key == "None":
+ raise ValueError(
+ "Missing or empty required API keys in "
+ "environment variables\n"
+ "You can obtain the API key from https://jina.ai/reranker/"
+ )
+ self.url = 'https://api.jina.ai/v1/rerank'
+ self.headers = {
+ 'Content-Type': 'application/json',
+ 'Accept': 'application/json',
+ 'Authorization': f'Bearer {self._api_key}',
+ }
+ else:
+ import torch
+ from transformers import AutoModel
+
+ self.model = AutoModel.from_pretrained(
+ self.model_name,
+ torch_dtype="auto",
+ trust_remote_code=True,
+ )
+ self.device = (
+ device
+ if device is not None
+ else ("cuda" if torch.cuda.is_available() else "cpu")
+ )
+ self.model.to(self.device)
+ self.model.eval()
def _sort_documents(
self, documents: List[str], scores: List[float]
- ) -> List[Tuple[str, float]]:
+ ) -> List[Dict[str, object]]:
r"""Sort documents by their scores in descending order.
Args:
@@ -70,7 +102,7 @@ class JinaRerankerToolkit(BaseToolkit):
scores (List[float]): Corresponding scores for each document.
Returns:
- List[Tuple[str, float]]: Sorted list of (document, score) pairs.
+ List[Dict[str, object]]: Sorted list of (document, score) pairs.
Raises:
ValueError: If documents and scores have different lengths.
@@ -80,14 +112,45 @@ class JinaRerankerToolkit(BaseToolkit):
doc_score_pairs = list(zip(documents, scores))
doc_score_pairs.sort(key=lambda x: x[1], reverse=True)
- return doc_score_pairs
+ results = [
+ {'document': {'text': doc}, 'relevance_score': score}
+ for doc, score in doc_score_pairs
+ ]
+
+ return results
+
+ def _call_jina_api(self, data: Dict[str, Any]) -> List[Dict[str, object]]:
+ r"""Makes a call to the JINA API for reranking.
+
+ Args:
+ data (Dict[str]): The data to be passed into the api body.
+
+ Returns:
+ List[Dict[str, object]]: A list of dictionary containing
+ the reranked documents and their relevance scores.
+ """
+ try:
+ response = requests.post(
+ self.url,
+ headers=self.headers,
+ data=json.dumps(data),
+ timeout=self.timeout,
+ )
+ response.raise_for_status()
+ results = [
+ {key: value for key, value in _res.items() if key != 'index'}
+ for _res in response.json()['results']
+ ]
+ return results
+ except requests.exceptions.RequestException as e:
+ raise RuntimeError(f"Failed to get response from Jina AI: {e}")
def rerank_text_documents(
self,
query: str,
documents: List[str],
max_length: int = 1024,
- ) -> List[Tuple[str, float]]:
+ ) -> List[Dict[str, object]]:
r"""Reranks text documents based on their relevance to a text query.
Args:
@@ -97,21 +160,34 @@ class JinaRerankerToolkit(BaseToolkit):
(default: :obj:`1024`)
Returns:
- List[Tuple[str, float]]: A list of tuples containing
+ List[Dict[str, object]]: A list of dictionary containing
the reranked documents and their relevance scores.
"""
- import torch
- if self.model is None:
- raise ValueError(
- "Model has not been initialized or failed to initialize."
- )
+ data = {
+ 'model': self.model_name,
+ 'query': query,
+ 'top_n': len(documents),
+ 'documents': documents,
+ 'return_documents': True,
+ }
- with torch.inference_mode():
- text_pairs = [[query, doc] for doc in documents]
- scores = self.model.compute_score(
- text_pairs, max_length=max_length, doc_type="text"
- )
+ if self.use_api:
+ return self._call_jina_api(data)
+
+ else:
+ import torch
+
+ if self.model is None:
+ raise ValueError(
+ "Model has not been initialized or failed to initialize."
+ )
+
+ with torch.inference_mode():
+ text_pairs = [[query, doc] for doc in documents]
+ scores = self.model.compute_score(
+ text_pairs, max_length=max_length, doc_type="text"
+ )
return self._sort_documents(documents, scores)
@@ -120,7 +196,7 @@ class JinaRerankerToolkit(BaseToolkit):
query: str,
documents: List[str],
max_length: int = 2048,
- ) -> List[Tuple[str, float]]:
+ ) -> List[Dict[str, object]]:
r"""Reranks image documents based on their relevance to a text query.
Args:
@@ -130,21 +206,33 @@ class JinaRerankerToolkit(BaseToolkit):
(default: :obj:`2048`)
Returns:
- List[Tuple[str, float]]: A list of tuples containing
+ List[Dict[str, object]]: A list of dictionary containing
the reranked image URLs/paths and their relevance scores.
"""
- import torch
-
- if self.model is None:
- raise ValueError(
- "Model has not been initialized or failed to initialize."
- )
-
- with torch.inference_mode():
- image_pairs = [[query, doc] for doc in documents]
- scores = self.model.compute_score(
- image_pairs, max_length=max_length, doc_type="image"
- )
+ data = {
+ 'model': self.model_name,
+ 'query': query,
+ 'top_n': len(documents),
+ 'documents': documents,
+ 'return_documents': True,
+ }
+
+ if self.use_api:
+ return self._call_jina_api(data)
+
+ else:
+ import torch
+
+ if self.model is None:
+ raise ValueError(
+ "Model has not been initialized or failed to initialize."
+ )
+
+ with torch.inference_mode():
+ image_pairs = [[query, doc] for doc in documents]
+ scores = self.model.compute_score(
+ image_pairs, max_length=max_length, doc_type="image"
+ )
return self._sort_documents(documents, scores)
@@ -153,7 +241,7 @@ class JinaRerankerToolkit(BaseToolkit):
image_query: str,
documents: List[str],
max_length: int = 2048,
- ) -> List[Tuple[str, float]]:
+ ) -> List[Dict[str, object]]:
r"""Reranks text documents based on their relevance to an image query.
Args:
@@ -163,30 +251,45 @@ class JinaRerankerToolkit(BaseToolkit):
(default: :obj:`2048`)
Returns:
- List[Tuple[str, float]]: A list of tuples containing
+ List[Dict[str, object]]: A list of dictionary containing
the reranked documents and their relevance scores.
"""
- import torch
-
- if self.model is None:
- raise ValueError("Model has not been initialized.")
- with torch.inference_mode():
- image_pairs = [[image_query, doc] for doc in documents]
- scores = self.model.compute_score(
- image_pairs,
- max_length=max_length,
- query_type="image",
- doc_type="text",
- )
-
- return self._sort_documents(documents, scores)
+ data = {
+ 'model': self.model_name,
+ 'query': image_query,
+ 'top_n': len(documents),
+ 'documents': documents,
+ 'return_documents': True,
+ }
+
+ if self.use_api:
+ return self._call_jina_api(data)
+
+ else:
+ import torch
+
+ if self.model is None:
+ raise ValueError(
+ "Model has not been initialized or failed to initialize."
+ )
+
+ with torch.inference_mode():
+ image_pairs = [[image_query, doc] for doc in documents]
+ scores = self.model.compute_score(
+ image_pairs,
+ max_length=max_length,
+ query_type="image",
+ doc_type="text",
+ )
+
+ return self._sort_documents(documents, scores)
def image_query_image_documents(
self,
image_query: str,
documents: List[str],
max_length: int = 2048,
- ) -> List[Tuple[str, float]]:
+ ) -> List[Dict[str, object]]:
r"""Reranks image documents based on their relevance to an image query.
Args:
@@ -196,24 +299,38 @@ class JinaRerankerToolkit(BaseToolkit):
(default: :obj:`2048`)
Returns:
- List[Tuple[str, float]]: A list of tuples containing
+ List[Dict[str, object]]: A list of dictionary containing
the reranked image URLs/paths and their relevance scores.
"""
- import torch
-
- if self.model is None:
- raise ValueError("Model has not been initialized.")
-
- with torch.inference_mode():
- image_pairs = [[image_query, doc] for doc in documents]
- scores = self.model.compute_score(
- image_pairs,
- max_length=max_length,
- query_type="image",
- doc_type="image",
- )
-
- return self._sort_documents(documents, scores)
+ data = {
+ 'model': self.model_name,
+ 'query': image_query,
+ 'top_n': len(documents),
+ 'documents': documents,
+ 'return_documents': True,
+ }
+
+ if self.use_api:
+ return self._call_jina_api(data)
+
+ else:
+ import torch
+
+ if self.model is None:
+ raise ValueError(
+ "Model has not been initialized or failed to initialize."
+ )
+
+ with torch.inference_mode():
+ image_pairs = [[image_query, doc] for doc in documents]
+ scores = self.model.compute_score(
+ image_pairs,
+ max_length=max_length,
+ query_type="image",
+ doc_type="image",
+ )
+
+ return self._sort_documents(documents, scores)
def get_tools(self) -> List[FunctionTool]:
r"""Returns a list of FunctionTool objects representing the
diff --git a/examples/toolkits/jina_reranker_toolkit.py b/examples/toolkits/jina_reranker_toolkit.py
index fefc4ccf7..18db17ea7 100644
--- a/examples/toolkits/jina_reranker_toolkit.py
+++ b/examples/toolkits/jina_reranker_toolkit.py
@@ -22,7 +22,7 @@ model = ModelFactory.create(
model_type=ModelType.DEFAULT,
)
-reranker_toolkit = JinaRerankerToolkit(device="cpu")
+reranker_toolkit = JinaRerankerToolkit(device="cpu", use_api=False)
reranker_tool = reranker_toolkit.get_tools()
agent = ChatAgent(model=model, tools=reranker_tool)
diff --git a/test/toolkits/test_jina_reranker_toolkit.py b/test/toolkits/test_jina_reranker_toolkit.py
index cf9549ba1..bcc4c7a6e 100644
--- a/test/toolkits/test_jina_reranker_toolkit.py
+++ b/test/toolkits/test_jina_reranker_toolkit.py
@@ -32,7 +32,7 @@ def reranker_toolkit(mock_model):
with patch(
'transformers.AutoModel.from_pretrained', return_value=mock_model
):
- toolkit = JinaRerankerToolkit()
+ toolkit = JinaRerankerToolkit(use_api=False)
# Replace the actual model with our mock
toolkit.model = mock_model
return toolkit
@@ -44,7 +44,7 @@ def test_init_with_default_params():
'transformers.AutoModel.from_pretrained'
) as mock_from_pretrained:
with patch('torch.cuda.is_available', return_value=False):
- JinaRerankerToolkit()
+ JinaRerankerToolkit(use_api=False)
mock_from_pretrained.assert_called_once_with(
'jinaai/jina-reranker-m0',
torch_dtype="auto",
@@ -61,7 +61,7 @@ def test_init_with_custom_device():
model_mock = MagicMock()
mock_from_pretrained.return_value = model_mock
- JinaRerankerToolkit(device="cpu")
+ JinaRerankerToolkit(use_api=False, device="cpu")
model_mock.to.assert_called_once_with("cpu")
model_mock.eval.assert_called_once()
@@ -72,14 +72,18 @@ def test_init_with_custom_device():
)
def test_sort_documents():
r"""Test the _sort_documents method."""
- toolkit = JinaRerankerToolkit()
+ toolkit = JinaRerankerToolkit(use_api=False)
documents = ["doc1", "doc2", "doc3"]
scores = [0.5, 0.9, 0.1]
result = toolkit._sort_documents(documents, scores)
# Should be sorted by score in descending order
- assert result == [("doc2", 0.9), ("doc1", 0.5), ("doc3", 0.1)]
+ assert result == [
+ {"document": "doc2", "score": 0.9},
+ {"document": "doc1", "score": 0.5},
+ {"document": "doc3", "score": 0.1},
+ ]
@pytest.mark.skip(
@@ -87,7 +91,7 @@ def test_sort_documents():
)
def test_sort_documents_with_mismatched_lengths():
r"""Test _sort_documents with mismatched document and score lists."""
- toolkit = JinaRerankerToolkit()
+ toolkit = JinaRerankerToolkit(use_api=False)
documents = ["doc1", "doc2"]
scores = [0.5, 0.9, 0.1]
@@ -113,7 +117,11 @@ def test_rerank_text_documents(reranker_toolkit):
)
# Check that the result is sorted correctly
- assert result == [("doc3", 0.9), ("doc1", 0.7), ("doc2", 0.3)]
+ assert result == [
+ {'document': {'text': 'doc3'}, 'relevance_score': 0.9},
+ {'document': {'text': 'doc1'}, 'relevance_score': 0.7},
+ {'document': {'text': 'doc2'}, 'relevance_score': 0.3},
+ ]
def test_rerank_text_documents_with_custom_max_length(reranker_toolkit):
@@ -132,7 +140,10 @@ def test_rerank_text_documents_with_custom_max_length(reranker_toolkit):
[[query, doc] for doc in documents], max_length=512, doc_type="text"
)
- assert result == [("doc2", 0.8), ("doc1", 0.6)]
+ assert result == [
+ {'document': {'text': 'doc2'}, 'relevance_score': 0.8},
+ {'document': {'text': 'doc1'}, 'relevance_score': 0.6},
+ ]
@pytest.mark.skip(
@@ -140,7 +151,7 @@ def test_rerank_text_documents_with_custom_max_length(reranker_toolkit):
)
def test_rerank_text_documents_model_not_initialized():
r"""Test reranking text documents when model is not initialized."""
- toolkit = JinaRerankerToolkit()
+ toolkit = JinaRerankerToolkit(use_api=False)
toolkit.model = None
with pytest.raises(
@@ -164,7 +175,10 @@ def test_rerank_image_documents(reranker_toolkit):
[[query, doc] for doc in documents], max_length=2048, doc_type="image"
)
- assert result == [("image2.jpg", 0.8), ("image1.jpg", 0.4)]
+ assert result == [
+ {'document': {'text': 'image2.jpg'}, 'relevance_score': 0.8},
+ {'document': {'text': 'image1.jpg'}, 'relevance_score': 0.4},
+ ]
def test_rerank_image_documents_custom_max_length(reranker_toolkit):
@@ -183,7 +197,10 @@ def test_rerank_image_documents_custom_max_length(reranker_toolkit):
[[query, doc] for doc in documents], max_length=1024, doc_type="image"
)
- assert result == [("image2.jpg", 0.8), ("image1.jpg", 0.4)]
+ assert result == [
+ {'document': {'text': 'image2.jpg'}, 'relevance_score': 0.8},
+ {'document': {'text': 'image1.jpg'}, 'relevance_score': 0.4},
+ ]
def test_image_query_text_documents(reranker_toolkit):
@@ -205,7 +222,10 @@ def test_image_query_text_documents(reranker_toolkit):
doc_type="text",
)
- assert result == [("doc2", 0.7), ("doc1", 0.3)]
+ assert result == [
+ {'document': {'text': 'doc2'}, 'relevance_score': 0.7},
+ {'document': {'text': 'doc1'}, 'relevance_score': 0.3},
+ ]
def test_image_query_image_documents(reranker_toolkit):
@@ -227,7 +247,10 @@ def test_image_query_image_documents(reranker_toolkit):
doc_type="image",
)
- assert result == [("image1.jpg", 0.6), ("image2.jpg", 0.2)]
+ assert result == [
+ {'document': {'text': 'image1.jpg'}, 'relevance_score': 0.6},
+ {'document': {'text': 'image2.jpg'}, 'relevance_score': 0.2},
+ ]
def test_with_inference_mode(reranker_toolkit):