Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Reward model (#1271)

Asher-hssb3a2fde31a22
diff --git a/camel/models/reward/__init__.py b/camel/models/reward/__init__.py
new file mode 100644
index 000000000..f09523733
--- /dev/null
+++ b/camel/models/reward/__init__.py
@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base_reward_model import BaseRewardModel
+from .evaluator import Evaluator
+from .nemetro_model import NemotroRewardModel
+
+__all__ = [
+ 'BaseRewardModel',
+ 'NemotroRewardModel',
+ 'Evaluator',
+]
diff --git a/camel/models/reward/base_reward_model.py b/camel/models/reward/base_reward_model.py
new file mode 100644
index 000000000..937fe07ff
--- /dev/null
+++ b/camel/models/reward/base_reward_model.py
@@ -0,0 +1,58 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, Union
+
+from camel.types import ModelType
+
+
+class BaseRewardModel(ABC):
+ r"""Abstract base class for reward models. Reward models are used to
+ evaluate messages and return scores based on different criteria.
+
+ Subclasses should implement the 'evaluate' and 'get_scores_types' methods.
+ """
+
+ def __init__(
+ self,
+ model_type: Union[ModelType, str],
+ api_key: Optional[str] = None,
+ url: Optional[str] = None,
+ ) -> None:
+ self.model_type = model_type
+ self.api_key = api_key
+ self.url = url
+
+ @abstractmethod
+ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:
+ r"""Evaluate the messages and return scores based on different
+ criteria.
+
+ Args:
+ messages (List[Dict[str, str]]): A list of messages where each
+ message is a dictionary with 'role' and 'content'.
+
+ Returns:
+ Dict[str, float]: A dictionary mapping score types to their values.
+ """
+ pass
+
+ @abstractmethod
+ def get_scores_types(self) -> List[str]:
+ r"""Get the list of score types that the reward model can return.
+
+ Returns:
+ List[str]: A list of score types that the reward model can return.
+ """
+ pass
diff --git a/camel/models/reward/evaluator.py b/camel/models/reward/evaluator.py
new file mode 100644
index 000000000..5f3e6b2d6
--- /dev/null
+++ b/camel/models/reward/evaluator.py
@@ -0,0 +1,63 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Dict, List
+
+from camel.models.reward import BaseRewardModel
+
+
+class Evaluator:
+ r"""Evaluator class to evaluate messages using a reward model and filter
+ data based on the scores.
+
+ Args:
+ reward_model (BaseRewardModel): A reward model to evaluate messages.
+ """
+
+ def __init__(self, reward_model: BaseRewardModel):
+ self.reward_model = reward_model
+
+ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:
+ r"""Evaluate the messages using the reward model.
+
+ Args:
+ messages (List[Dict[str, str]]): A list of messages where each
+ message is a dictionary with 'role' and 'content'.
+
+ Returns:
+ Dict[str, float]: A dictionary mapping score types to their values.
+ """
+ scores = self.reward_model.evaluate(messages)
+ return scores
+
+ def filter_data(
+ self, messages: List[Dict[str, str]], thresholds: Dict[str, float]
+ ) -> bool:
+ r"""Filter messages based on the scores.
+
+ Args:
+ messages (List[Dict[str, str]]): A list of messages where each
+ message is a dictionary with 'role' and 'content'.
+ thresholds (Dict[str, float]): A dictionary mapping score types to
+ their values.
+
+ Returns:
+ bool: A boolean indicating whether the messages pass the filter.
+ """
+ scores = self.evaluate(messages)
+ for score_type, threshold in thresholds.items():
+ if score_type not in scores:
+ raise ValueError(f"Score type {score_type} not found.")
+ if scores.get(score_type, 0) < threshold:
+ return False
+ return True
diff --git a/camel/models/reward/nemetro_model.py b/camel/models/reward/nemetro_model.py
new file mode 100644
index 000000000..762466316
--- /dev/null
+++ b/camel/models/reward/nemetro_model.py
@@ -0,0 +1,112 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import Dict, List, Optional, Union
+
+from openai import OpenAI
+
+from camel.models.reward import BaseRewardModel
+from camel.types import ChatCompletion, ModelType
+from camel.utils import api_keys_required
+
+
+class NemotroRewardModel(BaseRewardModel):
+ r"""Reward model based on the Nemetro model with OpenAI compatibility.
+
+ Args:
+ model_type (Union[ModelType, str]): Model for which a backend is
+ created.
+ api_key (Optional[str], optional): The API key for authenticating
+ with the model service. (default: :obj:`None`)
+ url (Optional[str], optional): The url to the model service.
+
+ Note:
+ The Nemetro model does not support model config.
+ """
+
+ def __init__(
+ self,
+ model_type: Union[ModelType, str],
+ api_key: Optional[str] = None,
+ url: Optional[str] = None,
+ ) -> None:
+ url = url or os.environ.get(
+ "NVIDIA_API_BASE_URL", "https://integrate.api.nvidia.com/v1"
+ )
+ api_key = api_key or os.environ.get("NVIDIA_API_KEY")
+ super().__init__(model_type, api_key, url)
+ self._client = OpenAI(
+ timeout=60,
+ max_retries=3,
+ base_url=self.url,
+ api_key=self.api_key,
+ )
+
+ @api_keys_required("NVIDIA_API_KEY")
+ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:
+ r"""Evaluate the messages using the Nemetro model.
+
+ Args:
+ messages (List[Dict[str, str]]): A list of messages where each
+ message is a dictionary format.
+
+ Returns:
+ Dict[str, float]: A dictionary mapping score types to their
+ values.
+ """
+ response = self._client.chat.completions.create(
+ messages=messages, # type: ignore[arg-type]
+ model=self.model_type,
+ )
+ scores = self._parse_scores(response)
+ return scores
+
+ def get_scores_types(self) -> List[str]:
+ r"""Get the list of score types that the reward model can return.
+
+ Returns:
+ List[str]: A list of score types that the reward model can return.
+ """
+ return [
+ "helpfulness",
+ "correctness",
+ "coherence",
+ "complexity",
+ "verbosity",
+ ]
+
+ def _parse_scores(self, response: ChatCompletion) -> Dict[str, float]:
+ r"""Parse the scores from the response.
+
+ Args:
+ response (ChatCompletion): A ChatCompletion object with the scores.
+
+ Returns:
+ Dict[str, float]: A dictionary mapping score types to their values.
+ """
+ try:
+ choices = response.choices
+ logprobs = (
+ choices[0].logprobs.content
+ if choices and choices[0].logprobs
+ else None
+ )
+ scores = (
+ {entry.token: entry.logprob for entry in logprobs if entry}
+ if logprobs
+ else {}
+ )
+ return scores
+ except Exception as e:
+ raise ValueError(f"Failed to parse scores: {e}")
diff --git a/examples/reward/nemotron_model_example.py b/examples/reward/nemotron_model_example.py
new file mode 100644
index 000000000..1a78ada34
--- /dev/null
+++ b/examples/reward/nemotron_model_example.py
@@ -0,0 +1,57 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from camel.models.reward import Evaluator, NemotroRewardModel
+from camel.types import ModelType
+
+reward_model = NemotroRewardModel(
+ model_type=ModelType.NVIDIA_NEMOTRON_340B_REWARD,
+ url="https://integrate.api.nvidia.com/v1",
+)
+
+evaluator = Evaluator(reward_model=reward_model)
+
+messages = [
+ {"role": "user", "content": "I am going to Paris, what should I see?"},
+ {
+ "role": "assistant",
+ "content": "Ah, Paris, the City of Light! There are so many amazing "
+ "things to see and do in this beautiful city ...",
+ },
+]
+
+scores = evaluator.evaluate(messages)
+print("Scores: ", scores)
+'''
+===============================================================================
+Scores: {'helpfulness': 1.6171875, 'correctness': 1.6484375, 'coherence': 3.
+3125, 'complexity': 0.546875, 'verbosity': 0.515625}
+===============================================================================
+'''
+
+scores_types = reward_model.get_scores_types()
+print("Types: ", scores_types)
+'''
+===============================================================================
+Types: ['helpfulness', 'correctness', 'coherence', 'complexity', 'verbosity']
+===============================================================================
+'''
+
+threshoids = {"helpfulness": 1.5, "correctness": 1.5}
+is_acceptable = evaluator.filter_data(messages, threshoids)
+print("Is acceptable: ", is_acceptable)
+'''
+===============================================================================
+Is acceptable: True
+===============================================================================
+'''
diff --git a/test/models/reward/test_nemetro_model.py b/test/models/reward/test_nemetro_model.py
new file mode 100644
index 000000000..bf9d1d40b
--- /dev/null
+++ b/test/models/reward/test_nemetro_model.py
@@ -0,0 +1,87 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from camel.models.reward import NemotroRewardModel
+from camel.types import ModelType
+
+
+ "model_type",
+ [
+ ModelType.NVIDIA_NEMOTRON_340B_REWARD,
+ ],
+)
+@patch("camel.models.reward.nemetro_model.OpenAI")
+def test_nemetro_reward_model(mock_openai, model_type: ModelType):
+ mock_client = MagicMock()
+ mock_openai.return_value = mock_client
+
+ model = NemotroRewardModel(model_type)
+ assert model.model_type == model_type
+ assert model._client == mock_client
+
+
+@patch("camel.models.reward.nemetro_model.OpenAI")
+def test_nemetro_reward_model_evaluate(mock_openai):
+ mock_client = MagicMock()
+ mock_openai.return_value = mock_client
+
+ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)
+ messages = [{"role": "user", "content": "Hello"}]
+ mock_response = MagicMock()
+ mock_client.chat.completions.create.return_value = mock_response
+
+ with patch.object(
+ model, "_parse_scores", return_value={"helpfulness": 0.9}
+ ):
+ scores = model.evaluate(messages)
+ assert scores == {"helpfulness": 0.9}
+ mock_client.chat.completions.create.assert_called_once_with(
+ messages=messages, model=ModelType.NVIDIA_NEMOTRON_340B_REWARD
+ )
+
+
+def test_nemetro_reward_model_get_scores_types():
+ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)
+ score_types = model.get_scores_types()
+ assert score_types == [
+ "helpfulness",
+ "correctness",
+ "coherence",
+ "complexity",
+ "verbosity",
+ ]
+
+
+def test_nemetro_reward_model_parse_scores():
+ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)
+ mock_response = MagicMock()
+ mock_response.choices = [
+ MagicMock(
+ logprobs=MagicMock(
+ content=[MagicMock(token="helpfulness", logprob=0.9)]
+ )
+ )
+ ]
+
+ scores = model._parse_scores(mock_response)
+ assert scores == {"helpfulness": 0.9}