🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Reward model (#1271)
Asher-hssb3a2fde31a22
diff --git a/camel/models/reward/__init__.py b/camel/models/reward/__init__.pynew file mode 100644index 000000000..f09523733--- /dev/null+++ b/camel/models/reward/__init__.py@@ -0,0 +1,22 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from .base_reward_model import BaseRewardModel+from .evaluator import Evaluator+from .nemetro_model import NemotroRewardModel++__all__ = [+ 'BaseRewardModel',+ 'NemotroRewardModel',+ 'Evaluator',+]diff --git a/camel/models/reward/base_reward_model.py b/camel/models/reward/base_reward_model.pynew file mode 100644index 000000000..937fe07ff--- /dev/null+++ b/camel/models/reward/base_reward_model.py@@ -0,0 +1,58 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from abc import ABC, abstractmethod+from typing import Dict, List, Optional, Union++from camel.types import ModelType+++class BaseRewardModel(ABC):+ r"""Abstract base class for reward models. Reward models are used to+ evaluate messages and return scores based on different criteria.++ Subclasses should implement the 'evaluate' and 'get_scores_types' methods.+ """++ def __init__(+ self,+ model_type: Union[ModelType, str],+ api_key: Optional[str] = None,+ url: Optional[str] = None,+ ) -> None:+ self.model_type = model_type+ self.api_key = api_key+ self.url = url++ @abstractmethod+ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:+ r"""Evaluate the messages and return scores based on different+ criteria.++ Args:+ messages (List[Dict[str, str]]): A list of messages where each+ message is a dictionary with 'role' and 'content'.++ Returns:+ Dict[str, float]: A dictionary mapping score types to their values.+ """+ pass++ @abstractmethod+ def get_scores_types(self) -> List[str]:+ r"""Get the list of score types that the reward model can return.++ Returns:+ List[str]: A list of score types that the reward model can return.+ """+ passdiff --git a/camel/models/reward/evaluator.py b/camel/models/reward/evaluator.pynew file mode 100644index 000000000..5f3e6b2d6--- /dev/null+++ b/camel/models/reward/evaluator.py@@ -0,0 +1,63 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from typing import Dict, List++from camel.models.reward import BaseRewardModel+++class Evaluator:+ r"""Evaluator class to evaluate messages using a reward model and filter+ data based on the scores.++ Args:+ reward_model (BaseRewardModel): A reward model to evaluate messages.+ """++ def __init__(self, reward_model: BaseRewardModel):+ self.reward_model = reward_model++ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:+ r"""Evaluate the messages using the reward model.++ Args:+ messages (List[Dict[str, str]]): A list of messages where each+ message is a dictionary with 'role' and 'content'.++ Returns:+ Dict[str, float]: A dictionary mapping score types to their values.+ """+ scores = self.reward_model.evaluate(messages)+ return scores++ def filter_data(+ self, messages: List[Dict[str, str]], thresholds: Dict[str, float]+ ) -> bool:+ r"""Filter messages based on the scores.++ Args:+ messages (List[Dict[str, str]]): A list of messages where each+ message is a dictionary with 'role' and 'content'.+ thresholds (Dict[str, float]): A dictionary mapping score types to+ their values.++ Returns:+ bool: A boolean indicating whether the messages pass the filter.+ """+ scores = self.evaluate(messages)+ for score_type, threshold in thresholds.items():+ if score_type not in scores:+ raise ValueError(f"Score type {score_type} not found.")+ if scores.get(score_type, 0) < threshold:+ return False+ return Truediff --git a/camel/models/reward/nemetro_model.py b/camel/models/reward/nemetro_model.pynew file mode 100644index 000000000..762466316--- /dev/null+++ b/camel/models/reward/nemetro_model.py@@ -0,0 +1,112 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import os+from typing import Dict, List, Optional, Union++from openai import OpenAI++from camel.models.reward import BaseRewardModel+from camel.types import ChatCompletion, ModelType+from camel.utils import api_keys_required+++class NemotroRewardModel(BaseRewardModel):+ r"""Reward model based on the Nemetro model with OpenAI compatibility.++ Args:+ model_type (Union[ModelType, str]): Model for which a backend is+ created.+ api_key (Optional[str], optional): The API key for authenticating+ with the model service. (default: :obj:`None`)+ url (Optional[str], optional): The url to the model service.++ Note:+ The Nemetro model does not support model config.+ """++ def __init__(+ self,+ model_type: Union[ModelType, str],+ api_key: Optional[str] = None,+ url: Optional[str] = None,+ ) -> None:+ url = url or os.environ.get(+ "NVIDIA_API_BASE_URL", "https://integrate.api.nvidia.com/v1"+ )+ api_key = api_key or os.environ.get("NVIDIA_API_KEY")+ super().__init__(model_type, api_key, url)+ self._client = OpenAI(+ timeout=60,+ max_retries=3,+ base_url=self.url,+ api_key=self.api_key,+ )++ @api_keys_required("NVIDIA_API_KEY")+ def evaluate(self, messages: List[Dict[str, str]]) -> Dict[str, float]:+ r"""Evaluate the messages using the Nemetro model.++ Args:+ messages (List[Dict[str, str]]): A list of messages where each+ message is a dictionary format.++ Returns:+ Dict[str, float]: A dictionary mapping score types to their+ values.+ """+ response = self._client.chat.completions.create(+ messages=messages, # type: ignore[arg-type]+ model=self.model_type,+ )+ scores = self._parse_scores(response)+ return scores++ def get_scores_types(self) -> List[str]:+ r"""Get the list of score types that the reward model can return.++ Returns:+ List[str]: A list of score types that the reward model can return.+ """+ return [+ "helpfulness",+ "correctness",+ "coherence",+ "complexity",+ "verbosity",+ ]++ def _parse_scores(self, response: ChatCompletion) -> Dict[str, float]:+ r"""Parse the scores from the response.++ Args:+ response (ChatCompletion): A ChatCompletion object with the scores.++ Returns:+ Dict[str, float]: A dictionary mapping score types to their values.+ """+ try:+ choices = response.choices+ logprobs = (+ choices[0].logprobs.content+ if choices and choices[0].logprobs+ else None+ )+ scores = (+ {entry.token: entry.logprob for entry in logprobs if entry}+ if logprobs+ else {}+ )+ return scores+ except Exception as e:+ raise ValueError(f"Failed to parse scores: {e}")diff --git a/examples/reward/nemotron_model_example.py b/examples/reward/nemotron_model_example.pynew file mode 100644index 000000000..1a78ada34--- /dev/null+++ b/examples/reward/nemotron_model_example.py@@ -0,0 +1,57 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+from camel.models.reward import Evaluator, NemotroRewardModel+from camel.types import ModelType++reward_model = NemotroRewardModel(+ model_type=ModelType.NVIDIA_NEMOTRON_340B_REWARD,+ url="https://integrate.api.nvidia.com/v1",+)++evaluator = Evaluator(reward_model=reward_model)++messages = [+ {"role": "user", "content": "I am going to Paris, what should I see?"},+ {+ "role": "assistant",+ "content": "Ah, Paris, the City of Light! There are so many amazing "+ "things to see and do in this beautiful city ...",+ },+]++scores = evaluator.evaluate(messages)+print("Scores: ", scores)+'''+===============================================================================+Scores: {'helpfulness': 1.6171875, 'correctness': 1.6484375, 'coherence': 3.+3125, 'complexity': 0.546875, 'verbosity': 0.515625}+===============================================================================+'''++scores_types = reward_model.get_scores_types()+print("Types: ", scores_types)+'''+===============================================================================+Types: ['helpfulness', 'correctness', 'coherence', 'complexity', 'verbosity']+===============================================================================+'''++threshoids = {"helpfulness": 1.5, "correctness": 1.5}+is_acceptable = evaluator.filter_data(messages, threshoids)+print("Is acceptable: ", is_acceptable)+'''+===============================================================================+Is acceptable: True+===============================================================================+'''diff --git a/test/models/reward/test_nemetro_model.py b/test/models/reward/test_nemetro_model.pynew file mode 100644index 000000000..bf9d1d40b--- /dev/null+++ b/test/models/reward/test_nemetro_model.py@@ -0,0 +1,87 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from unittest.mock import MagicMock, patch++import pytest++from camel.models.reward import NemotroRewardModel+from camel.types import ModelType++[email protected]_backend+ "model_type",+ [+ ModelType.NVIDIA_NEMOTRON_340B_REWARD,+ ],+)+@patch("camel.models.reward.nemetro_model.OpenAI")+def test_nemetro_reward_model(mock_openai, model_type: ModelType):+ mock_client = MagicMock()+ mock_openai.return_value = mock_client++ model = NemotroRewardModel(model_type)+ assert model.model_type == model_type+ assert model._client == mock_client++[email protected]_backend+@patch("camel.models.reward.nemetro_model.OpenAI")+def test_nemetro_reward_model_evaluate(mock_openai):+ mock_client = MagicMock()+ mock_openai.return_value = mock_client++ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)+ messages = [{"role": "user", "content": "Hello"}]+ mock_response = MagicMock()+ mock_client.chat.completions.create.return_value = mock_response++ with patch.object(+ model, "_parse_scores", return_value={"helpfulness": 0.9}+ ):+ scores = model.evaluate(messages)+ assert scores == {"helpfulness": 0.9}+ mock_client.chat.completions.create.assert_called_once_with(+ messages=messages, model=ModelType.NVIDIA_NEMOTRON_340B_REWARD+ )++[email protected]_backend+def test_nemetro_reward_model_get_scores_types():+ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)+ score_types = model.get_scores_types()+ assert score_types == [+ "helpfulness",+ "correctness",+ "coherence",+ "complexity",+ "verbosity",+ ]++[email protected]_backend+def test_nemetro_reward_model_parse_scores():+ model = NemotroRewardModel(ModelType.NVIDIA_NEMOTRON_340B_REWARD)+ mock_response = MagicMock()+ mock_response.choices = [+ MagicMock(+ logprobs=MagicMock(+ content=[MagicMock(token="helpfulness", logprob=0.9)]+ )+ )+ ]++ scores = model._parse_scores(mock_response)+ assert scores == {"helpfulness": 0.9}