Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Integrating BrowseComp Benchmark (#2293)

sunchengxuanivye48c283914a8
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.py
index 13211b830..e2ebca05d 100644
--- a/camel/agents/chat_agent.py
+++ b/camel/agents/chat_agent.py
@@ -384,7 +384,8 @@ class ChatAgent(BaseAgent):
# List of tuples (platform, type)
resolved_models_list = []
for model_spec in model_list:
- platform, type_ = model_spec[0], model_spec[1] # type: ignore[index]
+ # type: ignore[index]
+ platform, type_ = model_spec[0], model_spec[1]
resolved_models_list.append(
ModelFactory.create(
model_platform=platform, model_type=type_
@@ -1580,6 +1581,56 @@ class ChatAgent(BaseAgent):
"""
self.model_backend.add_strategy(name, strategy_fn)
+ def clone(self, with_memory: bool = False) -> ChatAgent:
+ r"""Creates a new instance of :obj:`ChatAgent` with the same
+ configuration as the current instance.
+
+ Args:
+ with_memory (bool): Whether to copy the memory (conversation
+ history) to the new agent. If True, the new agent will have
+ the same conversation history. If False, the new agent will
+ have a fresh memory with only the system message.
+ (default: :obj:`False`)
+
+ Returns:
+ ChatAgent: A new instance of :obj:`ChatAgent` with the same
+ configuration.
+ """
+ # Create a new instance with the same configuration
+ # If with_memory is True, set system_message to None
+ # If with_memory is False, use the original system message
+ # To avoid duplicated system memory.
+ system_message = None if with_memory else self._original_system_message
+
+ new_agent = ChatAgent(
+ system_message=system_message,
+ model=self.model_backend.models, # Pass the existing model_backend
+ memory=None, # clone memory later
+ message_window_size=getattr(self.memory, "window_size", None),
+ token_limit=getattr(
+ self.memory.get_context_creator(), "token_limit", None
+ ),
+ output_language=self._output_language,
+ tools=list(self._internal_tools.values()),
+ external_tools=[
+ schema for schema in self._external_tool_schemas.values()
+ ],
+ response_terminators=self.response_terminators,
+ scheduling_strategy=self.model_backend.scheduling_strategy.__name__,
+ single_iteration=self.single_iteration,
+ stop_event=self.stop_event,
+ )
+
+ # Copy memory if requested
+ if with_memory:
+ # Get all records from the current memory
+ context_records = self.memory.retrieve()
+ # Write them to the new agent's memory
+ for context_record in context_records:
+ new_agent.memory.write_record(context_record.memory_record)
+
+ return new_agent
+
def __repr__(self) -> str:
r"""Returns a string representation of the :obj:`ChatAgent`.
diff --git a/camel/agents/critic_agent.py b/camel/agents/critic_agent.py
index 13b2e2437..f292ea5ef 100644
--- a/camel/agents/critic_agent.py
+++ b/camel/agents/critic_agent.py
@@ -13,7 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import random
import warnings
-from typing import Any, Dict, Optional, Sequence
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
from colorama import Fore
@@ -22,6 +22,7 @@ from camel.memories import AgentMemory
from camel.messages import BaseMessage
from camel.models import BaseModelBackend
from camel.responses import ChatAgentResponse
+from camel.types import ModelPlatformType, ModelType
from camel.utils import get_first_int, print_text_animated
# AgentOps decorator setting
@@ -41,11 +42,18 @@ class CriticAgent(ChatAgent):
r"""A class for the critic agent that assists in selecting an option.
Args:
- system_message (BaseMessage): The system message for the critic
- agent.
- model (BaseModelBackend, optional): The model backend to use for
- generating responses. (default: :obj:`OpenAIModel` with
- `GPT_4O_MINI`)
+ system_message (Union[BaseMessage, str], optional): The system message
+ for the chat agent. (default: :obj:`None`)
+ model (Union[BaseModelBackend, Tuple[str, str], str, ModelType,
+ Tuple[ModelPlatformType, ModelType], List[BaseModelBackend],
+ List[str], List[ModelType], List[Tuple[str, str]],
+ List[Tuple[ModelPlatformType, ModelType]]], optional):
+ The model backend(s) to use. Can be a single instance,
+ a specification (string, enum, tuple), or a list of instances
+ or specifications to be managed by `ModelManager`. If a list of
+ specifications (not `BaseModelBackend` instances) is provided,
+ they will be instantiated using `ModelFactory`. (default:
+ :obj:`ModelPlatformType.DEFAULT` with `ModelType.DEFAULT`)
message_window_size (int, optional): The maximum number of previous
messages to include in the context window. If `None`, no windowing
is performed. (default: :obj:`6`)
@@ -58,8 +66,21 @@ class CriticAgent(ChatAgent):
def __init__(
self,
- system_message: BaseMessage,
- model: Optional[BaseModelBackend] = None,
+ system_message: Optional[Union[BaseMessage, str]] = None,
+ model: Optional[
+ Union[
+ BaseModelBackend,
+ Tuple[str, str],
+ str,
+ ModelType,
+ Tuple[ModelPlatformType, ModelType],
+ List[BaseModelBackend],
+ List[str],
+ List[ModelType],
+ List[Tuple[str, str]],
+ List[Tuple[ModelPlatformType, ModelType]],
+ ]
+ ] = None,
memory: Optional[AgentMemory] = None,
message_window_size: int = 6,
retry_attempts: int = 2,
@@ -200,3 +221,47 @@ class CriticAgent(ChatAgent):
terminated=False,
info={},
)
+
+ def clone(self, with_memory: bool = False) -> 'CriticAgent':
+ r"""Creates a new instance of :obj:`CriticAgent` with the same
+ configuration as the current instance.
+
+ Args:
+ with_memory (bool): Whether to copy the memory (conversation
+ history) to the new agent. If True, the new agent will have
+ the same conversation history. If False, the new agent will
+ have a fresh memory with only the system message.
+ (default: :obj:`False`)
+
+ Returns:
+ CriticAgent: A new instance of :obj:`CriticAgent` with the same
+ configuration.
+ """
+ # Create a new instance with the same configuration
+ # If with_memory is True, set system_message to None
+ # If with_memory is False, use the original system message
+ # To avoid duplicated system memory.
+ system_message = None if with_memory else self._original_system_message
+
+ new_agent = CriticAgent(
+ system_message=system_message,
+ model=self.model_backend.models, # Pass the existing model_backend
+ memory=None, # clone memory later
+ message_window_size=getattr(self.memory, "window_size", 6),
+ retry_attempts=self.retry_attempts,
+ verbose=self.verbose,
+ logger_color=self.logger_color,
+ )
+
+ # Copy memory if requested
+ if with_memory:
+ # Get all records from the current memory
+ context_records = self.memory.retrieve()
+ # Write them to the new agent's memory
+ for context_record in context_records:
+ new_agent.memory.write_record(context_record.memory_record)
+
+ # Copy CriticAgent-specific attributes
+ new_agent.options_dict = self.options_dict.copy()
+
+ return new_agent
diff --git a/camel/benchmarks/__init__.py b/camel/benchmarks/__init__.py
index d4e581600..7621ca546 100644
--- a/camel/benchmarks/__init__.py
+++ b/camel/benchmarks/__init__.py
@@ -15,12 +15,14 @@
from .apibank import APIBankBenchmark
from .apibench import APIBenchBenchmark
from .base import BaseBenchmark
+from .browsecomp import BrowseCompBenchmark
from .gaia import DefaultGAIARetriever, GAIABenchmark
from .nexus import NexusBenchmark
from .ragbench import RAGBenchBenchmark
__all__ = [
"BaseBenchmark",
+ "BrowseCompBenchmark",
"GAIABenchmark",
"DefaultGAIARetriever",
"NexusBenchmark",
diff --git a/camel/benchmarks/browsecomp.py b/camel/benchmarks/browsecomp.py
new file mode 100644
index 000000000..7d451be31
--- /dev/null
+++ b/camel/benchmarks/browsecomp.py
@@ -0,0 +1,854 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import base64
+import hashlib
+import json
+import os
+import random
+import traceback
+from collections import defaultdict
+from multiprocessing.pool import ThreadPool
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from pydantic import BaseModel, Field
+
+from camel.agents.chat_agent import ChatAgent
+from camel.benchmarks.base import BaseBenchmark
+from camel.logger import get_logger
+from camel.societies.role_playing import RolePlaying
+from camel.societies.workforce.workforce import Workforce
+from camel.tasks.task import Task
+
+logger = get_logger(__name__)
+
+
+class Message(BaseModel):
+ role: str
+ content: str
+ variant: Optional[str] = None
+
+
+MessageList = List[Message]
+
+
+class QueryResponse(BaseModel):
+ r"""A structured query response for benchmark evaluation.
+
+ This class defines the expected format for model responses to benchmark
+ questions, including explanation, exact answer, and confidence score.
+ """
+
+ explanation: str = Field(
+ description="""your explanation for your final answer."""
+ )
+ exact_answer: str = Field(description="""your succinct, final answer.""")
+ confidence: str = Field(
+ description="""
+your confidence score between 0|\%| and 100|\%| for your answer.
+"""
+ )
+
+
+class GradingResponse(BaseModel):
+ r"""A structured grading response for evaluating model answers.
+
+ This class defines the expected format for grading responses, including
+ extracted answer, reasoning about correctness, binary correctness judgment,
+ and confidence score extraction.
+ """
+
+ extracted_final_answer: str = Field(
+ description="""
+The final exact answer extracted from the [response].
+Put the extracted answer as 'None' if there is no exact, final answer to
+extract from the response."""
+ )
+ reasoning: str = Field(
+ description="""
+Explain why the extracted_final_answer is correct or incorrect
+based on [correct_answer], focusing only on if there are meaningful
+differences between [correct_answer] and the extracted_final_answer.
+Do not comment on any background to the problem, do not attempt
+to solve the problem, do not argue for any answer different
+than [correct_answer], focus only on whether the answers match."""
+ )
+ correct: str = Field(
+ description="""Answer 'yes' if extracted_final_answer matches the
+[correct_answer] given above, or is within a small margin of error for
+numerical problems. Answer 'no' otherwise, i.e. if there if there is any
+inconsistency, ambiguity, non-equivalency, or if the extracted answer is
+incorrect."""
+ )
+ confidence: str = Field(
+ description="""The extracted confidence score between 0|\%|
+and 100|\%| from [response]. Put 100 if there is no confidence score available.
+"""
+ )
+
+
+class SingleEvalResult(BaseModel):
+ r"""Result of evaluating a single benchmark sample.
+
+ This class stores the evaluation results for a single benchmark example,
+ including score, HTML representation, conversation history, and metrics.
+ """
+
+ score: Optional[float] = None
+ html: str
+ convo: MessageList
+ metrics: Dict[str, float] = Field(default_factory=dict)
+
+
+class EvalResult(BaseModel):
+ r"""Result of running a complete benchmark evaluation.
+
+ This class aggregates results from multiple sample evaluations, storing
+ the overall score, detailed metrics, HTML reports, and conversation logs.
+ """
+
+ score: Optional[float] = None # top-line metric
+ metrics: Optional[Dict[str, float]] = None # other metrics
+ htmls: List[str] # strings of valid HTML
+ convos: List[MessageList] # sampled conversations
+
+
+# Define the message template first
+_message_template = """
+<div class="message {{ role }}">
+ <div class="role">
+ {{ role }}
+ {% if variant %}<span class="variant">({{ variant }})</span>{% endif %}
+ </div>
+ <div class="content">
+ <pre>{{ content }}</pre>
+ </div>
+</div>
+"""
+
+# TODO: Add necessary prompts when tuning.
+QUERY_TEMPLATE = """
+{question}
+
+Your response should be in the following format:
+Explanation: {{your explanation for your final answer}}
+Exact Answer: {{your succinct, final answer}}
+Confidence: {{your confidence score between 0% and 100% for your answer}}
+""".strip()
+
+SUMMARIZE_TEMPLATE = """
+Based on the chat history:
+{chat_history}
+
+answer the question:
+{query}
+"""
+
+FORMAT_JSON_TEMPLATE = """
+format content into json:
+{content}
+"""
+
+GRADER_TEMPLATE = """
+Judge whether the following [response] to [question] is correct or not
+based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+Your judgement must be in the format and criteria specified below:
+
+extracted_final_answer: The final exact answer extracted from the [response].
+Put the extracted answer as 'None' if there is no exact, final answer to
+extract from the response.
+
+[correct_answer]: {correct_answer}
+
+reasoning: Explain why the extracted_final_answer is correct or incorrect
+based on [correct_answer], focusing only on if there are meaningful
+differences between [correct_answer] and the extracted_final_answer.
+Do not comment on any background to the problem, do not attempt
+to solve the problem, do not argue for any answer different
+than [correct_answer], focus only on whether the answers match.
+
+correct: Answer 'yes' if extracted_final_answer matches the
+[correct_answer] given above, or is within a small margin of error for
+numerical problems. Answer 'no' otherwise, i.e. if there is any
+inconsistency, ambiguity, non-equivalency, or if the extracted answer is
+incorrect.
+
+
+confidence: The extracted confidence score between 0|\%| and 100|\%|
+from [response]. Put 100 if there is no confidence score available.
+""".strip()
+
+
+HTML_JINJA = """
+<h3>Question:</h3>
+{{ message_to_html(prompt_messages) | safe }}
+<h3>Sampled message</h3>
+{{ message_to_html(next_message) | safe }}
+<h3>Results</h3>
+<p>Correct Answer: {{ correct_answer }}</p>
+<p>Extracted Answer: {{ extracted_answer }}</p>
+<p>Score: {{ score }}</p>
+"""
+_report_template = """<!DOCTYPE html>
+<html>
+ <head>
+ <style>
+ .message {
+ padding: 8px 16px;
+ margin-bottom: 8px;
+ border-radius: 4px;
+ }
+ .message.user {
+ background-color: #B2DFDB;
+ color: #00695C;
+ }
+ .message.assistant {
+ background-color: #B39DDB;
+ color: #4527A0;
+ }
+ .message.system {
+ background-color: #EEEEEE;
+ color: #212121;
+ }
+ .role {
+ font-weight: bold;
+ margin-bottom: 4px;
+ }
+ .variant {
+ color: #795548;
+ }
+ table, th, td {
+ border: 1px solid black;
+ }
+ pre {
+ white-space: pre-wrap;
+ }
+ </style>
+ </head>
+ <body>
+ {% if metrics %}
+ <h1>Metrics</h1>
+ <table>
+ <tr>
+ <th>Metric</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td><b>Score</b></td>
+ <td>{{ score | float | round(3) }}</td>
+ </tr>
+ {% for name, value in metrics.items() %}
+ <tr>
+ <td>{{ name }}</td>
+ <td>{{ value }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% endif %}
+ <h1>Examples</h1>
+ {% for html in htmls %}
+ {{ html | safe }}
+ <hr>
+ {% endfor %}
+ </body>
+</html>
+"""
+
+
+class JinjaEnv:
+ r"""A class that encapsulates the Jinja environment setup."""
+
+ _instance: Optional['JinjaEnv'] = None
+ _env = None
+
+ def __init__(self):
+ r"""Initialize the JinjaEnv instance if not already initialized."""
+ if not getattr(self, '_initialized', False):
+ self._initialized = True
+
+ def __new__(cls):
+ r"""Implement singleton pattern to ensure only one instance exists."""
+ if cls._instance is None:
+ cls._instance = super(JinjaEnv, cls).__new__(cls)
+ cls._instance._initialized = False
+ return cls._instance
+
+ @classmethod
+ def get_instance(cls):
+ r"""Get the singleton instance of JinjaEnv.
+
+ Returns:
+ JinjaEnv: The singleton instance.
+ """
+ if cls._instance is None:
+ cls._instance = cls()
+ return cls._instance
+
+ @property
+ def env(self):
+ r"""Lazily initialize and return the Jinja environment.
+
+ Returns:
+ jinja2.Environment: The Jinja environment instance.
+ """
+ if self._env is None:
+ # Lazy import of jinja2
+ import jinja2
+
+ # Create the Jinja environment
+ self._env = jinja2.Environment(
+ loader=jinja2.BaseLoader(),
+ undefined=jinja2.StrictUndefined,
+ autoescape=jinja2.select_autoescape(["html", "xml"]),
+ )
+
+ # Register the message_to_html function
+ self._env.globals["message_to_html"] = self.message_to_html
+
+ return self._env
+
+ def from_string(self, template_str):
+ r"""Create a template from the given string.
+
+ Args:
+ template_str (str): The template string.
+
+ Returns:
+ jinja2.Template: The compiled template.
+ """
+ return self.env.from_string(template_str)
+
+ @staticmethod
+ def message_to_html(message: Message) -> str:
+ r"""Generate HTML snippet (inside a <div>) for a message.
+
+ Args:
+ message (Message): The message to convert to HTML.
+
+ Returns:
+ str: The HTML representation of the message.
+ """
+ return (
+ JinjaEnv.get_instance()
+ .from_string(_message_template)
+ .render(
+ role=message.role,
+ content=message.content,
+ variant=message.variant,
+ )
+ )
+
+
+def derive_key(password: str, length: int) -> bytes:
+ r"""Derive a fixed-length key from the password using SHA256."""
+ hasher = hashlib.sha256()
+ hasher.update(password.encode())
+ key = hasher.digest()
+ return key * (length // len(key)) + key[: length % len(key)]
+
+
+def decrypt(ciphertext_b64: str, password: str) -> str:
+ r"""Decrypt base64-encoded ciphertext with XOR."""
+ encrypted = base64.b64decode(ciphertext_b64)
+ key = derive_key(password, len(encrypted))
+ decrypted = bytes(a ^ b for a, b in zip(encrypted, key))
+ return decrypted.decode()
+
+
+def _compute_stat(values: list, stat: str):
+ import numpy as np
+
+ if stat == "mean":
+ return np.mean(values)
+ elif stat == "std":
+ return np.std(values)
+ elif stat == "min":
+ return np.min(values)
+ elif stat == "max":
+ return np.max(values)
+ else:
+ raise ValueError(f"Unknown {stat =}")
+
+
+def aggregate_results(
+ single_eval_results: List[SingleEvalResult],
+ default_stats: Tuple[str, str] = ("mean", "std"),
+ name2stats: Optional[Dict[str, Tuple[str]]] = None,
+) -> EvalResult:
+ r"""Aggregate results from multiple evaluations into a single EvalResult.
+
+ Args:
+ single_eval_results (List[SingleEvalResult]): A list of
+ `SingleEvalResult` objects.
+ default_stats (Tuple[str, str]): A tuple of default statistics to
+ compute. (default: :obj:`("mean", "std")`)
+ name2stats (Optional[Dict[str, Tuple[str]]]): A dictionary mapping
+ metric names to statistics to compute. (default: :obj:`None`)
+
+ Returns:
+ EvalResult: An `EvalResult` object containing aggregated results.
+ """
+ name2stats = name2stats or {}
+ name2values = defaultdict(list)
+ htmls = []
+ convos = []
+
+ for single_eval_result in single_eval_results:
+ for name, value in single_eval_result.metrics.items():
+ name2values[name].append(value)
+ if single_eval_result.score is not None:
+ name2values["score"].append(single_eval_result.score)
+ htmls.append(single_eval_result.html)
+ convos.append(single_eval_result.convo)
+
+ final_metrics = {}
+ for name, values in name2values.items():
+ stats = name2stats.get(name, default_stats)
+ for stat in stats:
+ key = name if stat == "mean" else f"{name}:{stat}"
+ final_metrics[key] = _compute_stat(values, stat)
+
+ return EvalResult(
+ score=final_metrics.pop("score", None),
+ metrics=final_metrics,
+ htmls=htmls,
+ convos=convos,
+ )
+
+
+class BrowseCompBenchmark(BaseBenchmark):
+ r"""BrowseComp Benchmark for evaluating browser-based comprehension tasks.
+
+ This benchmark evaluates the ability of language models to comprehend and
+ answer questions based on browser-based content, measuring accuracy and
+ performance.
+ """
+
+ def __init__(
+ self,
+ save_to: str,
+ processes: int = 1,
+ num_examples: Optional[int] = None,
+ n_repeats: int = 1,
+ ):
+ r"""Initialize the BrowseComp benchmark.
+
+ Args:
+ save_to (str): The file to save the results.
+ processes (int, optional): The number of processes to use for
+ parallel processing. (default: :obj:`1`)
+ num_examples (Optional[int]): Number of examples to evaluate.
+ If None, all examples are used. Controls the sample size for
+ testing. (default: :obj:`None`)
+ n_repeats (int, optional): Number of times to repeat each example.
+ Useful for evaluating consistency across multiple runs.
+ (default: :obj:`1`)
+ """
+ # Browsecomp benchmark won't download any data
+ # use current path as the data_dir passing into super init
+ current_path = os.path.dirname(os.path.abspath(__file__))
+
+ super().__init__("browsecomp", current_path, save_to, processes)
+ self.num_examples = num_examples
+ self.n_repeats = n_repeats
+ self.examples: List[Dict[str, Any]] = []
+ self.load()
+ self._raw_results: List[Any] = []
+ self._validated_results: List[SingleEvalResult] = []
+ self._eval_result: EvalResult
+ self.jinja_env = JinjaEnv.get_instance()
+
+ def download(self):
+ r"""Download the BrowseComp dataset.
+
+ This method is implemented to maintain compatibility
+ with the BaseBenchmark interface, but BrowseComp doesn't
+ require downloading data separately.
+
+ Returns:
+ self: The benchmark instance
+ """
+ logger.info("BrowseComp benchmark does not require downloading data.")
+ return self
+
+ def load(self):
+ r"""Load the BrowseComp dataset.
+
+ This method loads the dataset from a remote CSV file, converts each
+ row to a dictionary, and applies sampling if num_examples is
+ specified. It also handles repeating examples if n_repeats > 1.
+
+ Returns:
+ self: The benchmark instance
+ """
+ # Load dataset from remote CSV
+ import pandas
+
+ df = pandas.read_csv(
+ "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv"
+ )
+ # Convert each row to a dictionary
+ examples = [row.to_dict() for _, row in df.iterrows()]
+
+ # Sample examples if num_examples is specified
+ if self.num_examples:
+ assert (
+ self.n_repeats == 1
+ ), "n_repeats only supported when max_examples = None"
+ rng = random.Random(0) # Use fixed seed for reproducibility
+ examples = rng.sample(examples, self.num_examples)
+
+ # Repeat examples if n_repeats > 1
+ self.examples = examples * self.n_repeats
+ return self
+
+ @property
+ def train(self):
+ r"""Get the training set.
+
+ This property is implemented to maintain compatibility with
+ the BaseBenchmark interface, but BrowseComp doesn't have a
+ training set.
+
+ Raises:
+ NotImplementedError: BrowseComp does not have a training set.
+ """
+ raise NotImplementedError("BrowseComp does not have a training set.")
+
+ def run( # type: ignore[override]
+ self,
+ pipeline_template: Union[ChatAgent, RolePlaying, Workforce],
+ chat_turn_limit: int = 10,
+ roleplaying_summarizer: Optional[ChatAgent] = None,
+ task_json_formatter: Optional[ChatAgent] = None,
+ ) -> None:
+ r"""Run the benchmark by processing each example in parallel.
+
+ This method applies the provided pipeline to each example in the
+ dataset using a process pool for parallel execution. It shows progress
+ using tqdm and stores the results in self._raw_results.
+
+ Args:
+ pipeline_template (Union[ChatAgent, RolePlaying, Workforce]): The
+ template agent or framework to use for processing examples.
+ Can be a ChatAgent, RolePlaying, or Workforce instance that
+ will be cloned for each example.
+ chat_turn_limit (int): Maximum number of conversation turns allowed
+ when using RolePlaying pipeline. (default: :obj:`10`)
+ roleplaying_summarizer (Optional[ChatAgent]): Optional ChatAgent to
+ summarize RolePlaying conversations. If None and RolePlaying is
+ used, a default summarizer will be created.
+ (default: :obj:`None`)
+ task_json_formatter (Optional[ChatAgent]): Optional ChatAgent to
+ format task JSON. If None and Workforce is used, a default
+ formatter will be created. (default: :obj:`None`)
+ """
+ from tqdm import tqdm
+
+ # Use a process pool for parallel execution
+ def process_benchmark_row(row: Dict[str, Any]) -> Dict[str, Any]:
+ r"""This inner function processes a single benchmark row by
+ extracting the problem and answer, creating a pipeline instance,
+ and generating a response using the appropriate method based on
+ the pipeline type.
+
+ Args:
+ row (Dict[str, Any]): A row from the dataset containing
+ encrypted problem and answer, along with a canary for
+ decryption.
+
+ Returns:
+ Dict[str, Any]: A dictionary containing the decrypted problem,
+ expected answer, model response, and structured response
+ fields.
+ """
+
+ problem = decrypt(row.get("problem", ""), row.get("canary", ""))
+ answer = decrypt(row.get("answer", ""), row.get("canary", ""))
+ try:
+ input_message = QUERY_TEMPLATE.format(question=problem)
+
+ if isinstance(pipeline_template, (ChatAgent)):
+ pipeline = pipeline_template.clone() # type: ignore[assignment]
+
+ response_text = pipeline.step(
+ input_message, response_format=QueryResponse
+ )
+ elif isinstance(pipeline_template, Workforce):
+ pipeline = pipeline_template.clone() # type: ignore[assignment]
+ task = Task(content=input_message, id="0")
+ task = pipeline.process_task(task) # type: ignore[attr-defined]
+ if task_json_formatter:
+ formatter_in_process = task_json_formatter.clone()
+ else:
+ formatter_in_process = ChatAgent(
+ "You are a helpful assistant."
+ )
+ response_text = formatter_in_process.step(
+ FORMAT_JSON_TEMPLATE.format(content=task.result),
+ response_format=QueryResponse,
+ )
+
+ elif isinstance(pipeline_template, RolePlaying):
+ # RolePlaying is different.
+ pipeline = pipeline_template.clone( # type: ignore[assignment]
+ task_prompt=input_message
+ )
+
+ n = 0
+ input_msg = pipeline.init_chat() # type: ignore[attr-defined]
+ chat_history = []
+ while n < chat_turn_limit:
+ n += 1
+ assistant_response, user_response = pipeline.step(
+ input_msg
+ )
+ if assistant_response.terminated: # type: ignore[attr-defined]
+ break
+ if user_response.terminated: # type: ignore[attr-defined]
+ break
+ if "CAMEL_TASK_DONE" in user_response.msg.content: # type: ignore[attr-defined]
+ break
+
+ chat_history.append(
+ f"AI User: {user_response.msg.content}" # type: ignore[attr-defined]
+ )
+ chat_history.append(
+ f"AI Assistant: {assistant_response.msg.content}" # type: ignore[attr-defined]
+ )
+ input_msg = assistant_response.msg # type: ignore[attr-defined]
+
+ chat_history_str = "\n".join(chat_history)
+ if roleplaying_summarizer:
+ summarizer_in_process = roleplaying_summarizer.clone()
+ else:
+ summarizer_in_process = ChatAgent(
+ "You are a helpful assistant."
+ )
+
+ summarize_prompt = SUMMARIZE_TEMPLATE.format(
+ chat_history=chat_history_str,
+ query=input_message,
+ )
+ response_text = summarizer_in_process.step(
+ summarize_prompt, response_format=QueryResponse
+ )
+ else:
+ raise NotImplementedError(
+ f"{type(pipeline_template)} is not supported."
+ )
+ # Parse the response JSON
+ response_dict = json.loads(response_text.msg.content)
+
+ # Format the response as a key-value string
+ formatted_response = f"""
+ Explanation: {response_dict['explanation']}
+
+ Exact Answer: {response_dict['exact_answer']}
+ Confidence: {response_dict['confidence']}"""
+
+ # Create the result dictionary
+ raw_result = {}
+ raw_result['problem'] = problem
+ raw_result['expected_answer'] = answer
+ raw_result['response'] = formatted_response
+ # Keep the original dict for reference
+ raw_result['response_dict'] = response_dict
+
+ return raw_result
+ except Exception as e:
+ # Log any errors that occur during evaluation
+ logger.error(f"Error evaluating result: {e}")
+ logger.error(traceback.format_exc())
+ return {
+ 'problem': problem,
+ 'expected_answer': answer,
+ 'response': traceback.format_exc(),
+ 'response_dict': {},
+ }
+
+ pool_class = ThreadPool
+ with pool_class(min(self.processes, len(self.examples))) as pool:
+ self._raw_results = list(
+ tqdm(
+ pool.imap(process_benchmark_row, self.examples),
+ total=len(self.examples),
+ )
+ )
+
+ def make_report(self, eval_result: EvalResult) -> str:
+ r"""Create a standalone HTML report from an EvalResult."""
+ return self.jinja_env.from_string(_report_template).render(
+ score=eval_result.score,
+ metrics=eval_result.metrics,
+ htmls=eval_result.htmls,
+ )
+
+ def validate(self, grader: Optional[ChatAgent] = None) -> None:
+ r"""Validate the raw results using the GRADER_TEMPLATE and ChatAgent.
+
+ This method evaluates the correctness of each response by
+ multi-threading. A dedicated chat agent is created in each thread.
+ The chat agent will compare raw result with the expected answer. The
+ grading results will be aggregated in a report.
+
+ Args:
+ grader: The ChatAgent used for validation. If None, a default
+ agent will be created in each thread. If provided, the
+ provided agent will be used as a template and be cloned into
+ new agents in each thread. (default: :obj:`None`)
+ """
+ from tqdm import tqdm
+
+ def validate_each_one(raw_result: Dict[str, Any]) -> SingleEvalResult:
+ r"""This inner function formats the prompt for the ChatAgent
+ grader, sends it for evaluation, extracts the correctness
+ assessment, and creates an HTML representation of the result.
+
+ Args:
+ raw_result (Dict[str, Any]): A dictionary containing 'problem',
+ 'response', and 'expected_answer' keys.
+
+ Returns:
+ SingleEvalResult: An evaluation result object with score,
+ metrics, and HTML.
+ """
+ # Format the template
+ prompt = GRADER_TEMPLATE.format(
+ question=raw_result['problem'],
+ response=raw_result['response'],
+ correct_answer=raw_result['expected_answer'],
+ )
+ if grader:
+ grader_in_process = grader.clone()
+ else:
+ grader_in_process = ChatAgent("You are a helpful assistant.")
+
+ # Create a conversation list for the result
+ convo = [
+ Message(content=raw_result['problem'], role="user"),
+ Message(content=raw_result['response'], role="assistant"),
+ ]
+
+ try:
+ response = grader_in_process.step(
+ prompt, response_format=GradingResponse
+ )
+
+ content = json.loads(response.msg.content)
+
+ grade_result = content['correct']
+
+ # Convert to binary metrics (1 for correct, 0 for incorrect)
+ is_correct = int(grade_result == "yes")
+ is_incorrect = int(grade_result == "no")
+
+ # Set the score (1 for correct, 0 for incorrect)
+ score = is_correct
+
+ # Generate HTML representation of the result
+ html = self.jinja_env.from_string(HTML_JINJA).render(
+ prompt_messages=Message(
+ content=raw_result.get('problem', ''), role="user"
+ ),
+ next_message=Message(
+ content=raw_result.get('response', ''),
+ role="assistant",
+ ),
+ score=score,
+ correct_answer=raw_result.get('expected_answer', ''),
+ extracted_answer=raw_result.get('response_dict', {}).get(
+ 'exact_answer', ''
+ ),
+ )
+ # Return the evaluation result
+ return SingleEvalResult(
+ html=html,
+ score=score,
+ convo=convo,
+ metrics={
+ "is_correct": is_correct,
+ "is_incorrect": is_incorrect,
+ },
+ )
+ except Exception as e:
+ # Log any errors that occur during evaluation
+ logger.error(f"Error evaluating result: {e}")
+ logger.error(traceback.format_exc())
+ html = self.jinja_env.from_string(HTML_JINJA).render(
+ prompt_messages=Message(
+ content=raw_result.get('problem', ''), role="user"
+ ),
+ next_message=Message(
+ content=raw_result.get('response', ''),
+ role="assistant",
+ ),
+ score=0,
+ correct_answer=raw_result.get('expected_answer', ''),
+ extracted_answer=raw_result.get('response_dict', {}).get(
+ 'exact_answer', ''
+ ),
+ )
+ return SingleEvalResult(
+ html=html,
+ score=0,
+ convo=convo,
+ metrics={
+ "is_correct": 0,
+ "is_incorrect": 1,
+ },
+ )
+
+ pool_class = ThreadPool
+ with pool_class(min(self.processes, len(self._raw_results))) as pool:
+ self._validated_results = list(
+ tqdm(
+ pool.imap(validate_each_one, self._raw_results),
+ total=len(self._raw_results),
+ )
+ )
+
+ aggregate_metrics = {
+ "is_correct": sum(
+ result.metrics["is_correct"]
+ for result in self._validated_results
+ )
+ / len(self._validated_results),
+ "is_incorrect": sum(
+ result.metrics["is_incorrect"]
+ for result in self._validated_results
+ )
+ / len(self._validated_results),
+ }
+ logger.info("AGGREGATE METRICS")
+ logger.info(aggregate_metrics)
+ logger.info("##################")
+
+ output_d = {
+ "accuracy": aggregate_metrics["is_correct"],
+ }
+
+ logger.info(f"Accuracy: {output_d['accuracy']:.3f}")
+
+ self._eval_result = aggregate_results(self._validated_results)
+ # ^^^ how to use a sampler
+ report_filename = self.save_to
+ logger.info(f"Writing report to {report_filename}")
+ with open(report_filename, "w") as fh:
+ fh.write(self.make_report(self._eval_result))
diff --git a/camel/human.py b/camel/human.py
index 1011ed57a..e0e82c20c 100644
--- a/camel/human.py
+++ b/camel/human.py
@@ -136,3 +136,17 @@ class Human:
content = self.parse_input(human_input)
message = meta_chat_message.create_new_instance(content)
return ChatAgentResponse(msgs=[message], terminated=False, info={})
+
+ def clone(self, with_memory: bool = False) -> 'Human':
+ r"""Creates a new instance of the Human class with the same
+ attributes.
+
+ Args:
+ with_memory (bool): Flag indicating whether to include memory in
+ the cloned instance. Currently not used.
+ (default: :obj:`False`)
+
+ Returns:
+ Human: A new Human instance with the same name and logger_color.
+ """
+ return Human(name=self.name, logger_color=self.logger_color)
diff --git a/camel/societies/role_playing.py b/camel/societies/role_playing.py
index 2c6c05cc4..3df870376 100644
--- a/camel/societies/role_playing.py
+++ b/camel/societies/role_playing.py
@@ -120,6 +120,9 @@ class RolePlaying:
self.task_type = task_type
self.task_prompt = task_prompt
+ self.task_specify_agent_kwargs = task_specify_agent_kwargs
+ self.task_planner_agent_kwargs = task_planner_agent_kwargs
+
self.specified_task_prompt: Optional[TextPrompt] = None
self._init_specified_task_prompt(
assistant_role_name,
@@ -680,3 +683,50 @@ class RolePlaying:
info=user_response.info,
),
)
+
+ def clone(
+ self, task_prompt: str, with_memory: bool = False
+ ) -> 'RolePlaying':
+ r"""Creates a new instance of RolePlaying with the same configuration.
+
+ Args:
+ task_prompt (str): The task prompt to be used by the new instance.
+ with_memory (bool, optional): Whether to copy the memory
+ (conversation history) to the new instance. If True, the new
+ instance will have the same conversation history. If False,
+ the new instance will have a fresh memory.
+ (default: :obj:`False`)
+
+ Returns:
+ RolePlaying: A new instance of RolePlaying with the same
+ configuration.
+ """
+
+ new_instance = RolePlaying(
+ assistant_role_name=self.assistant_agent.role_name,
+ user_role_name=self.user_agent.role_name,
+ task_prompt=task_prompt,
+ with_task_specify=self.with_task_specify,
+ task_specify_agent_kwargs=self.task_specify_agent_kwargs,
+ with_task_planner=self.with_task_planner,
+ task_planner_agent_kwargs=self.task_planner_agent_kwargs,
+ with_critic_in_the_loop=False,
+ model=self.model,
+ task_type=self.task_type,
+ )
+ tmp_assistant_sys_msg = new_instance.assistant_sys_msg
+ new_instance.assistant_agent = self.assistant_agent.clone(with_memory)
+ new_instance.assistant_sys_msg = tmp_assistant_sys_msg
+ new_instance.assistant_agent._system_message = tmp_assistant_sys_msg
+
+ tmp_user_sys_msg = new_instance.user_sys_msg
+ new_instance.user_agent = self.user_agent.clone(with_memory)
+ new_instance.user_sys_msg = tmp_user_sys_msg
+ new_instance.user_agent._system_message = tmp_user_sys_msg
+
+ new_instance.with_critic_in_the_loop = self.with_critic_in_the_loop
+ new_instance.critic_sys_msg = self.critic_sys_msg
+ if self.critic:
+ new_instance.critic = self.critic.clone(with_memory)
+
+ return new_instance
diff --git a/camel/societies/workforce/role_playing_worker.py b/camel/societies/workforce/role_playing_worker.py
index 3e6802a22..cc9b5664e 100644
--- a/camel/societies/workforce/role_playing_worker.py
+++ b/camel/societies/workforce/role_playing_worker.py
@@ -38,14 +38,17 @@ class RolePlayingWorker(Worker):
description (str): Description of the node.
assistant_role_name (str): The role name of the assistant agent.
user_role_name (str): The role name of the user agent.
- assistant_agent_kwargs (Optional[Dict], optional): The keyword
- arguments to initialize the assistant agent in the role playing,
- like the model name, etc. Defaults to None.
- user_agent_kwargs (Optional[Dict], optional): The keyword arguments to
+ assistant_agent_kwargs (Optional[Dict]): The keyword arguments to
+ initialize the assistant agent in the role playing, like the model
+ name, etc. (default: :obj:`None`)
+ user_agent_kwargs (Optional[Dict]): The keyword arguments to
initialize the user agent in the role playing, like the model name,
- etc. Defaults to None.
- chat_turn_limit (int, optional): The maximum number of chat turns in
- the role playing. Defaults to 3.
+ etc. (default: :obj:`None`)
+ summarize_agent_kwargs (Optional[Dict]): The keyword arguments to
+ initialize the summarize agent, like the model name, etc.
+ (default: :obj:`None`)
+ chat_turn_limit (int): The maximum number of chat turns in the role
+ playing. (default: :obj:`3`)
"""
def __init__(
@@ -55,9 +58,11 @@ class RolePlayingWorker(Worker):
user_role_name: str,
assistant_agent_kwargs: Optional[Dict] = None,
user_agent_kwargs: Optional[Dict] = None,
+ summarize_agent_kwargs: Optional[Dict] = None,
chat_turn_limit: int = 3,
) -> None:
super().__init__(description)
+ self.summarize_agent_kwargs = summarize_agent_kwargs
summ_sys_msg = BaseMessage.make_assistant_message(
role_name="Summarizer",
content="You are a good summarizer. You will be presented with "
@@ -65,7 +70,11 @@ class RolePlayingWorker(Worker):
"are trying to solve a task. Your job is summarizing the result "
"of the task based on the chat history.",
)
- self.summarize_agent = ChatAgent(summ_sys_msg)
+ summarize_agent_dict = (
+ summarize_agent_kwargs if summarize_agent_kwargs else {}
+ )
+ summarize_agent_dict['system_message'] = summ_sys_msg
+ self.summarize_agent = ChatAgent(**summarize_agent_dict)
self.chat_turn_limit = chat_turn_limit
self.assistant_role_name = assistant_role_name
self.user_role_name = user_role_name
diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py
index 82f94682a..373dd9cf1 100644
--- a/camel/societies/workforce/workforce.py
+++ b/camel/societies/workforce/workforce.py
@@ -202,6 +202,7 @@ class Workforce(BaseNode):
user_role_name: str,
assistant_agent_kwargs: Optional[Dict] = None,
user_agent_kwargs: Optional[Dict] = None,
+ summarize_agent_kwargs: Optional[Dict] = None,
chat_turn_limit: int = 3,
) -> Workforce:
r"""Add a worker node to the workforce that uses `RolePlaying` system.
@@ -210,25 +211,29 @@ class Workforce(BaseNode):
description (str): Description of the node.
assistant_role_name (str): The role name of the assistant agent.
user_role_name (str): The role name of the user agent.
- assistant_agent_kwargs (Optional[Dict], optional): The keyword
- arguments to initialize the assistant agent in the role
- playing, like the model name, etc. Defaults to `None`.
- user_agent_kwargs (Optional[Dict], optional): The keyword arguments
- to initialize the user agent in the role playing, like the
- model name, etc. Defaults to `None`.
- chat_turn_limit (int, optional): The maximum number of chat turns
- in the role playing. Defaults to 3.
+ assistant_agent_kwargs (Optional[Dict]): The keyword arguments to
+ initialize the assistant agent in the role playing, like the
+ model name, etc. (default: :obj:`None`)
+ user_agent_kwargs (Optional[Dict]): The keyword arguments to
+ initialize the user agent in the role playing, like the
+ model name, etc. (default: :obj:`None`)
+ summarize_agent_kwargs (Optional[Dict]): The keyword arguments to
+ initialize the summarize agent, like the model name, etc.
+ (default: :obj:`None`)
+ chat_turn_limit (int): The maximum number of chat turns in the
+ role playing. (default: :obj:`3`)
Returns:
Workforce: The workforce node itself.
"""
worker_node = RolePlayingWorker(
- description,
- assistant_role_name,
- user_role_name,
- assistant_agent_kwargs,
- user_agent_kwargs,
- chat_turn_limit,
+ description=description,
+ assistant_role_name=assistant_role_name,
+ user_role_name=user_role_name,
+ assistant_agent_kwargs=assistant_agent_kwargs,
+ user_agent_kwargs=user_agent_kwargs,
+ summarize_agent_kwargs=summarize_agent_kwargs,
+ chat_turn_limit=chat_turn_limit,
)
self._children.append(worker_node)
return self
@@ -501,3 +506,54 @@ class Workforce(BaseNode):
for child_task in self._child_listening_tasks:
child_task.cancel()
self._running = False
+
+ def clone(self, with_memory: bool = False) -> 'Workforce':
+ r"""Creates a new instance of Workforce with the same configuration.
+
+ Args:
+ with_memory (bool, optional): Whether to copy the memory
+ (conversation history) to the new instance. If True, the new
+ instance will have the same conversation history. If False,
+ the new instance will have a fresh memory.
+ (default: :obj:`False`)
+
+ Returns:
+ Workforce: A new instance of Workforce with the same configuration.
+ """
+
+ # Create a new instance with the same configuration
+ new_instance = Workforce(
+ description=self.description,
+ coordinator_agent_kwargs={},
+ task_agent_kwargs={},
+ new_worker_agent_kwargs=self.new_worker_agent_kwargs,
+ )
+
+ new_instance.task_agent = self.task_agent.clone(with_memory)
+ new_instance.coordinator_agent = self.coordinator_agent.clone(
+ with_memory
+ )
+
+ for child in self._children:
+ if isinstance(child, SingleAgentWorker):
+ cloned_worker = child.worker.clone(with_memory)
+ new_instance.add_single_agent_worker(
+ child.description, cloned_worker
+ )
+ elif isinstance(child, RolePlayingWorker):
+ new_instance.add_role_playing_worker(
+ child.description,
+ child.assistant_role_name,
+ child.user_role_name,
+ child.assistant_agent_kwargs,
+ child.user_agent_kwargs,
+ child.summarize_agent_kwargs,
+ child.chat_turn_limit,
+ )
+ elif isinstance(child, Workforce):
+ new_instance.add_workforce(child.clone(with_memory))
+ else:
+ logger.warning(f"{type(child)} is not being cloned.")
+ continue
+
+ return new_instance
diff --git a/examples/benchmarks/browsecomp_chat_agent.py b/examples/benchmarks/browsecomp_chat_agent.py
new file mode 100644
index 000000000..eeac6d36c
--- /dev/null
+++ b/examples/benchmarks/browsecomp_chat_agent.py
@@ -0,0 +1,45 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from camel.agents.chat_agent import ChatAgent
+from camel.benchmarks.browsecomp import BrowseCompBenchmark
+from camel.models.model_factory import ModelFactory
+from camel.types.enums import ModelPlatformType, ModelType
+
+if __name__ == '__main__':
+ # Define model configuration
+ model_config = {
+ "model_platform": ModelPlatformType.DEFAULT,
+ "model_type": ModelType.DEFAULT,
+ }
+
+ grader_agent = ChatAgent(
+ "You are a helpful assistant.",
+ model=ModelFactory.create(**model_config),
+ )
+
+ # Create a benchmark instance with output file "report.html"
+ # it samples 2 examples and uses 2 parallel processes for efficiency
+ # set num_example=None in case of running full benchmark.
+ benchmark = BrowseCompBenchmark("report.html", num_examples=2, processes=2)
+
+ # This will process each example in parallel using separate agents
+ # Pass the agent configuration instead of the agent itself
+ benchmark.run(pipeline_template=grader_agent)
+
+ # Validate the results using the configured model
+ # This will generate a report.html file with the evaluation results
+ # The output AGGRECATION METRICS are rates:
+ # {'is_correct': 0.0, 'is_incorrect': 1.0} means is_incorrect=100%
+ benchmark.validate(grader=grader_agent)
diff --git a/examples/benchmarks/browsecomp_roleplaying.py b/examples/benchmarks/browsecomp_roleplaying.py
new file mode 100644
index 000000000..3d3ba9c81
--- /dev/null
+++ b/examples/benchmarks/browsecomp_roleplaying.py
@@ -0,0 +1,57 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from camel.agents.chat_agent import ChatAgent
+from camel.benchmarks.browsecomp import BrowseCompBenchmark
+from camel.models.model_factory import ModelFactory
+from camel.societies.role_playing import RolePlaying
+from camel.types.enums import ModelPlatformType, ModelType, TaskType
+
+if __name__ == '__main__':
+ model_config = {
+ "model_platform": ModelPlatformType.DEFAULT,
+ "model_type": ModelType.DEFAULT,
+ }
+
+ model = ModelFactory.create(**model_config)
+ summarize_agent = ChatAgent('You are a helpful assistant.', model=model)
+
+ # Create a RolePlaying session, its task_prompt will be overwritten.
+ role_play_session = RolePlaying(
+ assistant_role_name="assistant",
+ assistant_agent_kwargs=dict(model=model),
+ user_role_name="user",
+ user_agent_kwargs=dict(model=model),
+ task_prompt="Analyze web content and extract accurate information",
+ with_task_specify=False,
+ task_type=TaskType.AI_SOCIETY,
+ )
+
+ benchmark = BrowseCompBenchmark(
+ "report_role_playing.html", num_examples=2, processes=2
+ )
+
+ benchmark.run(
+ pipeline_template=role_play_session,
+ roleplaying_summarizer=summarize_agent,
+ )
+
+ # Create a grader agent for validation
+ grader_agent = ChatAgent(
+ "You are a helpful assistant.",
+ model=ModelFactory.create(**model_config),
+ )
+
+ # Validate the results
+ benchmark.validate(grader=grader_agent)
diff --git a/examples/benchmarks/browsecomp_workforce.py b/examples/benchmarks/browsecomp_workforce.py
new file mode 100644
index 000000000..c1754b39e
--- /dev/null
+++ b/examples/benchmarks/browsecomp_workforce.py
@@ -0,0 +1,94 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from camel.agents.chat_agent import ChatAgent
+from camel.benchmarks.browsecomp import BrowseCompBenchmark
+from camel.messages.base import BaseMessage
+from camel.models.model_factory import ModelFactory
+from camel.societies.workforce.workforce import Workforce
+from camel.types.enums import ModelPlatformType, ModelType
+
+if __name__ == '__main__':
+ # Configure the model
+ model_config = {
+ "model_platform": ModelPlatformType.DEFAULT,
+ "model_type": ModelType.DEFAULT,
+ }
+
+ # Create a model instance
+ model = ModelFactory.create(**model_config)
+
+ # Create a grader agent for validation
+ grader_agent = ChatAgent(
+ "You are a helpful assistant.",
+ model=model,
+ )
+
+ formatter_agent = ChatAgent('You are a helpful assistant.', model=model)
+
+ # Create specialized agents for the workforce
+ web_researcher_sys_msg = BaseMessage.make_assistant_message(
+ role_name="Web Researcher",
+ content="""You are an expert at researching information on the web.
+You can search for and analyze web content to extract accurate information.
+You excel at understanding complex queries and finding precise answers.""",
+ )
+
+ # Create the agents with the model
+ web_researcher_agent = ChatAgent(
+ web_researcher_sys_msg,
+ model=model,
+ )
+
+ # Create a workforce for BrowseComp tasks
+ workforce = Workforce(
+ description="BrowseComp Research Team",
+ coordinator_agent_kwargs=dict(model=model),
+ task_agent_kwargs=dict(model=model),
+ new_worker_agent_kwargs=dict(model=model),
+ )
+
+ # Add workers to the workforce
+ workforce.add_single_agent_worker(
+ description="Web content researcher", worker=web_researcher_agent
+ )
+
+ # Add a role-playing worker for complex queries
+ workforce.add_role_playing_worker(
+ description="Collaborative research team",
+ assistant_role_name="Research Assistant",
+ user_role_name="Research Lead",
+ assistant_agent_kwargs=dict(model=model),
+ user_agent_kwargs=dict(model=model),
+ summarize_agent_kwargs=dict(model=model),
+ chat_turn_limit=3,
+ )
+
+ # Create the BrowseComp benchmark
+ benchmark = BrowseCompBenchmark(
+ "report_workforce.html", num_examples=3, processes=2
+ )
+
+ # Run the benchmark with the workforce
+ print("Running BrowseComp benchmark with workforce...")
+ benchmark.run(
+ pipeline_template=workforce,
+ task_json_formatter=formatter_agent,
+ )
+
+ # Validate the results
+ print("Validating results...")
+ benchmark.validate(grader=grader_agent)
+
+ print("Benchmark completed. Results saved to report_workforce.html")