🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Integrating BrowseComp Benchmark (#2293)
sunchengxuanivye48c283914a8
diff --git a/camel/agents/chat_agent.py b/camel/agents/chat_agent.pyindex 13211b830..e2ebca05d 100644--- a/camel/agents/chat_agent.py+++ b/camel/agents/chat_agent.py@@ -384,7 +384,8 @@ class ChatAgent(BaseAgent):# List of tuples (platform, type)resolved_models_list = []for model_spec in model_list:- platform, type_ = model_spec[0], model_spec[1] # type: ignore[index]+ # type: ignore[index]+ platform, type_ = model_spec[0], model_spec[1]resolved_models_list.append(ModelFactory.create(model_platform=platform, model_type=type_@@ -1580,6 +1581,56 @@ class ChatAgent(BaseAgent):"""self.model_backend.add_strategy(name, strategy_fn)+ def clone(self, with_memory: bool = False) -> ChatAgent:+ r"""Creates a new instance of :obj:`ChatAgent` with the same+ configuration as the current instance.++ Args:+ with_memory (bool): Whether to copy the memory (conversation+ history) to the new agent. If True, the new agent will have+ the same conversation history. If False, the new agent will+ have a fresh memory with only the system message.+ (default: :obj:`False`)++ Returns:+ ChatAgent: A new instance of :obj:`ChatAgent` with the same+ configuration.+ """+ # Create a new instance with the same configuration+ # If with_memory is True, set system_message to None+ # If with_memory is False, use the original system message+ # To avoid duplicated system memory.+ system_message = None if with_memory else self._original_system_message++ new_agent = ChatAgent(+ system_message=system_message,+ model=self.model_backend.models, # Pass the existing model_backend+ memory=None, # clone memory later+ message_window_size=getattr(self.memory, "window_size", None),+ token_limit=getattr(+ self.memory.get_context_creator(), "token_limit", None+ ),+ output_language=self._output_language,+ tools=list(self._internal_tools.values()),+ external_tools=[+ schema for schema in self._external_tool_schemas.values()+ ],+ response_terminators=self.response_terminators,+ scheduling_strategy=self.model_backend.scheduling_strategy.__name__,+ single_iteration=self.single_iteration,+ stop_event=self.stop_event,+ )++ # Copy memory if requested+ if with_memory:+ # Get all records from the current memory+ context_records = self.memory.retrieve()+ # Write them to the new agent's memory+ for context_record in context_records:+ new_agent.memory.write_record(context_record.memory_record)++ return new_agent+def __repr__(self) -> str:r"""Returns a string representation of the :obj:`ChatAgent`.diff --git a/camel/agents/critic_agent.py b/camel/agents/critic_agent.pyindex 13b2e2437..f292ea5ef 100644--- a/camel/agents/critic_agent.py+++ b/camel/agents/critic_agent.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import randomimport warnings-from typing import Any, Dict, Optional, Sequence+from typing import Any, Dict, List, Optional, Sequence, Tuple, Unionfrom colorama import Fore@@ -22,6 +22,7 @@ from camel.memories import AgentMemoryfrom camel.messages import BaseMessagefrom camel.models import BaseModelBackendfrom camel.responses import ChatAgentResponse+from camel.types import ModelPlatformType, ModelTypefrom camel.utils import get_first_int, print_text_animated# AgentOps decorator setting@@ -41,11 +42,18 @@ class CriticAgent(ChatAgent):r"""A class for the critic agent that assists in selecting an option.Args:- system_message (BaseMessage): The system message for the critic- agent.- model (BaseModelBackend, optional): The model backend to use for- generating responses. (default: :obj:`OpenAIModel` with- `GPT_4O_MINI`)+ system_message (Union[BaseMessage, str], optional): The system message+ for the chat agent. (default: :obj:`None`)+ model (Union[BaseModelBackend, Tuple[str, str], str, ModelType,+ Tuple[ModelPlatformType, ModelType], List[BaseModelBackend],+ List[str], List[ModelType], List[Tuple[str, str]],+ List[Tuple[ModelPlatformType, ModelType]]], optional):+ The model backend(s) to use. Can be a single instance,+ a specification (string, enum, tuple), or a list of instances+ or specifications to be managed by `ModelManager`. If a list of+ specifications (not `BaseModelBackend` instances) is provided,+ they will be instantiated using `ModelFactory`. (default:+ :obj:`ModelPlatformType.DEFAULT` with `ModelType.DEFAULT`)message_window_size (int, optional): The maximum number of previousmessages to include in the context window. If `None`, no windowingis performed. (default: :obj:`6`)@@ -58,8 +66,21 @@ class CriticAgent(ChatAgent):def __init__(self,- system_message: BaseMessage,- model: Optional[BaseModelBackend] = None,+ system_message: Optional[Union[BaseMessage, str]] = None,+ model: Optional[+ Union[+ BaseModelBackend,+ Tuple[str, str],+ str,+ ModelType,+ Tuple[ModelPlatformType, ModelType],+ List[BaseModelBackend],+ List[str],+ List[ModelType],+ List[Tuple[str, str]],+ List[Tuple[ModelPlatformType, ModelType]],+ ]+ ] = None,memory: Optional[AgentMemory] = None,message_window_size: int = 6,retry_attempts: int = 2,@@ -200,3 +221,47 @@ class CriticAgent(ChatAgent):terminated=False,info={},)++ def clone(self, with_memory: bool = False) -> 'CriticAgent':+ r"""Creates a new instance of :obj:`CriticAgent` with the same+ configuration as the current instance.++ Args:+ with_memory (bool): Whether to copy the memory (conversation+ history) to the new agent. If True, the new agent will have+ the same conversation history. If False, the new agent will+ have a fresh memory with only the system message.+ (default: :obj:`False`)++ Returns:+ CriticAgent: A new instance of :obj:`CriticAgent` with the same+ configuration.+ """+ # Create a new instance with the same configuration+ # If with_memory is True, set system_message to None+ # If with_memory is False, use the original system message+ # To avoid duplicated system memory.+ system_message = None if with_memory else self._original_system_message++ new_agent = CriticAgent(+ system_message=system_message,+ model=self.model_backend.models, # Pass the existing model_backend+ memory=None, # clone memory later+ message_window_size=getattr(self.memory, "window_size", 6),+ retry_attempts=self.retry_attempts,+ verbose=self.verbose,+ logger_color=self.logger_color,+ )++ # Copy memory if requested+ if with_memory:+ # Get all records from the current memory+ context_records = self.memory.retrieve()+ # Write them to the new agent's memory+ for context_record in context_records:+ new_agent.memory.write_record(context_record.memory_record)++ # Copy CriticAgent-specific attributes+ new_agent.options_dict = self.options_dict.copy()++ return new_agentdiff --git a/camel/benchmarks/__init__.py b/camel/benchmarks/__init__.pyindex d4e581600..7621ca546 100644--- a/camel/benchmarks/__init__.py+++ b/camel/benchmarks/__init__.py@@ -15,12 +15,14 @@from .apibank import APIBankBenchmarkfrom .apibench import APIBenchBenchmarkfrom .base import BaseBenchmark+from .browsecomp import BrowseCompBenchmarkfrom .gaia import DefaultGAIARetriever, GAIABenchmarkfrom .nexus import NexusBenchmarkfrom .ragbench import RAGBenchBenchmark__all__ = ["BaseBenchmark",+ "BrowseCompBenchmark","GAIABenchmark","DefaultGAIARetriever","NexusBenchmark",diff --git a/camel/benchmarks/browsecomp.py b/camel/benchmarks/browsecomp.pynew file mode 100644index 000000000..7d451be31--- /dev/null+++ b/camel/benchmarks/browsecomp.py@@ -0,0 +1,854 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import base64+import hashlib+import json+import os+import random+import traceback+from collections import defaultdict+from multiprocessing.pool import ThreadPool+from typing import Any, Dict, List, Optional, Tuple, Union++from pydantic import BaseModel, Field++from camel.agents.chat_agent import ChatAgent+from camel.benchmarks.base import BaseBenchmark+from camel.logger import get_logger+from camel.societies.role_playing import RolePlaying+from camel.societies.workforce.workforce import Workforce+from camel.tasks.task import Task++logger = get_logger(__name__)+++class Message(BaseModel):+ role: str+ content: str+ variant: Optional[str] = None+++MessageList = List[Message]+++class QueryResponse(BaseModel):+ r"""A structured query response for benchmark evaluation.++ This class defines the expected format for model responses to benchmark+ questions, including explanation, exact answer, and confidence score.+ """++ explanation: str = Field(+ description="""your explanation for your final answer."""+ )+ exact_answer: str = Field(description="""your succinct, final answer.""")+ confidence: str = Field(+ description="""+your confidence score between 0|\%| and 100|\%| for your answer.+"""+ )+++class GradingResponse(BaseModel):+ r"""A structured grading response for evaluating model answers.++ This class defines the expected format for grading responses, including+ extracted answer, reasoning about correctness, binary correctness judgment,+ and confidence score extraction.+ """++ extracted_final_answer: str = Field(+ description="""+The final exact answer extracted from the [response].+Put the extracted answer as 'None' if there is no exact, final answer to+extract from the response."""+ )+ reasoning: str = Field(+ description="""+Explain why the extracted_final_answer is correct or incorrect+based on [correct_answer], focusing only on if there are meaningful+differences between [correct_answer] and the extracted_final_answer.+Do not comment on any background to the problem, do not attempt+to solve the problem, do not argue for any answer different+than [correct_answer], focus only on whether the answers match."""+ )+ correct: str = Field(+ description="""Answer 'yes' if extracted_final_answer matches the+[correct_answer] given above, or is within a small margin of error for+numerical problems. Answer 'no' otherwise, i.e. if there if there is any+inconsistency, ambiguity, non-equivalency, or if the extracted answer is+incorrect."""+ )+ confidence: str = Field(+ description="""The extracted confidence score between 0|\%|+and 100|\%| from [response]. Put 100 if there is no confidence score available.+"""+ )+++class SingleEvalResult(BaseModel):+ r"""Result of evaluating a single benchmark sample.++ This class stores the evaluation results for a single benchmark example,+ including score, HTML representation, conversation history, and metrics.+ """++ score: Optional[float] = None+ html: str+ convo: MessageList+ metrics: Dict[str, float] = Field(default_factory=dict)+++class EvalResult(BaseModel):+ r"""Result of running a complete benchmark evaluation.++ This class aggregates results from multiple sample evaluations, storing+ the overall score, detailed metrics, HTML reports, and conversation logs.+ """++ score: Optional[float] = None # top-line metric+ metrics: Optional[Dict[str, float]] = None # other metrics+ htmls: List[str] # strings of valid HTML+ convos: List[MessageList] # sampled conversations+++# Define the message template first+_message_template = """+<div class="message {{ role }}">+ <div class="role">+ {{ role }}+ {% if variant %}<span class="variant">({{ variant }})</span>{% endif %}+ </div>+ <div class="content">+ <pre>{{ content }}</pre>+ </div>+</div>+"""++# TODO: Add necessary prompts when tuning.+QUERY_TEMPLATE = """+{question}++Your response should be in the following format:+Explanation: {{your explanation for your final answer}}+Exact Answer: {{your succinct, final answer}}+Confidence: {{your confidence score between 0% and 100% for your answer}}+""".strip()++SUMMARIZE_TEMPLATE = """+Based on the chat history:+{chat_history}++answer the question:+{query}+"""++FORMAT_JSON_TEMPLATE = """+format content into json:+{content}+"""++GRADER_TEMPLATE = """+Judge whether the following [response] to [question] is correct or not+based on the precise and unambiguous [correct_answer] below.++[question]: {question}++[response]: {response}++Your judgement must be in the format and criteria specified below:++extracted_final_answer: The final exact answer extracted from the [response].+Put the extracted answer as 'None' if there is no exact, final answer to+extract from the response.++[correct_answer]: {correct_answer}++reasoning: Explain why the extracted_final_answer is correct or incorrect+based on [correct_answer], focusing only on if there are meaningful+differences between [correct_answer] and the extracted_final_answer.+Do not comment on any background to the problem, do not attempt+to solve the problem, do not argue for any answer different+than [correct_answer], focus only on whether the answers match.++correct: Answer 'yes' if extracted_final_answer matches the+[correct_answer] given above, or is within a small margin of error for+numerical problems. Answer 'no' otherwise, i.e. if there is any+inconsistency, ambiguity, non-equivalency, or if the extracted answer is+incorrect.+++confidence: The extracted confidence score between 0|\%| and 100|\%|+from [response]. Put 100 if there is no confidence score available.+""".strip()+++HTML_JINJA = """+<h3>Question:</h3>+{{ message_to_html(prompt_messages) | safe }}+<h3>Sampled message</h3>+{{ message_to_html(next_message) | safe }}+<h3>Results</h3>+<p>Correct Answer: {{ correct_answer }}</p>+<p>Extracted Answer: {{ extracted_answer }}</p>+<p>Score: {{ score }}</p>+"""+_report_template = """<!DOCTYPE html>+<html>+ <head>+ <style>+ .message {+ padding: 8px 16px;+ margin-bottom: 8px;+ border-radius: 4px;+ }+ .message.user {+ background-color: #B2DFDB;+ color: #00695C;+ }+ .message.assistant {+ background-color: #B39DDB;+ color: #4527A0;+ }+ .message.system {+ background-color: #EEEEEE;+ color: #212121;+ }+ .role {+ font-weight: bold;+ margin-bottom: 4px;+ }+ .variant {+ color: #795548;+ }+ table, th, td {+ border: 1px solid black;+ }+ pre {+ white-space: pre-wrap;+ }+ </style>+ </head>+ <body>+ {% if metrics %}+ <h1>Metrics</h1>+ <table>+ <tr>+ <th>Metric</th>+ <th>Value</th>+ </tr>+ <tr>+ <td><b>Score</b></td>+ <td>{{ score | float | round(3) }}</td>+ </tr>+ {% for name, value in metrics.items() %}+ <tr>+ <td>{{ name }}</td>+ <td>{{ value }}</td>+ </tr>+ {% endfor %}+ </table>+ {% endif %}+ <h1>Examples</h1>+ {% for html in htmls %}+ {{ html | safe }}+ <hr>+ {% endfor %}+ </body>+</html>+"""+++class JinjaEnv:+ r"""A class that encapsulates the Jinja environment setup."""++ _instance: Optional['JinjaEnv'] = None+ _env = None++ def __init__(self):+ r"""Initialize the JinjaEnv instance if not already initialized."""+ if not getattr(self, '_initialized', False):+ self._initialized = True++ def __new__(cls):+ r"""Implement singleton pattern to ensure only one instance exists."""+ if cls._instance is None:+ cls._instance = super(JinjaEnv, cls).__new__(cls)+ cls._instance._initialized = False+ return cls._instance++ @classmethod+ def get_instance(cls):+ r"""Get the singleton instance of JinjaEnv.++ Returns:+ JinjaEnv: The singleton instance.+ """+ if cls._instance is None:+ cls._instance = cls()+ return cls._instance++ @property+ def env(self):+ r"""Lazily initialize and return the Jinja environment.++ Returns:+ jinja2.Environment: The Jinja environment instance.+ """+ if self._env is None:+ # Lazy import of jinja2+ import jinja2++ # Create the Jinja environment+ self._env = jinja2.Environment(+ loader=jinja2.BaseLoader(),+ undefined=jinja2.StrictUndefined,+ autoescape=jinja2.select_autoescape(["html", "xml"]),+ )++ # Register the message_to_html function+ self._env.globals["message_to_html"] = self.message_to_html++ return self._env++ def from_string(self, template_str):+ r"""Create a template from the given string.++ Args:+ template_str (str): The template string.++ Returns:+ jinja2.Template: The compiled template.+ """+ return self.env.from_string(template_str)++ @staticmethod+ def message_to_html(message: Message) -> str:+ r"""Generate HTML snippet (inside a <div>) for a message.++ Args:+ message (Message): The message to convert to HTML.++ Returns:+ str: The HTML representation of the message.+ """+ return (+ JinjaEnv.get_instance()+ .from_string(_message_template)+ .render(+ role=message.role,+ content=message.content,+ variant=message.variant,+ )+ )+++def derive_key(password: str, length: int) -> bytes:+ r"""Derive a fixed-length key from the password using SHA256."""+ hasher = hashlib.sha256()+ hasher.update(password.encode())+ key = hasher.digest()+ return key * (length // len(key)) + key[: length % len(key)]+++def decrypt(ciphertext_b64: str, password: str) -> str:+ r"""Decrypt base64-encoded ciphertext with XOR."""+ encrypted = base64.b64decode(ciphertext_b64)+ key = derive_key(password, len(encrypted))+ decrypted = bytes(a ^ b for a, b in zip(encrypted, key))+ return decrypted.decode()+++def _compute_stat(values: list, stat: str):+ import numpy as np++ if stat == "mean":+ return np.mean(values)+ elif stat == "std":+ return np.std(values)+ elif stat == "min":+ return np.min(values)+ elif stat == "max":+ return np.max(values)+ else:+ raise ValueError(f"Unknown {stat =}")+++def aggregate_results(+ single_eval_results: List[SingleEvalResult],+ default_stats: Tuple[str, str] = ("mean", "std"),+ name2stats: Optional[Dict[str, Tuple[str]]] = None,+) -> EvalResult:+ r"""Aggregate results from multiple evaluations into a single EvalResult.++ Args:+ single_eval_results (List[SingleEvalResult]): A list of+ `SingleEvalResult` objects.+ default_stats (Tuple[str, str]): A tuple of default statistics to+ compute. (default: :obj:`("mean", "std")`)+ name2stats (Optional[Dict[str, Tuple[str]]]): A dictionary mapping+ metric names to statistics to compute. (default: :obj:`None`)++ Returns:+ EvalResult: An `EvalResult` object containing aggregated results.+ """+ name2stats = name2stats or {}+ name2values = defaultdict(list)+ htmls = []+ convos = []++ for single_eval_result in single_eval_results:+ for name, value in single_eval_result.metrics.items():+ name2values[name].append(value)+ if single_eval_result.score is not None:+ name2values["score"].append(single_eval_result.score)+ htmls.append(single_eval_result.html)+ convos.append(single_eval_result.convo)++ final_metrics = {}+ for name, values in name2values.items():+ stats = name2stats.get(name, default_stats)+ for stat in stats:+ key = name if stat == "mean" else f"{name}:{stat}"+ final_metrics[key] = _compute_stat(values, stat)++ return EvalResult(+ score=final_metrics.pop("score", None),+ metrics=final_metrics,+ htmls=htmls,+ convos=convos,+ )+++class BrowseCompBenchmark(BaseBenchmark):+ r"""BrowseComp Benchmark for evaluating browser-based comprehension tasks.++ This benchmark evaluates the ability of language models to comprehend and+ answer questions based on browser-based content, measuring accuracy and+ performance.+ """++ def __init__(+ self,+ save_to: str,+ processes: int = 1,+ num_examples: Optional[int] = None,+ n_repeats: int = 1,+ ):+ r"""Initialize the BrowseComp benchmark.++ Args:+ save_to (str): The file to save the results.+ processes (int, optional): The number of processes to use for+ parallel processing. (default: :obj:`1`)+ num_examples (Optional[int]): Number of examples to evaluate.+ If None, all examples are used. Controls the sample size for+ testing. (default: :obj:`None`)+ n_repeats (int, optional): Number of times to repeat each example.+ Useful for evaluating consistency across multiple runs.+ (default: :obj:`1`)+ """+ # Browsecomp benchmark won't download any data+ # use current path as the data_dir passing into super init+ current_path = os.path.dirname(os.path.abspath(__file__))++ super().__init__("browsecomp", current_path, save_to, processes)+ self.num_examples = num_examples+ self.n_repeats = n_repeats+ self.examples: List[Dict[str, Any]] = []+ self.load()+ self._raw_results: List[Any] = []+ self._validated_results: List[SingleEvalResult] = []+ self._eval_result: EvalResult+ self.jinja_env = JinjaEnv.get_instance()++ def download(self):+ r"""Download the BrowseComp dataset.++ This method is implemented to maintain compatibility+ with the BaseBenchmark interface, but BrowseComp doesn't+ require downloading data separately.++ Returns:+ self: The benchmark instance+ """+ logger.info("BrowseComp benchmark does not require downloading data.")+ return self++ def load(self):+ r"""Load the BrowseComp dataset.++ This method loads the dataset from a remote CSV file, converts each+ row to a dictionary, and applies sampling if num_examples is+ specified. It also handles repeating examples if n_repeats > 1.++ Returns:+ self: The benchmark instance+ """+ # Load dataset from remote CSV+ import pandas++ df = pandas.read_csv(+ "https://openaipublic.blob.core.windows.net/simple-evals/browse_comp_test_set.csv"+ )+ # Convert each row to a dictionary+ examples = [row.to_dict() for _, row in df.iterrows()]++ # Sample examples if num_examples is specified+ if self.num_examples:+ assert (+ self.n_repeats == 1+ ), "n_repeats only supported when max_examples = None"+ rng = random.Random(0) # Use fixed seed for reproducibility+ examples = rng.sample(examples, self.num_examples)++ # Repeat examples if n_repeats > 1+ self.examples = examples * self.n_repeats+ return self++ @property+ def train(self):+ r"""Get the training set.++ This property is implemented to maintain compatibility with+ the BaseBenchmark interface, but BrowseComp doesn't have a+ training set.++ Raises:+ NotImplementedError: BrowseComp does not have a training set.+ """+ raise NotImplementedError("BrowseComp does not have a training set.")++ def run( # type: ignore[override]+ self,+ pipeline_template: Union[ChatAgent, RolePlaying, Workforce],+ chat_turn_limit: int = 10,+ roleplaying_summarizer: Optional[ChatAgent] = None,+ task_json_formatter: Optional[ChatAgent] = None,+ ) -> None:+ r"""Run the benchmark by processing each example in parallel.++ This method applies the provided pipeline to each example in the+ dataset using a process pool for parallel execution. It shows progress+ using tqdm and stores the results in self._raw_results.++ Args:+ pipeline_template (Union[ChatAgent, RolePlaying, Workforce]): The+ template agent or framework to use for processing examples.+ Can be a ChatAgent, RolePlaying, or Workforce instance that+ will be cloned for each example.+ chat_turn_limit (int): Maximum number of conversation turns allowed+ when using RolePlaying pipeline. (default: :obj:`10`)+ roleplaying_summarizer (Optional[ChatAgent]): Optional ChatAgent to+ summarize RolePlaying conversations. If None and RolePlaying is+ used, a default summarizer will be created.+ (default: :obj:`None`)+ task_json_formatter (Optional[ChatAgent]): Optional ChatAgent to+ format task JSON. If None and Workforce is used, a default+ formatter will be created. (default: :obj:`None`)+ """+ from tqdm import tqdm++ # Use a process pool for parallel execution+ def process_benchmark_row(row: Dict[str, Any]) -> Dict[str, Any]:+ r"""This inner function processes a single benchmark row by+ extracting the problem and answer, creating a pipeline instance,+ and generating a response using the appropriate method based on+ the pipeline type.++ Args:+ row (Dict[str, Any]): A row from the dataset containing+ encrypted problem and answer, along with a canary for+ decryption.++ Returns:+ Dict[str, Any]: A dictionary containing the decrypted problem,+ expected answer, model response, and structured response+ fields.+ """++ problem = decrypt(row.get("problem", ""), row.get("canary", ""))+ answer = decrypt(row.get("answer", ""), row.get("canary", ""))+ try:+ input_message = QUERY_TEMPLATE.format(question=problem)++ if isinstance(pipeline_template, (ChatAgent)):+ pipeline = pipeline_template.clone() # type: ignore[assignment]++ response_text = pipeline.step(+ input_message, response_format=QueryResponse+ )+ elif isinstance(pipeline_template, Workforce):+ pipeline = pipeline_template.clone() # type: ignore[assignment]+ task = Task(content=input_message, id="0")+ task = pipeline.process_task(task) # type: ignore[attr-defined]+ if task_json_formatter:+ formatter_in_process = task_json_formatter.clone()+ else:+ formatter_in_process = ChatAgent(+ "You are a helpful assistant."+ )+ response_text = formatter_in_process.step(+ FORMAT_JSON_TEMPLATE.format(content=task.result),+ response_format=QueryResponse,+ )++ elif isinstance(pipeline_template, RolePlaying):+ # RolePlaying is different.+ pipeline = pipeline_template.clone( # type: ignore[assignment]+ task_prompt=input_message+ )++ n = 0+ input_msg = pipeline.init_chat() # type: ignore[attr-defined]+ chat_history = []+ while n < chat_turn_limit:+ n += 1+ assistant_response, user_response = pipeline.step(+ input_msg+ )+ if assistant_response.terminated: # type: ignore[attr-defined]+ break+ if user_response.terminated: # type: ignore[attr-defined]+ break+ if "CAMEL_TASK_DONE" in user_response.msg.content: # type: ignore[attr-defined]+ break++ chat_history.append(+ f"AI User: {user_response.msg.content}" # type: ignore[attr-defined]+ )+ chat_history.append(+ f"AI Assistant: {assistant_response.msg.content}" # type: ignore[attr-defined]+ )+ input_msg = assistant_response.msg # type: ignore[attr-defined]++ chat_history_str = "\n".join(chat_history)+ if roleplaying_summarizer:+ summarizer_in_process = roleplaying_summarizer.clone()+ else:+ summarizer_in_process = ChatAgent(+ "You are a helpful assistant."+ )++ summarize_prompt = SUMMARIZE_TEMPLATE.format(+ chat_history=chat_history_str,+ query=input_message,+ )+ response_text = summarizer_in_process.step(+ summarize_prompt, response_format=QueryResponse+ )+ else:+ raise NotImplementedError(+ f"{type(pipeline_template)} is not supported."+ )+ # Parse the response JSON+ response_dict = json.loads(response_text.msg.content)++ # Format the response as a key-value string+ formatted_response = f"""+ Explanation: {response_dict['explanation']}++ Exact Answer: {response_dict['exact_answer']}+ Confidence: {response_dict['confidence']}"""++ # Create the result dictionary+ raw_result = {}+ raw_result['problem'] = problem+ raw_result['expected_answer'] = answer+ raw_result['response'] = formatted_response+ # Keep the original dict for reference+ raw_result['response_dict'] = response_dict++ return raw_result+ except Exception as e:+ # Log any errors that occur during evaluation+ logger.error(f"Error evaluating result: {e}")+ logger.error(traceback.format_exc())+ return {+ 'problem': problem,+ 'expected_answer': answer,+ 'response': traceback.format_exc(),+ 'response_dict': {},+ }++ pool_class = ThreadPool+ with pool_class(min(self.processes, len(self.examples))) as pool:+ self._raw_results = list(+ tqdm(+ pool.imap(process_benchmark_row, self.examples),+ total=len(self.examples),+ )+ )++ def make_report(self, eval_result: EvalResult) -> str:+ r"""Create a standalone HTML report from an EvalResult."""+ return self.jinja_env.from_string(_report_template).render(+ score=eval_result.score,+ metrics=eval_result.metrics,+ htmls=eval_result.htmls,+ )++ def validate(self, grader: Optional[ChatAgent] = None) -> None:+ r"""Validate the raw results using the GRADER_TEMPLATE and ChatAgent.++ This method evaluates the correctness of each response by+ multi-threading. A dedicated chat agent is created in each thread.+ The chat agent will compare raw result with the expected answer. The+ grading results will be aggregated in a report.++ Args:+ grader: The ChatAgent used for validation. If None, a default+ agent will be created in each thread. If provided, the+ provided agent will be used as a template and be cloned into+ new agents in each thread. (default: :obj:`None`)+ """+ from tqdm import tqdm++ def validate_each_one(raw_result: Dict[str, Any]) -> SingleEvalResult:+ r"""This inner function formats the prompt for the ChatAgent+ grader, sends it for evaluation, extracts the correctness+ assessment, and creates an HTML representation of the result.++ Args:+ raw_result (Dict[str, Any]): A dictionary containing 'problem',+ 'response', and 'expected_answer' keys.++ Returns:+ SingleEvalResult: An evaluation result object with score,+ metrics, and HTML.+ """+ # Format the template+ prompt = GRADER_TEMPLATE.format(+ question=raw_result['problem'],+ response=raw_result['response'],+ correct_answer=raw_result['expected_answer'],+ )+ if grader:+ grader_in_process = grader.clone()+ else:+ grader_in_process = ChatAgent("You are a helpful assistant.")++ # Create a conversation list for the result+ convo = [+ Message(content=raw_result['problem'], role="user"),+ Message(content=raw_result['response'], role="assistant"),+ ]++ try:+ response = grader_in_process.step(+ prompt, response_format=GradingResponse+ )++ content = json.loads(response.msg.content)++ grade_result = content['correct']++ # Convert to binary metrics (1 for correct, 0 for incorrect)+ is_correct = int(grade_result == "yes")+ is_incorrect = int(grade_result == "no")++ # Set the score (1 for correct, 0 for incorrect)+ score = is_correct++ # Generate HTML representation of the result+ html = self.jinja_env.from_string(HTML_JINJA).render(+ prompt_messages=Message(+ content=raw_result.get('problem', ''), role="user"+ ),+ next_message=Message(+ content=raw_result.get('response', ''),+ role="assistant",+ ),+ score=score,+ correct_answer=raw_result.get('expected_answer', ''),+ extracted_answer=raw_result.get('response_dict', {}).get(+ 'exact_answer', ''+ ),+ )+ # Return the evaluation result+ return SingleEvalResult(+ html=html,+ score=score,+ convo=convo,+ metrics={+ "is_correct": is_correct,+ "is_incorrect": is_incorrect,+ },+ )+ except Exception as e:+ # Log any errors that occur during evaluation+ logger.error(f"Error evaluating result: {e}")+ logger.error(traceback.format_exc())+ html = self.jinja_env.from_string(HTML_JINJA).render(+ prompt_messages=Message(+ content=raw_result.get('problem', ''), role="user"+ ),+ next_message=Message(+ content=raw_result.get('response', ''),+ role="assistant",+ ),+ score=0,+ correct_answer=raw_result.get('expected_answer', ''),+ extracted_answer=raw_result.get('response_dict', {}).get(+ 'exact_answer', ''+ ),+ )+ return SingleEvalResult(+ html=html,+ score=0,+ convo=convo,+ metrics={+ "is_correct": 0,+ "is_incorrect": 1,+ },+ )++ pool_class = ThreadPool+ with pool_class(min(self.processes, len(self._raw_results))) as pool:+ self._validated_results = list(+ tqdm(+ pool.imap(validate_each_one, self._raw_results),+ total=len(self._raw_results),+ )+ )++ aggregate_metrics = {+ "is_correct": sum(+ result.metrics["is_correct"]+ for result in self._validated_results+ )+ / len(self._validated_results),+ "is_incorrect": sum(+ result.metrics["is_incorrect"]+ for result in self._validated_results+ )+ / len(self._validated_results),+ }+ logger.info("AGGREGATE METRICS")+ logger.info(aggregate_metrics)+ logger.info("##################")++ output_d = {+ "accuracy": aggregate_metrics["is_correct"],+ }++ logger.info(f"Accuracy: {output_d['accuracy']:.3f}")++ self._eval_result = aggregate_results(self._validated_results)+ # ^^^ how to use a sampler+ report_filename = self.save_to+ logger.info(f"Writing report to {report_filename}")+ with open(report_filename, "w") as fh:+ fh.write(self.make_report(self._eval_result))diff --git a/camel/human.py b/camel/human.pyindex 1011ed57a..e0e82c20c 100644--- a/camel/human.py+++ b/camel/human.py@@ -136,3 +136,17 @@ class Human:content = self.parse_input(human_input)message = meta_chat_message.create_new_instance(content)return ChatAgentResponse(msgs=[message], terminated=False, info={})++ def clone(self, with_memory: bool = False) -> 'Human':+ r"""Creates a new instance of the Human class with the same+ attributes.++ Args:+ with_memory (bool): Flag indicating whether to include memory in+ the cloned instance. Currently not used.+ (default: :obj:`False`)++ Returns:+ Human: A new Human instance with the same name and logger_color.+ """+ return Human(name=self.name, logger_color=self.logger_color)diff --git a/camel/societies/role_playing.py b/camel/societies/role_playing.pyindex 2c6c05cc4..3df870376 100644--- a/camel/societies/role_playing.py+++ b/camel/societies/role_playing.py@@ -120,6 +120,9 @@ class RolePlaying:self.task_type = task_typeself.task_prompt = task_prompt+ self.task_specify_agent_kwargs = task_specify_agent_kwargs+ self.task_planner_agent_kwargs = task_planner_agent_kwargs+self.specified_task_prompt: Optional[TextPrompt] = Noneself._init_specified_task_prompt(assistant_role_name,@@ -680,3 +683,50 @@ class RolePlaying:info=user_response.info,),)++ def clone(+ self, task_prompt: str, with_memory: bool = False+ ) -> 'RolePlaying':+ r"""Creates a new instance of RolePlaying with the same configuration.++ Args:+ task_prompt (str): The task prompt to be used by the new instance.+ with_memory (bool, optional): Whether to copy the memory+ (conversation history) to the new instance. If True, the new+ instance will have the same conversation history. If False,+ the new instance will have a fresh memory.+ (default: :obj:`False`)++ Returns:+ RolePlaying: A new instance of RolePlaying with the same+ configuration.+ """++ new_instance = RolePlaying(+ assistant_role_name=self.assistant_agent.role_name,+ user_role_name=self.user_agent.role_name,+ task_prompt=task_prompt,+ with_task_specify=self.with_task_specify,+ task_specify_agent_kwargs=self.task_specify_agent_kwargs,+ with_task_planner=self.with_task_planner,+ task_planner_agent_kwargs=self.task_planner_agent_kwargs,+ with_critic_in_the_loop=False,+ model=self.model,+ task_type=self.task_type,+ )+ tmp_assistant_sys_msg = new_instance.assistant_sys_msg+ new_instance.assistant_agent = self.assistant_agent.clone(with_memory)+ new_instance.assistant_sys_msg = tmp_assistant_sys_msg+ new_instance.assistant_agent._system_message = tmp_assistant_sys_msg++ tmp_user_sys_msg = new_instance.user_sys_msg+ new_instance.user_agent = self.user_agent.clone(with_memory)+ new_instance.user_sys_msg = tmp_user_sys_msg+ new_instance.user_agent._system_message = tmp_user_sys_msg++ new_instance.with_critic_in_the_loop = self.with_critic_in_the_loop+ new_instance.critic_sys_msg = self.critic_sys_msg+ if self.critic:+ new_instance.critic = self.critic.clone(with_memory)++ return new_instancediff --git a/camel/societies/workforce/role_playing_worker.py b/camel/societies/workforce/role_playing_worker.pyindex 3e6802a22..cc9b5664e 100644--- a/camel/societies/workforce/role_playing_worker.py+++ b/camel/societies/workforce/role_playing_worker.py@@ -38,14 +38,17 @@ class RolePlayingWorker(Worker):description (str): Description of the node.assistant_role_name (str): The role name of the assistant agent.user_role_name (str): The role name of the user agent.- assistant_agent_kwargs (Optional[Dict], optional): The keyword- arguments to initialize the assistant agent in the role playing,- like the model name, etc. Defaults to None.- user_agent_kwargs (Optional[Dict], optional): The keyword arguments to+ assistant_agent_kwargs (Optional[Dict]): The keyword arguments to+ initialize the assistant agent in the role playing, like the model+ name, etc. (default: :obj:`None`)+ user_agent_kwargs (Optional[Dict]): The keyword arguments toinitialize the user agent in the role playing, like the model name,- etc. Defaults to None.- chat_turn_limit (int, optional): The maximum number of chat turns in- the role playing. Defaults to 3.+ etc. (default: :obj:`None`)+ summarize_agent_kwargs (Optional[Dict]): The keyword arguments to+ initialize the summarize agent, like the model name, etc.+ (default: :obj:`None`)+ chat_turn_limit (int): The maximum number of chat turns in the role+ playing. (default: :obj:`3`)"""def __init__(@@ -55,9 +58,11 @@ class RolePlayingWorker(Worker):user_role_name: str,assistant_agent_kwargs: Optional[Dict] = None,user_agent_kwargs: Optional[Dict] = None,+ summarize_agent_kwargs: Optional[Dict] = None,chat_turn_limit: int = 3,) -> None:super().__init__(description)+ self.summarize_agent_kwargs = summarize_agent_kwargssumm_sys_msg = BaseMessage.make_assistant_message(role_name="Summarizer",content="You are a good summarizer. You will be presented with "@@ -65,7 +70,11 @@ class RolePlayingWorker(Worker):"are trying to solve a task. Your job is summarizing the result ""of the task based on the chat history.",)- self.summarize_agent = ChatAgent(summ_sys_msg)+ summarize_agent_dict = (+ summarize_agent_kwargs if summarize_agent_kwargs else {}+ )+ summarize_agent_dict['system_message'] = summ_sys_msg+ self.summarize_agent = ChatAgent(**summarize_agent_dict)self.chat_turn_limit = chat_turn_limitself.assistant_role_name = assistant_role_nameself.user_role_name = user_role_namediff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.pyindex 82f94682a..373dd9cf1 100644--- a/camel/societies/workforce/workforce.py+++ b/camel/societies/workforce/workforce.py@@ -202,6 +202,7 @@ class Workforce(BaseNode):user_role_name: str,assistant_agent_kwargs: Optional[Dict] = None,user_agent_kwargs: Optional[Dict] = None,+ summarize_agent_kwargs: Optional[Dict] = None,chat_turn_limit: int = 3,) -> Workforce:r"""Add a worker node to the workforce that uses `RolePlaying` system.@@ -210,25 +211,29 @@ class Workforce(BaseNode):description (str): Description of the node.assistant_role_name (str): The role name of the assistant agent.user_role_name (str): The role name of the user agent.- assistant_agent_kwargs (Optional[Dict], optional): The keyword- arguments to initialize the assistant agent in the role- playing, like the model name, etc. Defaults to `None`.- user_agent_kwargs (Optional[Dict], optional): The keyword arguments- to initialize the user agent in the role playing, like the- model name, etc. Defaults to `None`.- chat_turn_limit (int, optional): The maximum number of chat turns- in the role playing. Defaults to 3.+ assistant_agent_kwargs (Optional[Dict]): The keyword arguments to+ initialize the assistant agent in the role playing, like the+ model name, etc. (default: :obj:`None`)+ user_agent_kwargs (Optional[Dict]): The keyword arguments to+ initialize the user agent in the role playing, like the+ model name, etc. (default: :obj:`None`)+ summarize_agent_kwargs (Optional[Dict]): The keyword arguments to+ initialize the summarize agent, like the model name, etc.+ (default: :obj:`None`)+ chat_turn_limit (int): The maximum number of chat turns in the+ role playing. (default: :obj:`3`)Returns:Workforce: The workforce node itself."""worker_node = RolePlayingWorker(- description,- assistant_role_name,- user_role_name,- assistant_agent_kwargs,- user_agent_kwargs,- chat_turn_limit,+ description=description,+ assistant_role_name=assistant_role_name,+ user_role_name=user_role_name,+ assistant_agent_kwargs=assistant_agent_kwargs,+ user_agent_kwargs=user_agent_kwargs,+ summarize_agent_kwargs=summarize_agent_kwargs,+ chat_turn_limit=chat_turn_limit,)self._children.append(worker_node)return self@@ -501,3 +506,54 @@ class Workforce(BaseNode):for child_task in self._child_listening_tasks:child_task.cancel()self._running = False++ def clone(self, with_memory: bool = False) -> 'Workforce':+ r"""Creates a new instance of Workforce with the same configuration.++ Args:+ with_memory (bool, optional): Whether to copy the memory+ (conversation history) to the new instance. If True, the new+ instance will have the same conversation history. If False,+ the new instance will have a fresh memory.+ (default: :obj:`False`)++ Returns:+ Workforce: A new instance of Workforce with the same configuration.+ """++ # Create a new instance with the same configuration+ new_instance = Workforce(+ description=self.description,+ coordinator_agent_kwargs={},+ task_agent_kwargs={},+ new_worker_agent_kwargs=self.new_worker_agent_kwargs,+ )++ new_instance.task_agent = self.task_agent.clone(with_memory)+ new_instance.coordinator_agent = self.coordinator_agent.clone(+ with_memory+ )++ for child in self._children:+ if isinstance(child, SingleAgentWorker):+ cloned_worker = child.worker.clone(with_memory)+ new_instance.add_single_agent_worker(+ child.description, cloned_worker+ )+ elif isinstance(child, RolePlayingWorker):+ new_instance.add_role_playing_worker(+ child.description,+ child.assistant_role_name,+ child.user_role_name,+ child.assistant_agent_kwargs,+ child.user_agent_kwargs,+ child.summarize_agent_kwargs,+ child.chat_turn_limit,+ )+ elif isinstance(child, Workforce):+ new_instance.add_workforce(child.clone(with_memory))+ else:+ logger.warning(f"{type(child)} is not being cloned.")+ continue++ return new_instancediff --git a/examples/benchmarks/browsecomp_chat_agent.py b/examples/benchmarks/browsecomp_chat_agent.pynew file mode 100644index 000000000..eeac6d36c--- /dev/null+++ b/examples/benchmarks/browsecomp_chat_agent.py@@ -0,0 +1,45 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from camel.agents.chat_agent import ChatAgent+from camel.benchmarks.browsecomp import BrowseCompBenchmark+from camel.models.model_factory import ModelFactory+from camel.types.enums import ModelPlatformType, ModelType++if __name__ == '__main__':+ # Define model configuration+ model_config = {+ "model_platform": ModelPlatformType.DEFAULT,+ "model_type": ModelType.DEFAULT,+ }++ grader_agent = ChatAgent(+ "You are a helpful assistant.",+ model=ModelFactory.create(**model_config),+ )++ # Create a benchmark instance with output file "report.html"+ # it samples 2 examples and uses 2 parallel processes for efficiency+ # set num_example=None in case of running full benchmark.+ benchmark = BrowseCompBenchmark("report.html", num_examples=2, processes=2)++ # This will process each example in parallel using separate agents+ # Pass the agent configuration instead of the agent itself+ benchmark.run(pipeline_template=grader_agent)++ # Validate the results using the configured model+ # This will generate a report.html file with the evaluation results+ # The output AGGRECATION METRICS are rates:+ # {'is_correct': 0.0, 'is_incorrect': 1.0} means is_incorrect=100%+ benchmark.validate(grader=grader_agent)diff --git a/examples/benchmarks/browsecomp_roleplaying.py b/examples/benchmarks/browsecomp_roleplaying.pynew file mode 100644index 000000000..3d3ba9c81--- /dev/null+++ b/examples/benchmarks/browsecomp_roleplaying.py@@ -0,0 +1,57 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from camel.agents.chat_agent import ChatAgent+from camel.benchmarks.browsecomp import BrowseCompBenchmark+from camel.models.model_factory import ModelFactory+from camel.societies.role_playing import RolePlaying+from camel.types.enums import ModelPlatformType, ModelType, TaskType++if __name__ == '__main__':+ model_config = {+ "model_platform": ModelPlatformType.DEFAULT,+ "model_type": ModelType.DEFAULT,+ }++ model = ModelFactory.create(**model_config)+ summarize_agent = ChatAgent('You are a helpful assistant.', model=model)++ # Create a RolePlaying session, its task_prompt will be overwritten.+ role_play_session = RolePlaying(+ assistant_role_name="assistant",+ assistant_agent_kwargs=dict(model=model),+ user_role_name="user",+ user_agent_kwargs=dict(model=model),+ task_prompt="Analyze web content and extract accurate information",+ with_task_specify=False,+ task_type=TaskType.AI_SOCIETY,+ )++ benchmark = BrowseCompBenchmark(+ "report_role_playing.html", num_examples=2, processes=2+ )++ benchmark.run(+ pipeline_template=role_play_session,+ roleplaying_summarizer=summarize_agent,+ )++ # Create a grader agent for validation+ grader_agent = ChatAgent(+ "You are a helpful assistant.",+ model=ModelFactory.create(**model_config),+ )++ # Validate the results+ benchmark.validate(grader=grader_agent)diff --git a/examples/benchmarks/browsecomp_workforce.py b/examples/benchmarks/browsecomp_workforce.pynew file mode 100644index 000000000..c1754b39e--- /dev/null+++ b/examples/benchmarks/browsecomp_workforce.py@@ -0,0 +1,94 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from camel.agents.chat_agent import ChatAgent+from camel.benchmarks.browsecomp import BrowseCompBenchmark+from camel.messages.base import BaseMessage+from camel.models.model_factory import ModelFactory+from camel.societies.workforce.workforce import Workforce+from camel.types.enums import ModelPlatformType, ModelType++if __name__ == '__main__':+ # Configure the model+ model_config = {+ "model_platform": ModelPlatformType.DEFAULT,+ "model_type": ModelType.DEFAULT,+ }++ # Create a model instance+ model = ModelFactory.create(**model_config)++ # Create a grader agent for validation+ grader_agent = ChatAgent(+ "You are a helpful assistant.",+ model=model,+ )++ formatter_agent = ChatAgent('You are a helpful assistant.', model=model)++ # Create specialized agents for the workforce+ web_researcher_sys_msg = BaseMessage.make_assistant_message(+ role_name="Web Researcher",+ content="""You are an expert at researching information on the web.+You can search for and analyze web content to extract accurate information.+You excel at understanding complex queries and finding precise answers.""",+ )++ # Create the agents with the model+ web_researcher_agent = ChatAgent(+ web_researcher_sys_msg,+ model=model,+ )++ # Create a workforce for BrowseComp tasks+ workforce = Workforce(+ description="BrowseComp Research Team",+ coordinator_agent_kwargs=dict(model=model),+ task_agent_kwargs=dict(model=model),+ new_worker_agent_kwargs=dict(model=model),+ )++ # Add workers to the workforce+ workforce.add_single_agent_worker(+ description="Web content researcher", worker=web_researcher_agent+ )++ # Add a role-playing worker for complex queries+ workforce.add_role_playing_worker(+ description="Collaborative research team",+ assistant_role_name="Research Assistant",+ user_role_name="Research Lead",+ assistant_agent_kwargs=dict(model=model),+ user_agent_kwargs=dict(model=model),+ summarize_agent_kwargs=dict(model=model),+ chat_turn_limit=3,+ )++ # Create the BrowseComp benchmark+ benchmark = BrowseCompBenchmark(+ "report_workforce.html", num_examples=3, processes=2+ )++ # Run the benchmark with the workforce+ print("Running BrowseComp benchmark with workforce...")+ benchmark.run(+ pipeline_template=workforce,+ task_json_formatter=formatter_agent,+ )++ # Validate the results+ print("Validating results...")+ benchmark.validate(grader=grader_agent)++ print("Benchmark completed. Results saved to report_workforce.html")