🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Python Verifier implementation (#1702)
halleritef47897cb4b0f
diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.pyindex 19d67579f..1c9dbdc88 100644--- a/camel/verifiers/__init__.py+++ b/camel/verifiers/__init__.py@@ -13,5 +13,11 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from .base import BaseVerifierfrom .models import VerificationOutcome, VerifierInput+from .python_verifier import PythonVerifier-__all__ = ["BaseVerifier", "VerificationOutcome", "VerifierInput"]+__all__ = [+ "BaseVerifier",+ "VerificationOutcome",+ "VerifierInput",+ "PythonVerifier",+]diff --git a/camel/verifiers/models.py b/camel/verifiers/models.pyindex 37bcbc478..69a0c3c6b 100644--- a/camel/verifiers/models.py+++ b/camel/verifiers/models.py@@ -19,12 +19,14 @@ from pydantic import BaseModel, Fieldclass VerifierInput(BaseModel):+ r"""Structured input to the verifier"""+llm_response: str = Field(description="The LLM response to be verified.""Needs to be in a format that the verifier can handle.")ground_truth: Optional[str] = Field(- description="The ground truth data, if available."+ None, description="The ground truth data, if available.")@@ -36,6 +38,10 @@ class VerificationOutcome(Enum):ERROR = "error"TIMEOUT = "timeout"+ def __bool__(self):+ r"""Only VerificationOutcome.SUCCESS is truthy; others are falsy."""+ return self is VerificationOutcome.SUCCESS+class VerificationResult(BaseModel):r"""Structured result from a verification."""diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pynew file mode 100644index 000000000..6e19cc1bb--- /dev/null+++ b/camel/verifiers/python_verifier.py@@ -0,0 +1,202 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio+import os+import shutil+import subprocess+import tempfile+import venv+from typing import List, Optional++from camel.logger import get_logger+from camel.verifiers import BaseVerifier++from .models import VerificationOutcome, VerificationResult, VerifierInput++logger = get_logger(__name__)+++class PythonVerifier(BaseVerifier):+ r"""The PythonVerifier class verifies Python-based implementations+ by executing them in an isolated virtual environment.++ Features:+ - Creates a virtual environment with a specified Python version.+ - Installs required packages before executing the provided script.+ - Executes the script and compares the output against a ground truth,+ if supplied.+ - Automatically cleans up the virtual environment after execution.++ The verification process ensures that the code runs in a controlled+ environment, minimizing external dependencies and conflicts.+ """++ def __init__(+ self,+ timeout: Optional[float] = 30.0,+ required_packages: Optional[List[str]] = None,+ ):+ r"""Initializes the PythonVerifier.++ Args:+ timeout (Optional[float], optional): The execution timeout in+ seconds. (default: :obj:`30.0`)+ required_packages (Optional[List[str]], optional): A list of+ packages to install in the virtual environment.+ (default: :obj:`None`)+ """+ # TODO: Use CAMEL's Interpreter to execute the code+ super().__init__(timeout=timeout)+ self.venv_path: Optional[str] = None+ self.required_packages = required_packages or []++ if os.name == 'nt': # Windows+ self.bin_dir = 'Scripts'+ else: # Unix-like systems+ self.bin_dir = 'bin'++ async def _setup(self) -> None:+ r"""Set up a virtual environment for execution+ and install required packages.+ """+ self.venv_path = tempfile.mkdtemp()+ venv.create(self.venv_path, with_pip=True)+ logger.info(f"Virtual environment created at {self.venv_path}")++ venv_pip = os.path.join(self.venv_path, self.bin_dir, "pip")++ if self.required_packages:+ try:+ subprocess.run(+ [venv_pip, "install", *self.required_packages],+ check=True,+ capture_output=True,+ )+ logger.info(+ "Installed required packages:"+ f"{', '.join(self.required_packages)}"+ )+ except subprocess.CalledProcessError as e:+ logger.error(+ "Failed to install required packages: "+ f"{e.stderr.decode().strip()}"+ )++ async def _cleanup(self) -> None:+ r"""Clean up the virtual environment."""+ if self.venv_path:+ shutil.rmtree(self.venv_path)+ logger.info(f"Virtual environment at {self.venv_path} removed")+ self.venv_path = None++ async def _verify_implementation(+ self, result: VerifierInput+ ) -> VerificationResult:+ r"""Executes the LLM-generated response in a Python virtual+ environment.++ Args:+ result (VerifierInput): Contains the LLM-generated Python code to+ execute and optional ground truth for comparison.++ Returns:+ VerificationResult: Contains verification status (SUCCESS/FAILURE/+ ERROR), execution output, error messages if any, and execution+ duration.++ Raises:+ asyncio.TimeoutError: If execution exceeds the configured timeout.+ Exception: Any unexpected errors during execution are caught and+ converted to an ERROR verification result.+ """+ if not self.venv_path:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message="Virtual environment is not set up.",+ )++ script = result.llm_response.strip()+ venv_python = os.path.join(self.venv_path, self.bin_dir, "python")++ if not os.path.exists(venv_python):+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message="Python binary not found in virtual environment",+ )++ try:+ process = await asyncio.create_subprocess_exec(+ venv_python,+ "-c",+ script,+ stdout=asyncio.subprocess.PIPE,+ stderr=asyncio.subprocess.PIPE,+ )++ stdout, stderr = await asyncio.wait_for(+ process.communicate(), timeout=self._timeout+ )++ output_result = stdout.decode().strip()+ error_output = stderr.decode().strip()++ if process.returncode == 0:+ # If ground truth is provided, compare it with the result+ if result.ground_truth is not None:+ # Normalize both strings by removing extra whitespace+ normalized_output = ' '.join(output_result.strip().split())+ normalized_truth = ' '.join(+ str(result.ground_truth).strip().split()+ )++ if normalized_output == normalized_truth:+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result=output_result,+ )+ else:+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ error_message="Output doesn't match ground truth",+ result=output_result,+ )+ else:+ return VerificationResult(+ status=VerificationOutcome.SUCCESS,+ result=output_result,+ )++ else:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ error_message=error_output,+ result=output_result,+ )++ except asyncio.TimeoutError:+ return VerificationResult(+ status=VerificationOutcome.TIMEOUT,+ result="",+ error_message="Execution timed out.",+ )++ except Exception as e:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=f"Execution error: {e}",+ )diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.pynew file mode 100644index 000000000..f5caf5c74--- /dev/null+++ b/examples/verifier/python_verifier_example.py@@ -0,0 +1,43 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import asyncio++from camel.verifiers import PythonVerifier, VerifierInput++verifier = PythonVerifier(required_packages=["numpy"])+asyncio.run(verifier.setup())++numpy_test_code = """+import numpy as np+a = np.array([1, 2, 3])+b = np.array([4, 5, 6])+result = np.dot(a, b)+print(result)+"""+++# Since the output of the above numpy code evaluates to 32,+# we expect the verification outcome to be a success.+response = VerifierInput(llm_response=numpy_test_code, ground_truth="32")+result = asyncio.run(verifier.verify(response))+print(f"Result: {result.status}")++response = VerifierInput(llm_response=numpy_test_code, ground_truth="40")+result = asyncio.run(verifier.verify(response))++# Now we expect the VerificationOutcome to be a failure,+# because the answer is wrong.+print(f"Result: {result.status}")++asyncio.run(verifier.cleanup())diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pynew file mode 100644index 000000000..0cdf9385e--- /dev/null+++ b/test/verifiers/test_python_verifier.py@@ -0,0 +1,150 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+import os++import pytest++from camel.verifiers.models import (+ VerificationOutcome,+ VerifierInput,+)+from camel.verifiers.python_verifier import PythonVerifier+++def python_verifier():+ r"""Fixture to provide a PythonVerifier instance."""+ return PythonVerifier(timeout=5.0, required_packages=["numpy"])+++async def test_python_verifier_setup_and_cleanup(python_verifier):+ r"""Test setup and cleanup of PythonVerifier."""+ await python_verifier._setup()+ assert python_verifier.venv_path is not None+ assert os.path.exists(python_verifier.venv_path)++ await python_verifier._cleanup()+ assert python_verifier.venv_path is None or not os.path.exists(+ python_verifier.venv_path+ )+++async def test_python_verifier_execution_success(python_verifier):+ r"""Test successful execution of a Python script."""+ await python_verifier._setup()++ script = 'print("Hello, World!")'+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script)+ )++ assert result.status == VerificationOutcome.SUCCESS+ assert result.result == "Hello, World!"++ await python_verifier._cleanup()+++async def test_python_verifier_execution_failure(python_verifier):+ r"""Test execution failure due to syntax error."""+ await python_verifier._setup()++ script = 'print("Hello"' # Syntax error+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script)+ )++ assert result.status == VerificationOutcome.ERROR+ assert "SyntaxError" in result.error_message++ await python_verifier._cleanup()+++async def test_python_verifier_execution_timeout(python_verifier):+ r"""Test execution timeout handling."""+ python_verifier.timeout = 1 # Short timeout for testing+ await python_verifier._setup()++ script = 'import time; time.sleep(5)'+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script)+ )++ assert result.status == VerificationOutcome.TIMEOUT+ assert "Execution timed out" in result.error_message++ await python_verifier._cleanup()+++async def test_python_verifier_output_mismatch(python_verifier):+ r"""Test verification when output does not match ground truth."""+ await python_verifier._setup()++ script = 'print("Wrong output")'+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script, ground_truth="Expected output")+ )++ assert result.status == VerificationOutcome.FAILURE+ assert result.error_message == "Output doesn't match ground truth"++ await python_verifier._cleanup()+++async def test_python_verifier_correct_output_matching(python_verifier):+ r"""Test verification when output matches ground truth."""+ await python_verifier._setup()++ script = 'print("Expected output")'+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script, ground_truth="Expected output")+ )++ assert result.status == VerificationOutcome.SUCCESS+ assert result.result == "Expected output"++ await python_verifier._cleanup()+++async def test_python_verifier_no_venv_error():+ r"""Test verification error when no virtual environment is set up."""+ verifier = PythonVerifier()+ script = 'print("Hello")'+ result = await verifier._verify_implementation(+ VerifierInput(llm_response=script)+ )++ assert result.status == VerificationOutcome.ERROR+ assert "Virtual environment is not set up" in result.error_message+++async def test_python_verifier_with_numpy(python_verifier):+ r"""Test execution with numpy installed in the virtual environment."""+ await python_verifier._setup()++ script = 'import numpy as np; print(np.array([1, 2, 3]))'+ result = await python_verifier._verify_implementation(+ VerifierInput(llm_response=script)+ )++ assert result.status == VerificationOutcome.SUCCESS+ assert "[1 2 3]" in result.result+ await python_verifier._cleanup()