Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

refactor: Simplify usage of Verifier (#1916)

hallerite99bc47235a57
diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.py
index fbe44f558..cd17576d5 100644
--- a/camel/datasets/few_shot_generator.py
+++ b/camel/datasets/few_shot_generator.py
@@ -22,7 +22,6 @@ from camel.agents import ChatAgent
from camel.logger import get_logger
from camel.models.base_model import BaseModelBackend
from camel.verifiers import BaseVerifier
-from camel.verifiers.models import VerifierInput
from .base_generator import BaseGenerator
from .models import DataPoint
@@ -203,10 +202,8 @@ class FewShotGenerator(BaseGenerator):
try:
verifier_response = await self.verifier.verify(
- VerifierInput(
- llm_response=rationale,
- ground_truth=None,
- )
+ solution=rationale,
+ ground_truth=None,
)
if not verifier_response or not verifier_response.result:
raise ValueError(
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index 5e7390771..814bd28db 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -23,9 +23,6 @@ from camel.verifiers.base import (
BaseVerifier,
VerificationResult,
)
-from camel.verifiers.models import (
- VerifierInput,
-)
from .models import Action, Observation, StepResult
@@ -189,10 +186,7 @@ class SingleStepEnv:
# verify the extracted
verification_result = await self.verifier.verify(
- VerifierInput(
- llm_response=extraction_result,
- ground_truth=self._state.final_answer,
- )
+ solution=extraction_result, ground_truth=self._state.final_answer
)
# compute rewards
diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.py
index 1c9dbdc88..36003c94f 100644
--- a/camel/verifiers/__init__.py
+++ b/camel/verifiers/__init__.py
@@ -12,12 +12,11 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from .base import BaseVerifier
-from .models import VerificationOutcome, VerifierInput
+from .models import VerificationOutcome
from .python_verifier import PythonVerifier
__all__ = [
"BaseVerifier",
"VerificationOutcome",
- "VerifierInput",
"PythonVerifier",
]
diff --git a/camel/verifiers/base.py b/camel/verifiers/base.py
index f96c8825a..687ca6756 100644
--- a/camel/verifiers/base.py
+++ b/camel/verifiers/base.py
@@ -19,11 +19,7 @@ from typing import List, Optional
from camel.logger import get_logger
from camel.utils import BatchProcessor
-from .models import (
- VerificationOutcome,
- VerificationResult,
- VerifierInput,
-)
+from .models import VerificationOutcome, VerificationResult
logger = get_logger(__name__)
@@ -157,26 +153,33 @@ class BaseVerifier(ABC):
r"""Implement verifier-specific cleanup logic."""
pass
- async def verify(self, result: VerifierInput) -> VerificationResult:
+ async def verify(
+ self, solution: str, ground_truth: Optional[str]
+ ) -> VerificationResult:
r"""Perform verification with full error handling.
- Verifies correctness, expected output, reasoning, and symbolic
- consistency.
+ This method verifies the correctness of a generated solution by
+ comparing it against the provided ground truth. It handles
+ execution errors, timeouts, and retry attempts to ensure robust
+ validation.
Args:
- result: The response to verify.
+ solution (str): The generated response that needs verification.
+ ground_truth (Optional[str]): The expected correct answer to
+ compare against.
Returns:
- VerificationResult: Structured result containing:
- - status: SUCCESS/FAILURE/ERROR/TIMEOUT
- - result: Verification outcome description
- - duration: Time taken for verification
- - metadata: Additional details
- - error_message: Error description if applicable
+ VerificationResult: A structured object containing:
+ - status (SUCCESS/FAILURE/ERROR/TIMEOUT)
+ - result (str): The verification outcome or processed output.
+ - duration (float): Time taken for verification.
+ - metadata (dict): Additional details such as retry attempts.
+ - error_message (Optional[str]): Error description,
+ if applicable.
Raises:
RuntimeError: If verification fails unexpectedly.
- asyncio.TimeoutError: If verification times out.
+ asyncio.TimeoutError: If verification exceeds the time limit.
"""
if not self._is_setup:
logger.warning(
@@ -191,11 +194,13 @@ class BaseVerifier(ABC):
try:
verification_result = (
await asyncio.wait_for(
- self._verify_implementation(result),
+ self._verify_implementation(solution, ground_truth),
timeout=self._timeout,
)
if self._timeout
- else await self._verify_implementation(result)
+ else await self._verify_implementation(
+ solution, ground_truth
+ )
)
verification_result.duration = time.time() - start_time
@@ -240,101 +245,133 @@ class BaseVerifier(ABC):
@abstractmethod
async def _verify_implementation(
- self, result: VerifierInput
+ self, solution: str, ground_truth: Optional[str]
) -> VerificationResult:
- r"""Implement the actual verification logic.
+ r"""Abstract method for verification logic.
+
+ Subclasses must implement this method to define how the solution
+ should be processed, evaluated, and compared to the ground truth.
Args:
- result: The response to verify.
+ solution (str): The generated response requiring verification.
+ ground_truth (Optional[str]): The expected reference output.
Returns:
- VerificationResult: Containing the verification outcome.
+ VerificationResult: Contains verification status and details.
Raises:
- NotImplementedError: Must be implemented in subclasses.
+ NotImplementedError: If the method is not implemented
+ in a subclass.
"""
raise NotImplementedError(
"Subclasses must implement _verify_implementation()"
)
+ async def verify_batch(
+ self,
+ solutions: List[str],
+ ground_truths: List[Optional[str]],
+ raise_on_error: bool = False,
+ ) -> List[VerificationResult]:
+ r"""Verify multiple solutions in parallel with controlled concurrency.
-async def verify_batch(
- self, results: List[VerifierInput], raise_on_error: bool = False
-) -> List[VerificationResult]:
- r"""Verify multiple results in parallel with controlled concurrency.
+ This method verifies multiple generated solutions against their
+ respective ground truths using parallel execution. It handles
+ timeouts, execution errors, and batch processing optimizations.
- Args:
- results: List of responses to verify.
- raise_on_error: Whether to raise an exception if any verification
- fails. (default: :obj:`False`)
+ Args:
+ solutions (List[str]): A list of generated solutions to be
+ verified.
+ ground_truths (List[Optional[str]]): A list of expected outputs for
+ comparison. Each element corresponds to a solution.
+ raise_on_error (bool, optional): If True, raises an exception if
+ any verification fails. (default: :obj:`False`)
- Returns:
- List[VerificationResult]: One for each input response.
+ Returns:
+ List[VerificationResult]: A list of verification results, one per
+ input solution.
- Raises:
- RuntimeError: If any verification fails and raise_on_error is True.
- asyncio.TimeoutError: If verifications time out and max retries
- exceeded.
- """
- if not self._is_setup:
- logger.warning(
- f"{self.__class__.__name__} not set up, calling setup()"
- )
- await self.setup()
-
- # Get current batch parameters from processor with defaults if not
- # present
- max_workers = getattr(
- self._batch_processor, 'max_workers', self._max_parallel or 1
- )
- batch_size = getattr(
- self._batch_processor, 'batch_size', self._initial_batch_size or 10
- )
- semaphore = asyncio.Semaphore(max(1, max_workers))
-
- async def _verify_with_semaphore(
- response: VerifierInput,
- ) -> VerificationResult:
- start_time = time.time()
- try:
- async with semaphore:
- verification_result = await self.verify(response)
- processing_time = time.time() - start_time
- success = verification_result.status == VerificationOutcome.SUCCESS
- self._batch_processor.adjust_batch_size(success, processing_time)
- return verification_result
- except Exception as e:
- processing_time = time.time() - start_time
- self._batch_processor.adjust_batch_size(False, processing_time)
- logger.error(f"Verification failed: {e!s}", exc_info=True)
- return VerificationResult(
- status=VerificationOutcome.ERROR,
- result="",
- error_message=str(e),
- metadata={"error_type": type(e).__name__},
+ Raises:
+ RuntimeError: If any verification fails and `raise_on_error` is
+ True.
+ asyncio.TimeoutError: If verifications time out after maximum
+ retries.
+ """
+
+ if not self._is_setup:
+ logger.warning(
+ f"{self.__class__.__name__} not set up, calling setup()"
)
+ await self.setup()
- # Process in batches
- all_results: List[VerificationResult] = []
- for i in range(0, len(results), batch_size):
- batch = results[i : i + batch_size]
- verification_tasks = [
- _verify_with_semaphore(result) for result in batch
- ]
- try:
- batch_results = await asyncio.gather(*verification_tasks)
- all_results.extend(batch_results)
- except Exception as e:
- logger.error(f"Batch verification failed: {e!s}", exc_info=True)
- if raise_on_error:
- raise RuntimeError(f"Batch verification failed: {e!s}") from e
+ # Retrieve batch processing settings
+ max_workers = getattr(
+ self._batch_processor, 'max_workers', self._max_parallel or 1
+ )
+ batch_size = getattr(
+ self._batch_processor, 'batch_size', self._initial_batch_size or 10
+ )
+ semaphore = asyncio.Semaphore(max(1, max_workers))
- if raise_on_error and any(
- r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}
- for r in all_results
- ):
- error_msg = "One or more verifications failed"
- logger.error(error_msg)
- raise RuntimeError(error_msg)
+ async def _verify_with_semaphore(
+ solution: str, ground_truth: Optional[str]
+ ) -> VerificationResult:
+ start_time = time.time()
+ try:
+ async with semaphore:
+ verification_result = await self.verify(
+ solution, ground_truth
+ )
+ processing_time = time.time() - start_time
+ success = (
+ verification_result.status == VerificationOutcome.SUCCESS
+ )
+ self._batch_processor.adjust_batch_size(
+ success, processing_time
+ )
+ return verification_result
+ except Exception as e:
+ processing_time = time.time() - start_time
+ self._batch_processor.adjust_batch_size(False, processing_time)
+ logger.error(f"Verification failed: {e!s}", exc_info=True)
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=str(e),
+ metadata={"error_type": type(e).__name__},
+ )
+
+ # Process in batches
+ all_results: List[VerificationResult] = []
+ for i in range(0, len(solutions), batch_size):
+ batch_solutions = solutions[i : i + batch_size]
+ batch_ground_truths = ground_truths[i : i + batch_size]
- return all_results
+ verification_tasks = [
+ _verify_with_semaphore(solution, ground_truth)
+ for solution, ground_truth in zip(
+ batch_solutions, batch_ground_truths
+ )
+ ]
+ try:
+ batch_results = await asyncio.gather(*verification_tasks)
+ all_results.extend(batch_results)
+ except Exception as e:
+ logger.error(
+ f"Batch verification failed: {e!s}", exc_info=True
+ )
+ if raise_on_error:
+ raise RuntimeError(
+ f"Batch verification failed: {e!s}"
+ ) from e
+
+ if raise_on_error and any(
+ r.status
+ in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}
+ for r in all_results
+ ):
+ error_msg = "One or more verifications failed"
+ logger.error(error_msg)
+ raise RuntimeError(error_msg)
+
+ return all_results
diff --git a/camel/verifiers/models.py b/camel/verifiers/models.py
index 69a0c3c6b..6c44d9a37 100644
--- a/camel/verifiers/models.py
+++ b/camel/verifiers/models.py
@@ -18,18 +18,6 @@ from typing import Any, Dict, Optional
from pydantic import BaseModel, Field
-class VerifierInput(BaseModel):
- r"""Structured input to the verifier"""
-
- llm_response: str = Field(
- description="The LLM response to be verified."
- "Needs to be in a format that the verifier can handle."
- )
- ground_truth: Optional[str] = Field(
- None, description="The ground truth data, if available."
- )
-
-
class VerificationOutcome(Enum):
r"""Enum representing the status of a verification."""
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index 6e19cc1bb..627c5b51f 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -20,10 +20,11 @@ import tempfile
import venv
from typing import List, Optional
+from camel.extractors import BaseExtractor
from camel.logger import get_logger
from camel.verifiers import BaseVerifier
-from .models import VerificationOutcome, VerificationResult, VerifierInput
+from .models import VerificationOutcome, VerificationResult
logger = get_logger(__name__)
@@ -47,6 +48,7 @@ class PythonVerifier(BaseVerifier):
self,
timeout: Optional[float] = 30.0,
required_packages: Optional[List[str]] = None,
+ extractor: Optional[BaseExtractor] = None,
):
r"""Initializes the PythonVerifier.
@@ -102,24 +104,33 @@ class PythonVerifier(BaseVerifier):
self.venv_path = None
async def _verify_implementation(
- self, result: VerifierInput
+ self, solution: str, ground_truth: Optional[str]
) -> VerificationResult:
- r"""Executes the LLM-generated response in a Python virtual
- environment.
+ r"""Executes and verifies the LLM-generated Python solution in an
+ isolated virtual environment.
+
+ This method runs the given Python solution inside a controlled virtual
+ environment, captures its execution output, and optionally compares it
+ against a provided ground truth. Handles timeouts and execution errors.
Args:
- result (VerifierInput): Contains the LLM-generated Python code to
- execute and optional ground truth for comparison.
+ solution (str): The Python code to execute and verify.
+ ground_truth (Optional[str]): The expected output for comparison.
+ If None, verification is based only on execution success.
Returns:
- VerificationResult: Contains verification status (SUCCESS/FAILURE/
- ERROR), execution output, error messages if any, and execution
- duration.
+ VerificationResult: A structured object containing:
+ - status (VerificationOutcome): SUCCESS, FAILURE, ERROR,
+ or TIMEOUT.
+ - result (str): The execution output of the solution.
+ - error_message (Optional[str]): Captured error message,
+ if any.
+ - duration (float, optional): Execution time (set externally).
Raises:
asyncio.TimeoutError: If execution exceeds the configured timeout.
- Exception: Any unexpected errors during execution are caught and
- converted to an ERROR verification result.
+ Exception: Any unexpected errors are caught and converted to an
+ ERROR verification result.
"""
if not self.venv_path:
return VerificationResult(
@@ -128,7 +139,7 @@ class PythonVerifier(BaseVerifier):
error_message="Virtual environment is not set up.",
)
- script = result.llm_response.strip()
+ script = solution.strip()
venv_python = os.path.join(self.venv_path, self.bin_dir, "python")
if not os.path.exists(venv_python):
@@ -156,11 +167,11 @@ class PythonVerifier(BaseVerifier):
if process.returncode == 0:
# If ground truth is provided, compare it with the result
- if result.ground_truth is not None:
+ if ground_truth is not None:
# Normalize both strings by removing extra whitespace
normalized_output = ' '.join(output_result.strip().split())
normalized_truth = ' '.join(
- str(result.ground_truth).strip().split()
+ str(ground_truth).strip().split()
)
if normalized_output == normalized_truth:
diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.py
index f5caf5c74..d104085e9 100644
--- a/examples/verifier/python_verifier_example.py
+++ b/examples/verifier/python_verifier_example.py
@@ -13,7 +13,7 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import asyncio
-from camel.verifiers import PythonVerifier, VerifierInput
+from camel.verifiers import PythonVerifier
verifier = PythonVerifier(required_packages=["numpy"])
asyncio.run(verifier.setup())
@@ -29,12 +29,14 @@ print(result)
# Since the output of the above numpy code evaluates to 32,
# we expect the verification outcome to be a success.
-response = VerifierInput(llm_response=numpy_test_code, ground_truth="32")
-result = asyncio.run(verifier.verify(response))
+result = asyncio.run(
+ verifier.verify(solution=numpy_test_code, ground_truth="32")
+)
print(f"Result: {result.status}")
-response = VerifierInput(llm_response=numpy_test_code, ground_truth="40")
-result = asyncio.run(verifier.verify(response))
+result = asyncio.run(
+ verifier.verify(solution=numpy_test_code, ground_truth="40")
+)
# Now we expect the VerificationOutcome to be a failure,
# because the answer is wrong.
diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.py
index 1f29a00d5..faa709c11 100644
--- a/test/verifiers/test_base_verifier.py
+++ b/test/verifiers/test_base_verifier.py
@@ -13,16 +13,13 @@
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
import asyncio
+from typing import Optional
from unittest.mock import AsyncMock, patch
import pytest
from camel.verifiers.base import BaseVerifier
-from camel.verifiers.models import (
- VerificationOutcome,
- VerificationResult,
- VerifierInput,
-)
+from camel.verifiers.models import VerificationOutcome, VerificationResult
class TestVerifier(BaseVerifier):
@@ -35,19 +32,19 @@ class TestVerifier(BaseVerifier):
self.cleanup_called = True
async def _verify_implementation(
- self, result: VerifierInput
+ self, solution: str, ground_truth: Optional[str] = None
) -> VerificationResult:
r"""Simple implementation that returns success or failure based on
input.
"""
- if "fail" in result.llm_response.lower():
+ if "fail" in solution.lower():
return VerificationResult(
status=VerificationOutcome.FAILURE,
result="Verification failed",
)
- elif "error" in result.llm_response.lower():
+ elif "error" in solution.lower():
raise ValueError("Simulated error in verification")
- elif "timeout" in result.llm_response.lower():
+ elif "timeout" in solution.lower():
raise asyncio.TimeoutError("Simulated timeout")
else:
return VerificationResult(
@@ -140,10 +137,8 @@ async def test_verify_success(test_verifier):
await test_verifier.setup()
result = await test_verifier.verify(
- VerifierInput(
- llm_response="This is a successful response",
- ground_truth="Expected response",
- )
+ solution="This is a successful response",
+ ground_truth="Expected response",
)
assert result.status == VerificationOutcome.SUCCESS
@@ -160,10 +155,8 @@ async def test_verify_failure(test_verifier):
await test_verifier.setup()
result = await test_verifier.verify(
- VerifierInput(
- llm_response="This will fail the verification",
- ground_truth="Expected response",
- )
+ solution="This will fail the verification",
+ ground_truth="Expected response",
)
assert result.status == VerificationOutcome.FAILURE
@@ -183,10 +176,8 @@ async def test_verify_error_with_retry(test_verifier):
with patch("asyncio.sleep", new_callable=AsyncMock):
result = await test_verifier.verify(
- VerifierInput(
- llm_response="This will cause an error",
- ground_truth="Expected response",
- )
+ solution="This will cause an error",
+ ground_truth="Expected response",
)
assert result.status == VerificationOutcome.ERROR
@@ -208,10 +199,8 @@ async def test_verify_timeout(test_verifier):
side_effect=asyncio.TimeoutError("Simulated timeout"),
):
result = await test_verifier.verify(
- VerifierInput(
- llm_response="This will timeout",
- ground_truth="Expected response",
- )
+ solution="This will timeout",
+ ground_truth="Expected response",
)
assert result.status == VerificationOutcome.TIMEOUT
@@ -233,9 +222,7 @@ async def test_verify_not_setup():
)
)
- await verifier.verify(
- VerifierInput(llm_response="Test", ground_truth="Expected")
- )
+ await verifier.verify(solution="Test", ground_truth="Expected")
verifier.setup.assert_called_once()
@@ -252,8 +239,8 @@ async def test_verify_batch(test_verifier):
)
)
- async def mock_verify(input_data):
- if "fail" in input_data.llm_response.lower():
+ async def mock_verify(solution, ground_truth):
+ if "fail" in solution.lower():
return VerificationResult(
status=VerificationOutcome.FAILURE,
result="Verification failed",
@@ -270,15 +257,10 @@ async def test_verify_batch(test_verifier):
test_verifier, *args, **kwargs
)
- inputs = [
- VerifierInput(llm_response="Success 1", ground_truth="Expected 1"),
- VerifierInput(llm_response="Success 2", ground_truth="Expected 2"),
- VerifierInput(
- llm_response="This will fail", ground_truth="Expected 3"
- ),
- ]
+ solutions = ["Success 1", "Success 2", "This will fail"]
+ ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]
- results = await test_verifier.verify_batch(inputs)
+ results = await test_verifier.verify_batch(solutions, ground_truthes)
assert len(results) == 3
assert results[0].status == VerificationOutcome.SUCCESS
@@ -300,8 +282,8 @@ async def test_verify_batch_with_error_handling(test_verifier):
)
)
- async def mock_verify(input_data):
- if "error" in input_data.llm_response.lower():
+ async def mock_verify(solution, ground_truth):
+ if "error" in solution.lower():
raise ValueError("Simulated error in verification")
return VerificationResult(
status=VerificationOutcome.SUCCESS,
@@ -315,20 +297,17 @@ async def test_verify_batch_with_error_handling(test_verifier):
test_verifier, *args, **kwargs
)
- inputs = [
- VerifierInput(llm_response="Success", ground_truth="Expected 1"),
- VerifierInput(
- llm_response="This will cause an error",
- ground_truth="Expected 2",
- ),
- ]
+ solutions = ["Success", "This will cause an error"]
+ ground_truthes = ["Expected 1", "Expected 2"]
with pytest.raises(
RuntimeError, match="One or more verifications failed"
):
- await test_verifier.verify_batch(inputs, raise_on_error=True)
+ await test_verifier.verify_batch(
+ solutions, ground_truthes, raise_on_error=True
+ )
- results = await test_verifier.verify_batch(inputs)
+ results = await test_verifier.verify_batch(solutions, ground_truthes)
assert len(results) == 2
assert results[0].status == VerificationOutcome.SUCCESS
assert results[1].status == VerificationOutcome.ERROR
@@ -354,28 +333,26 @@ async def test_verify_batch_concurrency_limiting(test_verifier):
async def __aexit__(self, *args):
self.count -= 1
- inputs = [
- VerifierInput(
- llm_response=f"Success {i}", ground_truth=f"Expected {i}"
- )
- for i in range(3)
- ]
+ solutions = ["Success 1", "Success 2", "Success 3"]
+ ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]
mock_sem = MockSemaphore(1)
- async def patched_verify_batch(verifier, inputs, raise_on_error=False):
+ async def patched_verify_batch(
+ verifier, solutions, ground_truthes, raise_on_error=False
+ ):
results = []
- for input_data in inputs:
+ for solution, ground_truth in zip(solutions, ground_truthes):
async with mock_sem:
await asyncio.sleep(0.01) # Simulate processing time
- results.append(await verifier.verify(input_data))
+ results.append(await verifier.verify(solution, ground_truth))
return results
test_verifier.verify_batch = lambda *args, **kwargs: patched_verify_batch(
test_verifier, *args, **kwargs
)
- await test_verifier.verify_batch(inputs)
+ await test_verifier.verify_batch(solutions, ground_truthes)
assert mock_sem.max_count == 1
@@ -392,25 +369,19 @@ async def test_full_verification_flow():
assert verifier._is_setup is True
success_result = await verifier.verify(
- VerifierInput(
- llm_response="This should succeed", ground_truth="Expected"
- )
+ solution="This should succeed", ground_truth="Expected"
)
assert success_result.status == VerificationOutcome.SUCCESS
failure_result = await verifier.verify(
- VerifierInput(
- llm_response="This will fail", ground_truth="Expected"
- )
+ solution="This will fail", ground_truth="Expected"
)
assert failure_result.status == VerificationOutcome.FAILURE
with patch("asyncio.sleep", new_callable=AsyncMock):
error_result = await verifier.verify(
- VerifierInput(
- llm_response="This will cause an error",
- ground_truth="Expected",
- )
+ solution="This will cause an error",
+ ground_truth="Expected",
)
assert error_result.status == VerificationOutcome.ERROR
diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.py
index 0cdf9385e..525d10fb9 100644
--- a/test/verifiers/test_python_verifier.py
+++ b/test/verifiers/test_python_verifier.py
@@ -17,7 +17,6 @@ import pytest
from camel.verifiers.models import (
VerificationOutcome,
- VerifierInput,
)
from camel.verifiers.python_verifier import PythonVerifier
@@ -48,7 +47,7 @@ async def test_python_verifier_execution_success(python_verifier):
script = 'print("Hello, World!")'
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script)
+ solution=script, ground_truth=None
)
assert result.status == VerificationOutcome.SUCCESS
@@ -64,7 +63,7 @@ async def test_python_verifier_execution_failure(python_verifier):
script = 'print("Hello"' # Syntax error
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script)
+ solution=script, ground_truth=None
)
assert result.status == VerificationOutcome.ERROR
@@ -81,7 +80,7 @@ async def test_python_verifier_execution_timeout(python_verifier):
script = 'import time; time.sleep(5)'
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script)
+ solution=script, ground_truth=None
)
assert result.status == VerificationOutcome.TIMEOUT
@@ -97,7 +96,7 @@ async def test_python_verifier_output_mismatch(python_verifier):
script = 'print("Wrong output")'
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script, ground_truth="Expected output")
+ solution=script, ground_truth="Expected output"
)
assert result.status == VerificationOutcome.FAILURE
@@ -113,7 +112,7 @@ async def test_python_verifier_correct_output_matching(python_verifier):
script = 'print("Expected output")'
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script, ground_truth="Expected output")
+ solution=script, ground_truth="Expected output"
)
assert result.status == VerificationOutcome.SUCCESS
@@ -128,7 +127,7 @@ async def test_python_verifier_no_venv_error():
verifier = PythonVerifier()
script = 'print("Hello")'
result = await verifier._verify_implementation(
- VerifierInput(llm_response=script)
+ solution=script, ground_truth=None
)
assert result.status == VerificationOutcome.ERROR
@@ -142,7 +141,7 @@ async def test_python_verifier_with_numpy(python_verifier):
script = 'import numpy as np; print(np.array([1, 2, 3]))'
result = await python_verifier._verify_implementation(
- VerifierInput(llm_response=script)
+ solution=script, ground_truth=None
)
assert result.status == VerificationOutcome.SUCCESS