🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
refactor: Simplify usage of Verifier (#1916)
hallerite99bc47235a57
diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.pyindex fbe44f558..cd17576d5 100644--- a/camel/datasets/few_shot_generator.py+++ b/camel/datasets/few_shot_generator.py@@ -22,7 +22,6 @@ from camel.agents import ChatAgentfrom camel.logger import get_loggerfrom camel.models.base_model import BaseModelBackendfrom camel.verifiers import BaseVerifier-from camel.verifiers.models import VerifierInputfrom .base_generator import BaseGeneratorfrom .models import DataPoint@@ -203,10 +202,8 @@ class FewShotGenerator(BaseGenerator):try:verifier_response = await self.verifier.verify(- VerifierInput(- llm_response=rationale,- ground_truth=None,- )+ solution=rationale,+ ground_truth=None,)if not verifier_response or not verifier_response.result:raise ValueError(diff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex 5e7390771..814bd28db 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -23,9 +23,6 @@ from camel.verifiers.base import (BaseVerifier,VerificationResult,)-from camel.verifiers.models import (- VerifierInput,-)from .models import Action, Observation, StepResult@@ -189,10 +186,7 @@ class SingleStepEnv:# verify the extractedverification_result = await self.verifier.verify(- VerifierInput(- llm_response=extraction_result,- ground_truth=self._state.final_answer,- )+ solution=extraction_result, ground_truth=self._state.final_answer)# compute rewardsdiff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.pyindex 1c9dbdc88..36003c94f 100644--- a/camel/verifiers/__init__.py+++ b/camel/verifiers/__init__.py@@ -12,12 +12,11 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from .base import BaseVerifier-from .models import VerificationOutcome, VerifierInput+from .models import VerificationOutcomefrom .python_verifier import PythonVerifier__all__ = ["BaseVerifier","VerificationOutcome",- "VerifierInput","PythonVerifier",]diff --git a/camel/verifiers/base.py b/camel/verifiers/base.pyindex f96c8825a..687ca6756 100644--- a/camel/verifiers/base.py+++ b/camel/verifiers/base.py@@ -19,11 +19,7 @@ from typing import List, Optionalfrom camel.logger import get_loggerfrom camel.utils import BatchProcessor-from .models import (- VerificationOutcome,- VerificationResult,- VerifierInput,-)+from .models import VerificationOutcome, VerificationResultlogger = get_logger(__name__)@@ -157,26 +153,33 @@ class BaseVerifier(ABC):r"""Implement verifier-specific cleanup logic."""pass- async def verify(self, result: VerifierInput) -> VerificationResult:+ async def verify(+ self, solution: str, ground_truth: Optional[str]+ ) -> VerificationResult:r"""Perform verification with full error handling.- Verifies correctness, expected output, reasoning, and symbolic- consistency.+ This method verifies the correctness of a generated solution by+ comparing it against the provided ground truth. It handles+ execution errors, timeouts, and retry attempts to ensure robust+ validation.Args:- result: The response to verify.+ solution (str): The generated response that needs verification.+ ground_truth (Optional[str]): The expected correct answer to+ compare against.Returns:- VerificationResult: Structured result containing:- - status: SUCCESS/FAILURE/ERROR/TIMEOUT- - result: Verification outcome description- - duration: Time taken for verification- - metadata: Additional details- - error_message: Error description if applicable+ VerificationResult: A structured object containing:+ - status (SUCCESS/FAILURE/ERROR/TIMEOUT)+ - result (str): The verification outcome or processed output.+ - duration (float): Time taken for verification.+ - metadata (dict): Additional details such as retry attempts.+ - error_message (Optional[str]): Error description,+ if applicable.Raises:RuntimeError: If verification fails unexpectedly.- asyncio.TimeoutError: If verification times out.+ asyncio.TimeoutError: If verification exceeds the time limit."""if not self._is_setup:logger.warning(@@ -191,11 +194,13 @@ class BaseVerifier(ABC):try:verification_result = (await asyncio.wait_for(- self._verify_implementation(result),+ self._verify_implementation(solution, ground_truth),timeout=self._timeout,)if self._timeout- else await self._verify_implementation(result)+ else await self._verify_implementation(+ solution, ground_truth+ ))verification_result.duration = time.time() - start_time@@ -240,101 +245,133 @@ class BaseVerifier(ABC):@abstractmethodasync def _verify_implementation(- self, result: VerifierInput+ self, solution: str, ground_truth: Optional[str]) -> VerificationResult:- r"""Implement the actual verification logic.+ r"""Abstract method for verification logic.++ Subclasses must implement this method to define how the solution+ should be processed, evaluated, and compared to the ground truth.Args:- result: The response to verify.+ solution (str): The generated response requiring verification.+ ground_truth (Optional[str]): The expected reference output.Returns:- VerificationResult: Containing the verification outcome.+ VerificationResult: Contains verification status and details.Raises:- NotImplementedError: Must be implemented in subclasses.+ NotImplementedError: If the method is not implemented+ in a subclass."""raise NotImplementedError("Subclasses must implement _verify_implementation()")+ async def verify_batch(+ self,+ solutions: List[str],+ ground_truths: List[Optional[str]],+ raise_on_error: bool = False,+ ) -> List[VerificationResult]:+ r"""Verify multiple solutions in parallel with controlled concurrency.-async def verify_batch(- self, results: List[VerifierInput], raise_on_error: bool = False-) -> List[VerificationResult]:- r"""Verify multiple results in parallel with controlled concurrency.+ This method verifies multiple generated solutions against their+ respective ground truths using parallel execution. It handles+ timeouts, execution errors, and batch processing optimizations.- Args:- results: List of responses to verify.- raise_on_error: Whether to raise an exception if any verification- fails. (default: :obj:`False`)+ Args:+ solutions (List[str]): A list of generated solutions to be+ verified.+ ground_truths (List[Optional[str]]): A list of expected outputs for+ comparison. Each element corresponds to a solution.+ raise_on_error (bool, optional): If True, raises an exception if+ any verification fails. (default: :obj:`False`)- Returns:- List[VerificationResult]: One for each input response.+ Returns:+ List[VerificationResult]: A list of verification results, one per+ input solution.- Raises:- RuntimeError: If any verification fails and raise_on_error is True.- asyncio.TimeoutError: If verifications time out and max retries- exceeded.- """- if not self._is_setup:- logger.warning(- f"{self.__class__.__name__} not set up, calling setup()"- )- await self.setup()-- # Get current batch parameters from processor with defaults if not- # present- max_workers = getattr(- self._batch_processor, 'max_workers', self._max_parallel or 1- )- batch_size = getattr(- self._batch_processor, 'batch_size', self._initial_batch_size or 10- )- semaphore = asyncio.Semaphore(max(1, max_workers))-- async def _verify_with_semaphore(- response: VerifierInput,- ) -> VerificationResult:- start_time = time.time()- try:- async with semaphore:- verification_result = await self.verify(response)- processing_time = time.time() - start_time- success = verification_result.status == VerificationOutcome.SUCCESS- self._batch_processor.adjust_batch_size(success, processing_time)- return verification_result- except Exception as e:- processing_time = time.time() - start_time- self._batch_processor.adjust_batch_size(False, processing_time)- logger.error(f"Verification failed: {e!s}", exc_info=True)- return VerificationResult(- status=VerificationOutcome.ERROR,- result="",- error_message=str(e),- metadata={"error_type": type(e).__name__},+ Raises:+ RuntimeError: If any verification fails and `raise_on_error` is+ True.+ asyncio.TimeoutError: If verifications time out after maximum+ retries.+ """++ if not self._is_setup:+ logger.warning(+ f"{self.__class__.__name__} not set up, calling setup()")+ await self.setup()- # Process in batches- all_results: List[VerificationResult] = []- for i in range(0, len(results), batch_size):- batch = results[i : i + batch_size]- verification_tasks = [- _verify_with_semaphore(result) for result in batch- ]- try:- batch_results = await asyncio.gather(*verification_tasks)- all_results.extend(batch_results)- except Exception as e:- logger.error(f"Batch verification failed: {e!s}", exc_info=True)- if raise_on_error:- raise RuntimeError(f"Batch verification failed: {e!s}") from e+ # Retrieve batch processing settings+ max_workers = getattr(+ self._batch_processor, 'max_workers', self._max_parallel or 1+ )+ batch_size = getattr(+ self._batch_processor, 'batch_size', self._initial_batch_size or 10+ )+ semaphore = asyncio.Semaphore(max(1, max_workers))- if raise_on_error and any(- r.status in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}- for r in all_results- ):- error_msg = "One or more verifications failed"- logger.error(error_msg)- raise RuntimeError(error_msg)+ async def _verify_with_semaphore(+ solution: str, ground_truth: Optional[str]+ ) -> VerificationResult:+ start_time = time.time()+ try:+ async with semaphore:+ verification_result = await self.verify(+ solution, ground_truth+ )+ processing_time = time.time() - start_time+ success = (+ verification_result.status == VerificationOutcome.SUCCESS+ )+ self._batch_processor.adjust_batch_size(+ success, processing_time+ )+ return verification_result+ except Exception as e:+ processing_time = time.time() - start_time+ self._batch_processor.adjust_batch_size(False, processing_time)+ logger.error(f"Verification failed: {e!s}", exc_info=True)+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=str(e),+ metadata={"error_type": type(e).__name__},+ )++ # Process in batches+ all_results: List[VerificationResult] = []+ for i in range(0, len(solutions), batch_size):+ batch_solutions = solutions[i : i + batch_size]+ batch_ground_truths = ground_truths[i : i + batch_size]- return all_results+ verification_tasks = [+ _verify_with_semaphore(solution, ground_truth)+ for solution, ground_truth in zip(+ batch_solutions, batch_ground_truths+ )+ ]+ try:+ batch_results = await asyncio.gather(*verification_tasks)+ all_results.extend(batch_results)+ except Exception as e:+ logger.error(+ f"Batch verification failed: {e!s}", exc_info=True+ )+ if raise_on_error:+ raise RuntimeError(+ f"Batch verification failed: {e!s}"+ ) from e++ if raise_on_error and any(+ r.status+ in {VerificationOutcome.ERROR, VerificationOutcome.TIMEOUT}+ for r in all_results+ ):+ error_msg = "One or more verifications failed"+ logger.error(error_msg)+ raise RuntimeError(error_msg)++ return all_resultsdiff --git a/camel/verifiers/models.py b/camel/verifiers/models.pyindex 69a0c3c6b..6c44d9a37 100644--- a/camel/verifiers/models.py+++ b/camel/verifiers/models.py@@ -18,18 +18,6 @@ from typing import Any, Dict, Optionalfrom pydantic import BaseModel, Field-class VerifierInput(BaseModel):- r"""Structured input to the verifier"""-- llm_response: str = Field(- description="The LLM response to be verified."- "Needs to be in a format that the verifier can handle."- )- ground_truth: Optional[str] = Field(- None, description="The ground truth data, if available."- )--class VerificationOutcome(Enum):r"""Enum representing the status of a verification."""diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex 6e19cc1bb..627c5b51f 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -20,10 +20,11 @@ import tempfileimport venvfrom typing import List, Optional+from camel.extractors import BaseExtractorfrom camel.logger import get_loggerfrom camel.verifiers import BaseVerifier-from .models import VerificationOutcome, VerificationResult, VerifierInput+from .models import VerificationOutcome, VerificationResultlogger = get_logger(__name__)@@ -47,6 +48,7 @@ class PythonVerifier(BaseVerifier):self,timeout: Optional[float] = 30.0,required_packages: Optional[List[str]] = None,+ extractor: Optional[BaseExtractor] = None,):r"""Initializes the PythonVerifier.@@ -102,24 +104,33 @@ class PythonVerifier(BaseVerifier):self.venv_path = Noneasync def _verify_implementation(- self, result: VerifierInput+ self, solution: str, ground_truth: Optional[str]) -> VerificationResult:- r"""Executes the LLM-generated response in a Python virtual- environment.+ r"""Executes and verifies the LLM-generated Python solution in an+ isolated virtual environment.++ This method runs the given Python solution inside a controlled virtual+ environment, captures its execution output, and optionally compares it+ against a provided ground truth. Handles timeouts and execution errors.Args:- result (VerifierInput): Contains the LLM-generated Python code to- execute and optional ground truth for comparison.+ solution (str): The Python code to execute and verify.+ ground_truth (Optional[str]): The expected output for comparison.+ If None, verification is based only on execution success.Returns:- VerificationResult: Contains verification status (SUCCESS/FAILURE/- ERROR), execution output, error messages if any, and execution- duration.+ VerificationResult: A structured object containing:+ - status (VerificationOutcome): SUCCESS, FAILURE, ERROR,+ or TIMEOUT.+ - result (str): The execution output of the solution.+ - error_message (Optional[str]): Captured error message,+ if any.+ - duration (float, optional): Execution time (set externally).Raises:asyncio.TimeoutError: If execution exceeds the configured timeout.- Exception: Any unexpected errors during execution are caught and- converted to an ERROR verification result.+ Exception: Any unexpected errors are caught and converted to an+ ERROR verification result."""if not self.venv_path:return VerificationResult(@@ -128,7 +139,7 @@ class PythonVerifier(BaseVerifier):error_message="Virtual environment is not set up.",)- script = result.llm_response.strip()+ script = solution.strip()venv_python = os.path.join(self.venv_path, self.bin_dir, "python")if not os.path.exists(venv_python):@@ -156,11 +167,11 @@ class PythonVerifier(BaseVerifier):if process.returncode == 0:# If ground truth is provided, compare it with the result- if result.ground_truth is not None:+ if ground_truth is not None:# Normalize both strings by removing extra whitespacenormalized_output = ' '.join(output_result.strip().split())normalized_truth = ' '.join(- str(result.ground_truth).strip().split()+ str(ground_truth).strip().split())if normalized_output == normalized_truth:diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.pyindex f5caf5c74..d104085e9 100644--- a/examples/verifier/python_verifier_example.py+++ b/examples/verifier/python_verifier_example.py@@ -13,7 +13,7 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import asyncio-from camel.verifiers import PythonVerifier, VerifierInput+from camel.verifiers import PythonVerifierverifier = PythonVerifier(required_packages=["numpy"])asyncio.run(verifier.setup())@@ -29,12 +29,14 @@ print(result)# Since the output of the above numpy code evaluates to 32,# we expect the verification outcome to be a success.-response = VerifierInput(llm_response=numpy_test_code, ground_truth="32")-result = asyncio.run(verifier.verify(response))+result = asyncio.run(+ verifier.verify(solution=numpy_test_code, ground_truth="32")+)print(f"Result: {result.status}")-response = VerifierInput(llm_response=numpy_test_code, ground_truth="40")-result = asyncio.run(verifier.verify(response))+result = asyncio.run(+ verifier.verify(solution=numpy_test_code, ground_truth="40")+)# Now we expect the VerificationOutcome to be a failure,# because the answer is wrong.diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.pyindex 1f29a00d5..faa709c11 100644--- a/test/verifiers/test_base_verifier.py+++ b/test/verifiers/test_base_verifier.py@@ -13,16 +13,13 @@# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========import asyncio+from typing import Optionalfrom unittest.mock import AsyncMock, patchimport pytestfrom camel.verifiers.base import BaseVerifier-from camel.verifiers.models import (- VerificationOutcome,- VerificationResult,- VerifierInput,-)+from camel.verifiers.models import VerificationOutcome, VerificationResultclass TestVerifier(BaseVerifier):@@ -35,19 +32,19 @@ class TestVerifier(BaseVerifier):self.cleanup_called = Trueasync def _verify_implementation(- self, result: VerifierInput+ self, solution: str, ground_truth: Optional[str] = None) -> VerificationResult:r"""Simple implementation that returns success or failure based oninput."""- if "fail" in result.llm_response.lower():+ if "fail" in solution.lower():return VerificationResult(status=VerificationOutcome.FAILURE,result="Verification failed",)- elif "error" in result.llm_response.lower():+ elif "error" in solution.lower():raise ValueError("Simulated error in verification")- elif "timeout" in result.llm_response.lower():+ elif "timeout" in solution.lower():raise asyncio.TimeoutError("Simulated timeout")else:return VerificationResult(@@ -140,10 +137,8 @@ async def test_verify_success(test_verifier):await test_verifier.setup()result = await test_verifier.verify(- VerifierInput(- llm_response="This is a successful response",- ground_truth="Expected response",- )+ solution="This is a successful response",+ ground_truth="Expected response",)assert result.status == VerificationOutcome.SUCCESS@@ -160,10 +155,8 @@ async def test_verify_failure(test_verifier):await test_verifier.setup()result = await test_verifier.verify(- VerifierInput(- llm_response="This will fail the verification",- ground_truth="Expected response",- )+ solution="This will fail the verification",+ ground_truth="Expected response",)assert result.status == VerificationOutcome.FAILURE@@ -183,10 +176,8 @@ async def test_verify_error_with_retry(test_verifier):with patch("asyncio.sleep", new_callable=AsyncMock):result = await test_verifier.verify(- VerifierInput(- llm_response="This will cause an error",- ground_truth="Expected response",- )+ solution="This will cause an error",+ ground_truth="Expected response",)assert result.status == VerificationOutcome.ERROR@@ -208,10 +199,8 @@ async def test_verify_timeout(test_verifier):side_effect=asyncio.TimeoutError("Simulated timeout"),):result = await test_verifier.verify(- VerifierInput(- llm_response="This will timeout",- ground_truth="Expected response",- )+ solution="This will timeout",+ ground_truth="Expected response",)assert result.status == VerificationOutcome.TIMEOUT@@ -233,9 +222,7 @@ async def test_verify_not_setup():))- await verifier.verify(- VerifierInput(llm_response="Test", ground_truth="Expected")- )+ await verifier.verify(solution="Test", ground_truth="Expected")verifier.setup.assert_called_once()@@ -252,8 +239,8 @@ async def test_verify_batch(test_verifier):))- async def mock_verify(input_data):- if "fail" in input_data.llm_response.lower():+ async def mock_verify(solution, ground_truth):+ if "fail" in solution.lower():return VerificationResult(status=VerificationOutcome.FAILURE,result="Verification failed",@@ -270,15 +257,10 @@ async def test_verify_batch(test_verifier):test_verifier, *args, **kwargs)- inputs = [- VerifierInput(llm_response="Success 1", ground_truth="Expected 1"),- VerifierInput(llm_response="Success 2", ground_truth="Expected 2"),- VerifierInput(- llm_response="This will fail", ground_truth="Expected 3"- ),- ]+ solutions = ["Success 1", "Success 2", "This will fail"]+ ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]- results = await test_verifier.verify_batch(inputs)+ results = await test_verifier.verify_batch(solutions, ground_truthes)assert len(results) == 3assert results[0].status == VerificationOutcome.SUCCESS@@ -300,8 +282,8 @@ async def test_verify_batch_with_error_handling(test_verifier):))- async def mock_verify(input_data):- if "error" in input_data.llm_response.lower():+ async def mock_verify(solution, ground_truth):+ if "error" in solution.lower():raise ValueError("Simulated error in verification")return VerificationResult(status=VerificationOutcome.SUCCESS,@@ -315,20 +297,17 @@ async def test_verify_batch_with_error_handling(test_verifier):test_verifier, *args, **kwargs)- inputs = [- VerifierInput(llm_response="Success", ground_truth="Expected 1"),- VerifierInput(- llm_response="This will cause an error",- ground_truth="Expected 2",- ),- ]+ solutions = ["Success", "This will cause an error"]+ ground_truthes = ["Expected 1", "Expected 2"]with pytest.raises(RuntimeError, match="One or more verifications failed"):- await test_verifier.verify_batch(inputs, raise_on_error=True)+ await test_verifier.verify_batch(+ solutions, ground_truthes, raise_on_error=True+ )- results = await test_verifier.verify_batch(inputs)+ results = await test_verifier.verify_batch(solutions, ground_truthes)assert len(results) == 2assert results[0].status == VerificationOutcome.SUCCESSassert results[1].status == VerificationOutcome.ERROR@@ -354,28 +333,26 @@ async def test_verify_batch_concurrency_limiting(test_verifier):async def __aexit__(self, *args):self.count -= 1- inputs = [- VerifierInput(- llm_response=f"Success {i}", ground_truth=f"Expected {i}"- )- for i in range(3)- ]+ solutions = ["Success 1", "Success 2", "Success 3"]+ ground_truthes = ["Expected 1", "Expected 2", "Expected 3"]mock_sem = MockSemaphore(1)- async def patched_verify_batch(verifier, inputs, raise_on_error=False):+ async def patched_verify_batch(+ verifier, solutions, ground_truthes, raise_on_error=False+ ):results = []- for input_data in inputs:+ for solution, ground_truth in zip(solutions, ground_truthes):async with mock_sem:await asyncio.sleep(0.01) # Simulate processing time- results.append(await verifier.verify(input_data))+ results.append(await verifier.verify(solution, ground_truth))return resultstest_verifier.verify_batch = lambda *args, **kwargs: patched_verify_batch(test_verifier, *args, **kwargs)- await test_verifier.verify_batch(inputs)+ await test_verifier.verify_batch(solutions, ground_truthes)assert mock_sem.max_count == 1@@ -392,25 +369,19 @@ async def test_full_verification_flow():assert verifier._is_setup is Truesuccess_result = await verifier.verify(- VerifierInput(- llm_response="This should succeed", ground_truth="Expected"- )+ solution="This should succeed", ground_truth="Expected")assert success_result.status == VerificationOutcome.SUCCESSfailure_result = await verifier.verify(- VerifierInput(- llm_response="This will fail", ground_truth="Expected"- )+ solution="This will fail", ground_truth="Expected")assert failure_result.status == VerificationOutcome.FAILUREwith patch("asyncio.sleep", new_callable=AsyncMock):error_result = await verifier.verify(- VerifierInput(- llm_response="This will cause an error",- ground_truth="Expected",- )+ solution="This will cause an error",+ ground_truth="Expected",)assert error_result.status == VerificationOutcome.ERRORdiff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pyindex 0cdf9385e..525d10fb9 100644--- a/test/verifiers/test_python_verifier.py+++ b/test/verifiers/test_python_verifier.py@@ -17,7 +17,6 @@ import pytestfrom camel.verifiers.models import (VerificationOutcome,- VerifierInput,)from camel.verifiers.python_verifier import PythonVerifier@@ -48,7 +47,7 @@ async def test_python_verifier_execution_success(python_verifier):script = 'print("Hello, World!")'result = await python_verifier._verify_implementation(- VerifierInput(llm_response=script)+ solution=script, ground_truth=None)assert result.status == VerificationOutcome.SUCCESS@@ -64,7 +63,7 @@ async def test_python_verifier_execution_failure(python_verifier):script = 'print("Hello"' # Syntax errorresult = await python_verifier._verify_implementation(- VerifierInput(llm_response=script)+ solution=script, ground_truth=None)assert result.status == VerificationOutcome.ERROR@@ -81,7 +80,7 @@ async def test_python_verifier_execution_timeout(python_verifier):script = 'import time; time.sleep(5)'result = await python_verifier._verify_implementation(- VerifierInput(llm_response=script)+ solution=script, ground_truth=None)assert result.status == VerificationOutcome.TIMEOUT@@ -97,7 +96,7 @@ async def test_python_verifier_output_mismatch(python_verifier):script = 'print("Wrong output")'result = await python_verifier._verify_implementation(- VerifierInput(llm_response=script, ground_truth="Expected output")+ solution=script, ground_truth="Expected output")assert result.status == VerificationOutcome.FAILURE@@ -113,7 +112,7 @@ async def test_python_verifier_correct_output_matching(python_verifier):script = 'print("Expected output")'result = await python_verifier._verify_implementation(- VerifierInput(llm_response=script, ground_truth="Expected output")+ solution=script, ground_truth="Expected output")assert result.status == VerificationOutcome.SUCCESS@@ -128,7 +127,7 @@ async def test_python_verifier_no_venv_error():verifier = PythonVerifier()script = 'print("Hello")'result = await verifier._verify_implementation(- VerifierInput(llm_response=script)+ solution=script, ground_truth=None)assert result.status == VerificationOutcome.ERROR@@ -142,7 +141,7 @@ async def test_python_verifier_with_numpy(python_verifier):script = 'import numpy as np; print(np.array([1, 2, 3]))'result = await python_verifier._verify_implementation(- VerifierInput(llm_response=script)+ solution=script, ground_truth=None)assert result.status == VerificationOutcome.SUCCESS