🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
fix: rename 'ground_truth' in verifier to 'reference_answer' (#2066)
hallerite41eebd297137
diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.pyindex cd17576d5..f437aac5f 100644--- a/camel/datasets/few_shot_generator.py+++ b/camel/datasets/few_shot_generator.py@@ -203,7 +203,7 @@ class FewShotGenerator(BaseGenerator):try:verifier_response = await self.verifier.verify(solution=rationale,- ground_truth=None,+ reference_answer=None,)if not verifier_response or not verifier_response.result:raise ValueError(diff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.pyindex becc3bde6..58d19d6be 100644--- a/camel/datasets/self_instruct_generator.py+++ b/camel/datasets/self_instruct_generator.py@@ -373,7 +373,7 @@ class SelfInstructGenerator(BaseGenerator):try:verifier_response = await self.verifier.verify(solution=rationale,- ground_truth=None,+ reference_answer=None,)if not verifier_response or not verifier_response.result:raise ValueError(diff --git a/camel/environments/single_step.py b/camel/environments/single_step.pyindex b51d8bbc8..a45f988e9 100644--- a/camel/environments/single_step.py+++ b/camel/environments/single_step.py@@ -324,7 +324,7 @@ class SingleStepEnv:verification_results = await self.verifier.verify_batch(solutions=proposed_solutions,- ground_truths=ground_truths, # type: ignore [arg-type]+ reference_answers=ground_truths, # type: ignore [arg-type]raise_on_error=True,)diff --git a/camel/verifiers/base.py b/camel/verifiers/base.pyindex 16b80a772..fc19d5da3 100644--- a/camel/verifiers/base.py+++ b/camel/verifiers/base.py@@ -163,7 +163,7 @@ class BaseVerifier(ABC):passasync def verify(- self, solution: str, ground_truth: Optional[str]+ self, solution: str, reference_answer: Optional[str]) -> VerificationResult:r"""Perform verification with full error handling.@@ -174,7 +174,7 @@ class BaseVerifier(ABC):Args:solution (str): The generated response that needs verification.- ground_truth (Optional[str]): The expected correct answer to+ reference_answer (Optional[str]): The expected correct answer tocompare against.Returns:@@ -215,13 +215,13 @@ class BaseVerifier(ABC):verification_result = (await asyncio.wait_for(self._verify_implementation(- verifiable_solution, ground_truth+ verifiable_solution, reference_answer),timeout=self._timeout,)if self._timeoutelse await self._verify_implementation(- verifiable_solution, ground_truth+ verifiable_solution, reference_answer))@@ -267,7 +267,7 @@ class BaseVerifier(ABC):@abstractmethodasync def _verify_implementation(- self, solution: str, ground_truth: Optional[str]+ self, solution: str, reference_answer: Optional[str]) -> VerificationResult:r"""Abstract method for verification logic.@@ -276,7 +276,7 @@ class BaseVerifier(ABC):Args:solution (str): The generated response requiring verification.- ground_truth (Optional[str]): The expected reference output.+ reference_answer (Optional[str]): The expected reference output.Returns:VerificationResult: Contains verification status and details.@@ -293,7 +293,7 @@ class BaseVerifier(ABC):async def verify_batch(self,solutions: List[str],- ground_truths: List[Optional[str]],+ reference_answers: List[Optional[str]],raise_on_error: bool = False,) -> List[VerificationResult]:r"""Verify multiple solutions in parallel with controlled concurrency.@@ -305,8 +305,8 @@ class BaseVerifier(ABC):Args:solutions (List[str]): A list of generated solutions to beverified.- ground_truths (List[Optional[str]]): A list of expected outputs for- comparison. Each element corresponds to a solution.+ reference_answers (List[Optional[str]]): A list of expected outputs+ for comparison. Each element corresponds to a solution.raise_on_error (bool, optional): If True, raises an exception ifany verification fails. (default: :obj:`False`)@@ -337,13 +337,13 @@ class BaseVerifier(ABC):semaphore = asyncio.Semaphore(max(1, max_workers))async def _verify_with_semaphore(- solution: str, ground_truth: Optional[str]+ solution: str, reference_answer: Optional[str]) -> VerificationResult:start_time = time.time()try:async with semaphore:verification_result = await self.verify(- solution, ground_truth+ solution, reference_answer)processing_time = time.time() - start_timesuccess = (@@ -368,12 +368,12 @@ class BaseVerifier(ABC):all_results: List[VerificationResult] = []for i in range(0, len(solutions), batch_size):batch_solutions = solutions[i : i + batch_size]- batch_ground_truths = ground_truths[i : i + batch_size]+ batch_reference_answers = reference_answers[i : i + batch_size]verification_tasks = [- _verify_with_semaphore(solution, ground_truth)- for solution, ground_truth in zip(- batch_solutions, batch_ground_truths+ _verify_with_semaphore(solution, reference_answer)+ for solution, reference_answer in zip(+ batch_solutions, batch_reference_answers)]try:diff --git a/camel/verifiers/math_verifier.py b/camel/verifiers/math_verifier.pyindex db3674648..fda0b93bf 100644--- a/camel/verifiers/math_verifier.py+++ b/camel/verifiers/math_verifier.py@@ -65,13 +65,13 @@ class MathVerifier(BaseVerifier):passasync def _verify_implementation(- self, solution: str, ground_truth: Optional[str]+ self, solution: str, reference_answer: Optional[str]) -> VerificationResult:r"""Verify mathematical expressions using Math-Verify.Args:solution: The solution to verify- ground_truth: The expected answer to compare against+ reference_answer: The expected answer to compare againstReturns:VerificationResult containing the verification status and details@@ -82,7 +82,7 @@ class MathVerifier(BaseVerifier):LatexExtractionConfig,)- if ground_truth is None:+ if reference_answer is None:return VerificationResult(status=VerificationOutcome.ERROR,result="",@@ -93,8 +93,8 @@ class MathVerifier(BaseVerifier):try:# Parse both expressions with LaTeX and plain expression support- parsed_ground_truth = parse(- ground_truth,+ parsed_reference_answer = parse(+ reference_answer,extraction_config=[LatexExtractionConfig(boxed_match_priority=0),ExprExtractionConfig(),@@ -108,16 +108,16 @@ class MathVerifier(BaseVerifier):],)- if not parsed_ground_truth or not parsed_solution:+ if not parsed_reference_answer or not parsed_solution:return VerificationResult(status=VerificationOutcome.ERROR,result="",error_message="Failed to parse expressions",)- # Order matters! ground_truth must be first argument+ # Order matters! reference_answer must be first argumentis_correct = verify(- parsed_ground_truth,+ parsed_reference_answer,parsed_solution,float_rounding=self.float_rounding,numeric_precision=self.numeric_precision,diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex a151f4c25..f4ea89818 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -222,7 +222,7 @@ class PythonVerifier(BaseVerifier):self.venv_path = Noneasync def _verify_implementation(- self, solution: str, ground_truth: Optional[str]+ self, solution: str, reference_answer: Optional[str]) -> VerificationResult:r"""Executes the provided Python solution in an isolated environmentand verifies its output against an expected ground truth expression.@@ -238,7 +238,7 @@ class PythonVerifier(BaseVerifier):Args:solution (str): The Python code or expression to execute andverify.- ground_truth (Optional[str]): The expected value as a Python+ reference_answer (Optional[str]): The expected value as a Pythonexpression. If None, only execution success is verified.Returns:@@ -263,9 +263,9 @@ class PythonVerifier(BaseVerifier):error_message=f"Expression evaluation error: {e}",)- if ground_truth is not None:+ if reference_answer is not None:try:- gt_val = ast.literal_eval(ground_truth)+ gt_val = ast.literal_eval(reference_answer)except Exception as e:return VerificationResult(status=VerificationOutcome.ERROR,@@ -315,7 +315,7 @@ class PythonVerifier(BaseVerifier):error_message=f"Solution code error:\n{sol_err}",)- if ground_truth is not None:+ if reference_answer is not None:try:# First, try to evaluate the output as-is.sol_val = ast.literal_eval(sol_out)@@ -325,7 +325,7 @@ class PythonVerifier(BaseVerifier):if sol_val is not None:try:- gt_val = ast.literal_eval(ground_truth)+ gt_val = ast.literal_eval(reference_answer)except Exception as e:return VerificationResult(status=VerificationOutcome.ERROR,@@ -347,7 +347,7 @@ class PythonVerifier(BaseVerifier):)else:# Fallback: string comparison- if sol_out.strip() == ground_truth.strip():+ if sol_out.strip() == reference_answer.strip():return VerificationResult(status=VerificationOutcome.SUCCESS,result=sol_out,@@ -357,7 +357,7 @@ class PythonVerifier(BaseVerifier):status=VerificationOutcome.FAILURE,result=sol_out,error_message="Fallback string mismatch: "- f"'{sol_out}' != '{ground_truth}'",+ f"'{sol_out}' != '{reference_answer}'",)else:return VerificationResult(diff --git a/examples/verifier/math_verifier_example.py b/examples/verifier/math_verifier_example.pyindex f5cf9e4c7..ca2929ebb 100644--- a/examples/verifier/math_verifier_example.py+++ b/examples/verifier/math_verifier_example.py@@ -32,7 +32,9 @@ async def main():try:# Test case 1: Basic numerical equivalence (should succeed)print("=== Test 1: Basic Numerical ===")- result = await verifier.verify(solution="0.333333", ground_truth="1/3")+ result = await verifier.verify(+ solution="0.333333", reference_answer="1/3"+ )print("Input: 0.333333 ≈ 1/3")print(f"Result: {result.status}")print(@@ -42,7 +44,7 @@ async def main():# Test case 2: LaTeX expressions (should succeed)print("=== Test 2: LaTeX Expression ===")result = await verifier.verify(- solution=r"$\frac{1}{2}$", ground_truth=r"0.5"+ solution=r"$\frac{1}{2}$", reference_answer=r"0.5")print("Input: \\frac{1}{2} = 0.5")print(f"Result: {result.status}")@@ -52,7 +54,9 @@ async def main():# Test case 3: Deliberate mismatch (should fail)print("=== Test 3: Expected Failure ===")- result = await verifier.verify(solution="0.5", ground_truth="0.3333")+ result = await verifier.verify(+ solution="0.5", reference_answer="0.3333"+ )print("Input: 0.5 ≠ 0.3333")print(f"Result: {result.status}")print(diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.pyindex 7c6f4979c..0db601ef0 100644--- a/examples/verifier/python_verifier_example.py+++ b/examples/verifier/python_verifier_example.py@@ -30,12 +30,12 @@ print(result)# Since the output of the above numpy code evaluates to 32,# we expect the verification outcome to be a success.result = asyncio.run(- verifier.verify(solution=numpy_test_code, ground_truth="32")+ verifier.verify(solution=numpy_test_code, reference_answer="32"))print(f"Result: {result}")result = asyncio.run(- verifier.verify(solution=numpy_test_code, ground_truth="40")+ verifier.verify(solution=numpy_test_code, reference_answer="40"))# Now we expect the VerificationOutcome to be a failure,diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.pyindex 319696cf9..be44429e4 100644--- a/test/environments/test_single_stage_env.py+++ b/test/environments/test_single_stage_env.py@@ -59,7 +59,7 @@ async def test_single_step_env_lifecycle_single():mock_verifier.setup = AsyncMock()mock_verifier.cleanup = AsyncMock()mock_verifier.verify_batch = AsyncMock(- side_effect=lambda solutions, ground_truths, **kwargs: [+ side_effect=lambda solutions, reference_answers, **kwargs: [VerificationResult(status=VerificationOutcome.SUCCESS,result="Verification successful",@@ -190,7 +190,7 @@ async def test_batched_single_step_env_lifecycle():mock_verifier.setup = AsyncMock()mock_verifier.cleanup = AsyncMock()mock_verifier.verify_batch = AsyncMock(- side_effect=lambda solutions, ground_truths, **kwargs: [+ side_effect=lambda solutions, reference_answers, **kwargs: [VerificationResult(status=VerificationOutcome.SUCCESS,result="Verification successful",@@ -262,7 +262,7 @@ def create_mock_verifier():verifier.setup = AsyncMock()verifier.cleanup = AsyncMock()verifier.verify_batch = AsyncMock(- side_effect=lambda solutions, ground_truths, **kwargs: [+ side_effect=lambda solutions, reference_answers, **kwargs: [VerificationResult(status=VerificationOutcome.SUCCESS,result="Mock verification",diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.pyindex 715fff7bb..831c8c7c9 100644--- a/test/verifiers/test_base_verifier.py+++ b/test/verifiers/test_base_verifier.py@@ -138,7 +138,7 @@ async def test_verify_success(test_verifier):result = await test_verifier.verify(solution="This is a successful response",- ground_truth="Expected response",+ reference_answer="Expected response",)assert result.status == VerificationOutcome.SUCCESS@@ -156,7 +156,7 @@ async def test_verify_failure(test_verifier):result = await test_verifier.verify(solution="This will fail the verification",- ground_truth="Expected response",+ reference_answer="Expected response",)assert result.status == VerificationOutcome.FAILURE@@ -177,7 +177,7 @@ async def test_verify_error_with_retry(test_verifier):with patch("asyncio.sleep", new_callable=AsyncMock):result = await test_verifier.verify(solution="This will cause an error",- ground_truth="Expected response",+ reference_answer="Expected response",)assert result.status == VerificationOutcome.ERROR@@ -200,7 +200,7 @@ async def test_verify_timeout(test_verifier):):result = await test_verifier.verify(solution="This will timeout",- ground_truth="Expected response",+ reference_answer="Expected response",)assert result.status == VerificationOutcome.TIMEOUT@@ -222,7 +222,7 @@ async def test_verify_not_setup():))- await verifier.verify(solution="Test", ground_truth="Expected")+ await verifier.verify(solution="Test", reference_answer="Expected")verifier.setup.assert_called_once()@@ -369,19 +369,19 @@ async def test_full_verification_flow():assert verifier._is_setup is Truesuccess_result = await verifier.verify(- solution="This should succeed", ground_truth="Expected"+ solution="This should succeed", reference_answer="Expected")assert success_result.status == VerificationOutcome.SUCCESSfailure_result = await verifier.verify(- solution="This will fail", ground_truth="Expected"+ solution="This will fail", reference_answer="Expected")assert failure_result.status == VerificationOutcome.FAILUREwith patch("asyncio.sleep", new_callable=AsyncMock):error_result = await verifier.verify(solution="This will cause an error",- ground_truth="Expected",+ reference_answer="Expected",)assert error_result.status == VerificationOutcome.ERRORdiff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pyindex 902d945be..394200fc8 100644--- a/test/verifiers/test_python_verifier.py+++ b/test/verifiers/test_python_verifier.py@@ -48,7 +48,9 @@ async def test_python_verifier_execution_success(python_verifier):await python_verifier.setup(uv=True)script = 'print("Hello, World!")'- result = await python_verifier.verify(solution=script, ground_truth=None)+ result = await python_verifier.verify(+ solution=script, reference_answer=None+ )assert result.status == VerificationOutcome.SUCCESSassert result.result == "Hello, World!"@@ -62,7 +64,7 @@ async def test_python_verifier_execution_failure(python_verifier):script = 'print("Hello"' # Syntax errorresult = await python_verifier._verify_implementation(- solution=script, ground_truth=None+ solution=script, reference_answer=None)assert result.status == VerificationOutcome.ERROR@@ -79,7 +81,7 @@ async def test_python_verifier_execution_timeout(python_verifier):script = 'import time; time.sleep(5)'result = await python_verifier._verify_implementation(- solution=script, ground_truth=None+ solution=script, reference_answer=None)assert result.status == VerificationOutcome.TIMEOUT@@ -95,7 +97,7 @@ async def test_python_verifier_output_mismatch(python_verifier):script = 'print("Wrong output")'result = await python_verifier._verify_implementation(- solution=script, ground_truth="Expected output"+ solution=script, reference_answer="Expected output")assert result.status == VerificationOutcome.FAILURE@@ -114,7 +116,7 @@ async def test_python_verifier_correct_output_matching(python_verifier):script = 'print("Expected output")'result = await python_verifier._verify_implementation(- solution=script, ground_truth="Expected output"+ solution=script, reference_answer="Expected output")assert result.status == VerificationOutcome.SUCCESS@@ -129,7 +131,7 @@ async def test_python_verifier_no_venv_error():verifier = PythonVerifier()script = 'print("Hello")'result = await verifier._verify_implementation(- solution=script, ground_truth=None+ solution=script, reference_answer=None)assert result.status == VerificationOutcome.ERROR@@ -143,7 +145,7 @@ async def test_python_verifier_with_numpy(python_verifier):script = 'import numpy as np; print(np.array([1, 2, 3]))'result = await python_verifier._verify_implementation(- solution=script, ground_truth=None+ solution=script, reference_answer=None)assert result.status == VerificationOutcome.SUCCESS