Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

fix: rename 'ground_truth' in verifier to 'reference_answer' (#2066)

hallerite41eebd297137
diff --git a/camel/datasets/few_shot_generator.py b/camel/datasets/few_shot_generator.py
index cd17576d5..f437aac5f 100644
--- a/camel/datasets/few_shot_generator.py
+++ b/camel/datasets/few_shot_generator.py
@@ -203,7 +203,7 @@ class FewShotGenerator(BaseGenerator):
try:
verifier_response = await self.verifier.verify(
solution=rationale,
- ground_truth=None,
+ reference_answer=None,
)
if not verifier_response or not verifier_response.result:
raise ValueError(
diff --git a/camel/datasets/self_instruct_generator.py b/camel/datasets/self_instruct_generator.py
index becc3bde6..58d19d6be 100644
--- a/camel/datasets/self_instruct_generator.py
+++ b/camel/datasets/self_instruct_generator.py
@@ -373,7 +373,7 @@ class SelfInstructGenerator(BaseGenerator):
try:
verifier_response = await self.verifier.verify(
solution=rationale,
- ground_truth=None,
+ reference_answer=None,
)
if not verifier_response or not verifier_response.result:
raise ValueError(
diff --git a/camel/environments/single_step.py b/camel/environments/single_step.py
index b51d8bbc8..a45f988e9 100644
--- a/camel/environments/single_step.py
+++ b/camel/environments/single_step.py
@@ -324,7 +324,7 @@ class SingleStepEnv:
verification_results = await self.verifier.verify_batch(
solutions=proposed_solutions,
- ground_truths=ground_truths, # type: ignore [arg-type]
+ reference_answers=ground_truths, # type: ignore [arg-type]
raise_on_error=True,
)
diff --git a/camel/verifiers/base.py b/camel/verifiers/base.py
index 16b80a772..fc19d5da3 100644
--- a/camel/verifiers/base.py
+++ b/camel/verifiers/base.py
@@ -163,7 +163,7 @@ class BaseVerifier(ABC):
pass
async def verify(
- self, solution: str, ground_truth: Optional[str]
+ self, solution: str, reference_answer: Optional[str]
) -> VerificationResult:
r"""Perform verification with full error handling.
@@ -174,7 +174,7 @@ class BaseVerifier(ABC):
Args:
solution (str): The generated response that needs verification.
- ground_truth (Optional[str]): The expected correct answer to
+ reference_answer (Optional[str]): The expected correct answer to
compare against.
Returns:
@@ -215,13 +215,13 @@ class BaseVerifier(ABC):
verification_result = (
await asyncio.wait_for(
self._verify_implementation(
- verifiable_solution, ground_truth
+ verifiable_solution, reference_answer
),
timeout=self._timeout,
)
if self._timeout
else await self._verify_implementation(
- verifiable_solution, ground_truth
+ verifiable_solution, reference_answer
)
)
@@ -267,7 +267,7 @@ class BaseVerifier(ABC):
@abstractmethod
async def _verify_implementation(
- self, solution: str, ground_truth: Optional[str]
+ self, solution: str, reference_answer: Optional[str]
) -> VerificationResult:
r"""Abstract method for verification logic.
@@ -276,7 +276,7 @@ class BaseVerifier(ABC):
Args:
solution (str): The generated response requiring verification.
- ground_truth (Optional[str]): The expected reference output.
+ reference_answer (Optional[str]): The expected reference output.
Returns:
VerificationResult: Contains verification status and details.
@@ -293,7 +293,7 @@ class BaseVerifier(ABC):
async def verify_batch(
self,
solutions: List[str],
- ground_truths: List[Optional[str]],
+ reference_answers: List[Optional[str]],
raise_on_error: bool = False,
) -> List[VerificationResult]:
r"""Verify multiple solutions in parallel with controlled concurrency.
@@ -305,8 +305,8 @@ class BaseVerifier(ABC):
Args:
solutions (List[str]): A list of generated solutions to be
verified.
- ground_truths (List[Optional[str]]): A list of expected outputs for
- comparison. Each element corresponds to a solution.
+ reference_answers (List[Optional[str]]): A list of expected outputs
+ for comparison. Each element corresponds to a solution.
raise_on_error (bool, optional): If True, raises an exception if
any verification fails. (default: :obj:`False`)
@@ -337,13 +337,13 @@ class BaseVerifier(ABC):
semaphore = asyncio.Semaphore(max(1, max_workers))
async def _verify_with_semaphore(
- solution: str, ground_truth: Optional[str]
+ solution: str, reference_answer: Optional[str]
) -> VerificationResult:
start_time = time.time()
try:
async with semaphore:
verification_result = await self.verify(
- solution, ground_truth
+ solution, reference_answer
)
processing_time = time.time() - start_time
success = (
@@ -368,12 +368,12 @@ class BaseVerifier(ABC):
all_results: List[VerificationResult] = []
for i in range(0, len(solutions), batch_size):
batch_solutions = solutions[i : i + batch_size]
- batch_ground_truths = ground_truths[i : i + batch_size]
+ batch_reference_answers = reference_answers[i : i + batch_size]
verification_tasks = [
- _verify_with_semaphore(solution, ground_truth)
- for solution, ground_truth in zip(
- batch_solutions, batch_ground_truths
+ _verify_with_semaphore(solution, reference_answer)
+ for solution, reference_answer in zip(
+ batch_solutions, batch_reference_answers
)
]
try:
diff --git a/camel/verifiers/math_verifier.py b/camel/verifiers/math_verifier.py
index db3674648..fda0b93bf 100644
--- a/camel/verifiers/math_verifier.py
+++ b/camel/verifiers/math_verifier.py
@@ -65,13 +65,13 @@ class MathVerifier(BaseVerifier):
pass
async def _verify_implementation(
- self, solution: str, ground_truth: Optional[str]
+ self, solution: str, reference_answer: Optional[str]
) -> VerificationResult:
r"""Verify mathematical expressions using Math-Verify.
Args:
solution: The solution to verify
- ground_truth: The expected answer to compare against
+ reference_answer: The expected answer to compare against
Returns:
VerificationResult containing the verification status and details
@@ -82,7 +82,7 @@ class MathVerifier(BaseVerifier):
LatexExtractionConfig,
)
- if ground_truth is None:
+ if reference_answer is None:
return VerificationResult(
status=VerificationOutcome.ERROR,
result="",
@@ -93,8 +93,8 @@ class MathVerifier(BaseVerifier):
try:
# Parse both expressions with LaTeX and plain expression support
- parsed_ground_truth = parse(
- ground_truth,
+ parsed_reference_answer = parse(
+ reference_answer,
extraction_config=[
LatexExtractionConfig(boxed_match_priority=0),
ExprExtractionConfig(),
@@ -108,16 +108,16 @@ class MathVerifier(BaseVerifier):
],
)
- if not parsed_ground_truth or not parsed_solution:
+ if not parsed_reference_answer or not parsed_solution:
return VerificationResult(
status=VerificationOutcome.ERROR,
result="",
error_message="Failed to parse expressions",
)
- # Order matters! ground_truth must be first argument
+ # Order matters! reference_answer must be first argument
is_correct = verify(
- parsed_ground_truth,
+ parsed_reference_answer,
parsed_solution,
float_rounding=self.float_rounding,
numeric_precision=self.numeric_precision,
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index a151f4c25..f4ea89818 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -222,7 +222,7 @@ class PythonVerifier(BaseVerifier):
self.venv_path = None
async def _verify_implementation(
- self, solution: str, ground_truth: Optional[str]
+ self, solution: str, reference_answer: Optional[str]
) -> VerificationResult:
r"""Executes the provided Python solution in an isolated environment
and verifies its output against an expected ground truth expression.
@@ -238,7 +238,7 @@ class PythonVerifier(BaseVerifier):
Args:
solution (str): The Python code or expression to execute and
verify.
- ground_truth (Optional[str]): The expected value as a Python
+ reference_answer (Optional[str]): The expected value as a Python
expression. If None, only execution success is verified.
Returns:
@@ -263,9 +263,9 @@ class PythonVerifier(BaseVerifier):
error_message=f"Expression evaluation error: {e}",
)
- if ground_truth is not None:
+ if reference_answer is not None:
try:
- gt_val = ast.literal_eval(ground_truth)
+ gt_val = ast.literal_eval(reference_answer)
except Exception as e:
return VerificationResult(
status=VerificationOutcome.ERROR,
@@ -315,7 +315,7 @@ class PythonVerifier(BaseVerifier):
error_message=f"Solution code error:\n{sol_err}",
)
- if ground_truth is not None:
+ if reference_answer is not None:
try:
# First, try to evaluate the output as-is.
sol_val = ast.literal_eval(sol_out)
@@ -325,7 +325,7 @@ class PythonVerifier(BaseVerifier):
if sol_val is not None:
try:
- gt_val = ast.literal_eval(ground_truth)
+ gt_val = ast.literal_eval(reference_answer)
except Exception as e:
return VerificationResult(
status=VerificationOutcome.ERROR,
@@ -347,7 +347,7 @@ class PythonVerifier(BaseVerifier):
)
else:
# Fallback: string comparison
- if sol_out.strip() == ground_truth.strip():
+ if sol_out.strip() == reference_answer.strip():
return VerificationResult(
status=VerificationOutcome.SUCCESS,
result=sol_out,
@@ -357,7 +357,7 @@ class PythonVerifier(BaseVerifier):
status=VerificationOutcome.FAILURE,
result=sol_out,
error_message="Fallback string mismatch: "
- f"'{sol_out}' != '{ground_truth}'",
+ f"'{sol_out}' != '{reference_answer}'",
)
else:
return VerificationResult(
diff --git a/examples/verifier/math_verifier_example.py b/examples/verifier/math_verifier_example.py
index f5cf9e4c7..ca2929ebb 100644
--- a/examples/verifier/math_verifier_example.py
+++ b/examples/verifier/math_verifier_example.py
@@ -32,7 +32,9 @@ async def main():
try:
# Test case 1: Basic numerical equivalence (should succeed)
print("=== Test 1: Basic Numerical ===")
- result = await verifier.verify(solution="0.333333", ground_truth="1/3")
+ result = await verifier.verify(
+ solution="0.333333", reference_answer="1/3"
+ )
print("Input: 0.333333 ≈ 1/3")
print(f"Result: {result.status}")
print(
@@ -42,7 +44,7 @@ async def main():
# Test case 2: LaTeX expressions (should succeed)
print("=== Test 2: LaTeX Expression ===")
result = await verifier.verify(
- solution=r"$\frac{1}{2}$", ground_truth=r"0.5"
+ solution=r"$\frac{1}{2}$", reference_answer=r"0.5"
)
print("Input: \\frac{1}{2} = 0.5")
print(f"Result: {result.status}")
@@ -52,7 +54,9 @@ async def main():
# Test case 3: Deliberate mismatch (should fail)
print("=== Test 3: Expected Failure ===")
- result = await verifier.verify(solution="0.5", ground_truth="0.3333")
+ result = await verifier.verify(
+ solution="0.5", reference_answer="0.3333"
+ )
print("Input: 0.5 ≠ 0.3333")
print(f"Result: {result.status}")
print(
diff --git a/examples/verifier/python_verifier_example.py b/examples/verifier/python_verifier_example.py
index 7c6f4979c..0db601ef0 100644
--- a/examples/verifier/python_verifier_example.py
+++ b/examples/verifier/python_verifier_example.py
@@ -30,12 +30,12 @@ print(result)
# Since the output of the above numpy code evaluates to 32,
# we expect the verification outcome to be a success.
result = asyncio.run(
- verifier.verify(solution=numpy_test_code, ground_truth="32")
+ verifier.verify(solution=numpy_test_code, reference_answer="32")
)
print(f"Result: {result}")
result = asyncio.run(
- verifier.verify(solution=numpy_test_code, ground_truth="40")
+ verifier.verify(solution=numpy_test_code, reference_answer="40")
)
# Now we expect the VerificationOutcome to be a failure,
diff --git a/test/environments/test_single_stage_env.py b/test/environments/test_single_stage_env.py
index 319696cf9..be44429e4 100644
--- a/test/environments/test_single_stage_env.py
+++ b/test/environments/test_single_stage_env.py
@@ -59,7 +59,7 @@ async def test_single_step_env_lifecycle_single():
mock_verifier.setup = AsyncMock()
mock_verifier.cleanup = AsyncMock()
mock_verifier.verify_batch = AsyncMock(
- side_effect=lambda solutions, ground_truths, **kwargs: [
+ side_effect=lambda solutions, reference_answers, **kwargs: [
VerificationResult(
status=VerificationOutcome.SUCCESS,
result="Verification successful",
@@ -190,7 +190,7 @@ async def test_batched_single_step_env_lifecycle():
mock_verifier.setup = AsyncMock()
mock_verifier.cleanup = AsyncMock()
mock_verifier.verify_batch = AsyncMock(
- side_effect=lambda solutions, ground_truths, **kwargs: [
+ side_effect=lambda solutions, reference_answers, **kwargs: [
VerificationResult(
status=VerificationOutcome.SUCCESS,
result="Verification successful",
@@ -262,7 +262,7 @@ def create_mock_verifier():
verifier.setup = AsyncMock()
verifier.cleanup = AsyncMock()
verifier.verify_batch = AsyncMock(
- side_effect=lambda solutions, ground_truths, **kwargs: [
+ side_effect=lambda solutions, reference_answers, **kwargs: [
VerificationResult(
status=VerificationOutcome.SUCCESS,
result="Mock verification",
diff --git a/test/verifiers/test_base_verifier.py b/test/verifiers/test_base_verifier.py
index 715fff7bb..831c8c7c9 100644
--- a/test/verifiers/test_base_verifier.py
+++ b/test/verifiers/test_base_verifier.py
@@ -138,7 +138,7 @@ async def test_verify_success(test_verifier):
result = await test_verifier.verify(
solution="This is a successful response",
- ground_truth="Expected response",
+ reference_answer="Expected response",
)
assert result.status == VerificationOutcome.SUCCESS
@@ -156,7 +156,7 @@ async def test_verify_failure(test_verifier):
result = await test_verifier.verify(
solution="This will fail the verification",
- ground_truth="Expected response",
+ reference_answer="Expected response",
)
assert result.status == VerificationOutcome.FAILURE
@@ -177,7 +177,7 @@ async def test_verify_error_with_retry(test_verifier):
with patch("asyncio.sleep", new_callable=AsyncMock):
result = await test_verifier.verify(
solution="This will cause an error",
- ground_truth="Expected response",
+ reference_answer="Expected response",
)
assert result.status == VerificationOutcome.ERROR
@@ -200,7 +200,7 @@ async def test_verify_timeout(test_verifier):
):
result = await test_verifier.verify(
solution="This will timeout",
- ground_truth="Expected response",
+ reference_answer="Expected response",
)
assert result.status == VerificationOutcome.TIMEOUT
@@ -222,7 +222,7 @@ async def test_verify_not_setup():
)
)
- await verifier.verify(solution="Test", ground_truth="Expected")
+ await verifier.verify(solution="Test", reference_answer="Expected")
verifier.setup.assert_called_once()
@@ -369,19 +369,19 @@ async def test_full_verification_flow():
assert verifier._is_setup is True
success_result = await verifier.verify(
- solution="This should succeed", ground_truth="Expected"
+ solution="This should succeed", reference_answer="Expected"
)
assert success_result.status == VerificationOutcome.SUCCESS
failure_result = await verifier.verify(
- solution="This will fail", ground_truth="Expected"
+ solution="This will fail", reference_answer="Expected"
)
assert failure_result.status == VerificationOutcome.FAILURE
with patch("asyncio.sleep", new_callable=AsyncMock):
error_result = await verifier.verify(
solution="This will cause an error",
- ground_truth="Expected",
+ reference_answer="Expected",
)
assert error_result.status == VerificationOutcome.ERROR
diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.py
index 902d945be..394200fc8 100644
--- a/test/verifiers/test_python_verifier.py
+++ b/test/verifiers/test_python_verifier.py
@@ -48,7 +48,9 @@ async def test_python_verifier_execution_success(python_verifier):
await python_verifier.setup(uv=True)
script = 'print("Hello, World!")'
- result = await python_verifier.verify(solution=script, ground_truth=None)
+ result = await python_verifier.verify(
+ solution=script, reference_answer=None
+ )
assert result.status == VerificationOutcome.SUCCESS
assert result.result == "Hello, World!"
@@ -62,7 +64,7 @@ async def test_python_verifier_execution_failure(python_verifier):
script = 'print("Hello"' # Syntax error
result = await python_verifier._verify_implementation(
- solution=script, ground_truth=None
+ solution=script, reference_answer=None
)
assert result.status == VerificationOutcome.ERROR
@@ -79,7 +81,7 @@ async def test_python_verifier_execution_timeout(python_verifier):
script = 'import time; time.sleep(5)'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth=None
+ solution=script, reference_answer=None
)
assert result.status == VerificationOutcome.TIMEOUT
@@ -95,7 +97,7 @@ async def test_python_verifier_output_mismatch(python_verifier):
script = 'print("Wrong output")'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth="Expected output"
+ solution=script, reference_answer="Expected output"
)
assert result.status == VerificationOutcome.FAILURE
@@ -114,7 +116,7 @@ async def test_python_verifier_correct_output_matching(python_verifier):
script = 'print("Expected output")'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth="Expected output"
+ solution=script, reference_answer="Expected output"
)
assert result.status == VerificationOutcome.SUCCESS
@@ -129,7 +131,7 @@ async def test_python_verifier_no_venv_error():
verifier = PythonVerifier()
script = 'print("Hello")'
result = await verifier._verify_implementation(
- solution=script, ground_truth=None
+ solution=script, reference_answer=None
)
assert result.status == VerificationOutcome.ERROR
@@ -143,7 +145,7 @@ async def test_python_verifier_with_numpy(python_verifier):
script = 'import numpy as np; print(np.array([1, 2, 3]))'
result = await python_verifier._verify_implementation(
- solution=script, ground_truth=None
+ solution=script, reference_answer=None
)
assert result.status == VerificationOutcome.SUCCESS