🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
update: add float type comparison (#2133)
Rishabh2fc573496434
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.pyindex 94d20759f..8181f8342 100644--- a/camel/verifiers/python_verifier.py+++ b/camel/verifiers/python_verifier.py@@ -20,7 +20,7 @@ import subprocessimport sysimport tempfileimport venv-from typing import List, Optional, Tuple+from typing import Any, List, Optional, Tuplefrom camel.extractors.base import BaseExtractorfrom camel.logger import get_logger@@ -51,6 +51,7 @@ class PythonVerifier(BaseVerifier):extractor: Optional[BaseExtractor] = None,timeout: Optional[float] = 30.0,required_packages: Optional[List[str]] = None,+ float_tolerance: Optional[float] = 1e-05,**kwargs,):r"""Initializes the PythonVerifier.@@ -63,11 +64,14 @@ class PythonVerifier(BaseVerifier):required_packages (Optional[List[str]], optional): A list ofpackages to install in the virtual environment.(default: :obj:`None`)+ float_tolerance (Optional[float], optional): The tolerance for+ floating point comparisons. (default: :obj:`1e-05`)"""# TODO: Use CAMEL's Interpreter to execute the codesuper().__init__(extractor=extractor, timeout=timeout, **kwargs)self.venv_path: Optional[str] = Noneself.required_packages = required_packages or []+ self.float_tolerance = float_toleranceif os.name == 'nt': # Windowsself.bin_dir = 'Scripts'@@ -272,7 +276,13 @@ class PythonVerifier(BaseVerifier):result="",error_message=f"Ground truth evaluation error: {e}",)- if sol_val == gt_val:++ if self.float_tolerance is not None:+ equal = self._is_equal_with_tolerance(sol_val, gt_val)+ else:+ equal = sol_val == gt_val++ if equal:return VerificationResult(status=VerificationOutcome.SUCCESS,result=str(sol_val),@@ -281,9 +291,18 @@ class PythonVerifier(BaseVerifier):return VerificationResult(status=VerificationOutcome.FAILURE,result=str(sol_val),- error_message="Output mismatch: "- f"{sol_val} != {gt_val}",+ error_message=(+ "Values not equal"+ + (+ " (with float tolerance "+ f"{self.float_tolerance})"+ if self.float_tolerance is not None+ else ""+ )+ + f": {sol_val} != {gt_val}"+ ),)+else:return VerificationResult(status=VerificationOutcome.SUCCESS,@@ -333,17 +352,21 @@ class PythonVerifier(BaseVerifier):error_message="Ground truth evaluation error:"f"{e}",)- if sol_val == gt_val:+ if self.float_tolerance is not None:+ equal = self._is_equal_with_tolerance(sol_val, gt_val)+ else:+ equal = sol_val == gt_val++ if equal:return VerificationResult(- status=VerificationOutcome.SUCCESS,- result=sol_out,+ status=VerificationOutcome.SUCCESS, result=sol_out)else:return VerificationResult(status=VerificationOutcome.FAILURE,result=sol_out,- error_message="Output mismatch: "- f"{sol_val} != {gt_val}",+ error_message=f"Output mismatch: {sol_val} "+ f"!= {gt_val}",)else:# Fallback: string comparison@@ -456,3 +479,64 @@ class PythonVerifier(BaseVerifier):return Trueexcept Exception:return False++ def _is_equal_with_tolerance(self, a: Any, b: Any) -> bool:+ r"""Compares two Python objects for equality with optional float+ tolerance.++ This method recursively compares nested structures (lists, tuples,+ sets, and dictionaries) and applies floating point tolerance when+ comparing numerical values. If no float tolerance is set, a runtime+ error is raised.++ Args:+ a (Any): First value to compare.+ b (Any): Second value to compare.++ Returns:+ bool: True if the values are considered equal within the+ specified float tolerance; False otherwise.++ Raises:+ RuntimeError: If float tolerance is not set (i.e., None).+ """+ if self.float_tolerance is None:+ raise RuntimeError(+ "Can't compare with tolerance if tolerance is None."+ )+ if isinstance(a, (int, float)) and isinstance(b, (int, float)):+ return abs(float(a) - float(b)) <= self.float_tolerance+ if isinstance(a, list) and isinstance(b, list):+ return len(a) == len(b) and all(+ self._is_equal_with_tolerance(x, y) for x, y in zip(a, b)+ )+ if isinstance(a, tuple) and isinstance(b, tuple):+ return len(a) == len(b) and all(+ self._is_equal_with_tolerance(x, y) for x, y in zip(a, b)+ )+ if isinstance(a, set) and isinstance(b, set):+ if len(a) != len(b):+ return False+ # Need to check both directions to ensure proper matching+ # Create a copy of b to track matched elements+ b_copy = list(b)+ for x in a:+ found_match = False+ for i, y in enumerate(b_copy):+ if self._is_equal_with_tolerance(x, y):+ found_match = True+ # Remove the matched element to prevent double-matching+ b_copy.pop(i)+ break+ if not found_match:+ return False+ return True+ if isinstance(a, dict) and isinstance(b, dict):+ if set(a.keys()) != set(b.keys()):+ return False+ return all(self._is_equal_with_tolerance(a[k], b[k]) for k in a)+ logger.warning(+ f"Falling back to simple comparison without "+ f"tolerance for {a} and {b}."+ )+ return a == b # fallbackdiff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.pyindex 394200fc8..2b548869a 100644--- a/test/verifiers/test_python_verifier.py+++ b/test/verifiers/test_python_verifier.py@@ -151,3 +151,91 @@ async def test_python_verifier_with_numpy(python_verifier):assert result.status == VerificationOutcome.SUCCESSassert "[1 2 3]" in result.resultawait python_verifier.cleanup()+++ "a,b,tol,expected",+ [+ (0.123456, 0.123457, 1e-5, True),+ (0.123456, 0.123467, 1e-5, False),+ ([1.0, 2.0], [1.0, 2.000001], 1e-5, True),+ ([1.0, 2.0], [1.0, 2.1], 1e-5, False),+ ({'a': 1.0, 'b': 2.0}, {'a': 1.0000001, 'b': 2.0}, 1e-5, True),+ ({'a': 1.0, 'b': 2.0}, {'a': 1.0001, 'b': 2.0}, 1e-5, False),+ ((1.0, 2.0), (1.0, 2.000001), 1e-5, True),+ ((1.0, 2.0), (1.0, 2.01), 1e-5, False),+ ({1.0, 2.0}, {2.000001, 1.0}, 1e-5, True),+ ({1.0, 2.0}, {2.1, 1.0}, 1e-5, False),+ ({1.0, 1.000001}, {1.000002, 2.0}, 1e-5, False),+ ({"x": [1.0, 2.0]}, {"x": [1.000001, 2.0]}, 1e-5, True),+ ({"x": [1.0, 2.0]}, {"x": [1.1, 2.0]}, 1e-5, False),+ ],+)+def test_is_equal_with_tolerance(a, b, tol, expected):+ r"""Test the function calculating whether two floats are equal+ given tolerance"""+ verifier = PythonVerifier(float_tolerance=tol)+ assert verifier._is_equal_with_tolerance(a, b) == expected+++ "script,expected_output,tol,should_pass",+ [+ # Float scalar+ ("0.123456", "0.123457", 1e-5, True),+ ("0.123456", "0.123467", 1e-5, False),+ # Lists+ ("[1.0, 2.0]", "[1.0, 2.000001]", 1e-5, True),+ ("[1.0, 2.0]", "[1.0, 2.1]", 1e-5, False),+ # Dicts+ ("{'a': 1.0, 'b': 2.0}", "{'a': 1.000001, 'b': 2.0}", 1e-5, True),+ ("{'a': 1.0, 'b': 2.0}", "{'a': 1.1, 'b': 2.0}", 1e-5, False),+ # Sets+ ("{1.0, 2.0}", "{1.0, 2.000001}", 1e-5, True),+ ("{1.0, 2.0}", "{1.0, 2.1}", 1e-5, False),+ # Tuples+ ("(1.0, 2.0)", "(1.0, 2.000001)", 1e-5, True),+ ("(1.0, 2.0)", "(1.0, 2.1)", 1e-5, False),+ # Mixed nesting+ ("{'a': [1.0, 2.0]}", "{'a': [1.000001, 2.0]}", 1e-5, True),+ ("{'a': [1.0, 2.0]}", "{'a': [1.1, 2.0]}", 1e-5, False),+ # Same as above, but as code blocks+ ("print(0.123456)", "0.123457", 1e-5, True),+ ("print(0.123456)", "0.123467", 1e-5, False),+ ("print([1.0, 2.0])", "[1.0, 2.000001]", 1e-5, True),+ ("print([1.0, 2.0])", "[1.0, 2.1]", 1e-5, False),+ (+ "print({'a': 1.0, 'b': 2.0})",+ "{'a': 1.000001, 'b': 2.0}",+ 1e-5,+ True,+ ),+ ("print({'a': 1.0, 'b': 2.0})", "{'a': 1.1, 'b': 2.0}", 1e-5, False),+ ("print({1.0, 2.0})", "{1.0, 2.000001}", 1e-5, True),+ ("print({1.0, 2.0})", "{1.0, 2.1}", 1e-5, False),+ # This test should fail with the simplified all(any(...)) logic+ ("print({1.0, 1.000001})", "{1.000002, 2.0}", 1e-5, False),+ ],+)+async def test_verify_with_float_tolerance(+ script, expected_output, tol, should_pass+):+ r"""Test verifier end-to-end with float tolerance"""+ verifier = PythonVerifier(float_tolerance=tol)+ await verifier.setup(uv=True)++ result = await verifier.verify(+ solution=script, reference_answer=expected_output+ )++ if should_pass:+ assert (+ result.status == VerificationOutcome.SUCCESS+ ), f"Expected success, got {result}"+ else:+ assert (+ result.status == VerificationOutcome.FAILURE+ ), f"Expected failure, got {result}"++ await verifier.cleanup()