Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

update: add float type comparison (#2133)

Rishabh2fc573496434
diff --git a/camel/verifiers/python_verifier.py b/camel/verifiers/python_verifier.py
index 94d20759f..8181f8342 100644
--- a/camel/verifiers/python_verifier.py
+++ b/camel/verifiers/python_verifier.py
@@ -20,7 +20,7 @@ import subprocess
import sys
import tempfile
import venv
-from typing import List, Optional, Tuple
+from typing import Any, List, Optional, Tuple
from camel.extractors.base import BaseExtractor
from camel.logger import get_logger
@@ -51,6 +51,7 @@ class PythonVerifier(BaseVerifier):
extractor: Optional[BaseExtractor] = None,
timeout: Optional[float] = 30.0,
required_packages: Optional[List[str]] = None,
+ float_tolerance: Optional[float] = 1e-05,
**kwargs,
):
r"""Initializes the PythonVerifier.
@@ -63,11 +64,14 @@ class PythonVerifier(BaseVerifier):
required_packages (Optional[List[str]], optional): A list of
packages to install in the virtual environment.
(default: :obj:`None`)
+ float_tolerance (Optional[float], optional): The tolerance for
+ floating point comparisons. (default: :obj:`1e-05`)
"""
# TODO: Use CAMEL's Interpreter to execute the code
super().__init__(extractor=extractor, timeout=timeout, **kwargs)
self.venv_path: Optional[str] = None
self.required_packages = required_packages or []
+ self.float_tolerance = float_tolerance
if os.name == 'nt': # Windows
self.bin_dir = 'Scripts'
@@ -272,7 +276,13 @@ class PythonVerifier(BaseVerifier):
result="",
error_message=f"Ground truth evaluation error: {e}",
)
- if sol_val == gt_val:
+
+ if self.float_tolerance is not None:
+ equal = self._is_equal_with_tolerance(sol_val, gt_val)
+ else:
+ equal = sol_val == gt_val
+
+ if equal:
return VerificationResult(
status=VerificationOutcome.SUCCESS,
result=str(sol_val),
@@ -281,9 +291,18 @@ class PythonVerifier(BaseVerifier):
return VerificationResult(
status=VerificationOutcome.FAILURE,
result=str(sol_val),
- error_message="Output mismatch: "
- f"{sol_val} != {gt_val}",
+ error_message=(
+ "Values not equal"
+ + (
+ " (with float tolerance "
+ f"{self.float_tolerance})"
+ if self.float_tolerance is not None
+ else ""
+ )
+ + f": {sol_val} != {gt_val}"
+ ),
)
+
else:
return VerificationResult(
status=VerificationOutcome.SUCCESS,
@@ -333,17 +352,21 @@ class PythonVerifier(BaseVerifier):
error_message="Ground truth evaluation error:"
f"{e}",
)
- if sol_val == gt_val:
+ if self.float_tolerance is not None:
+ equal = self._is_equal_with_tolerance(sol_val, gt_val)
+ else:
+ equal = sol_val == gt_val
+
+ if equal:
return VerificationResult(
- status=VerificationOutcome.SUCCESS,
- result=sol_out,
+ status=VerificationOutcome.SUCCESS, result=sol_out
)
else:
return VerificationResult(
status=VerificationOutcome.FAILURE,
result=sol_out,
- error_message="Output mismatch: "
- f"{sol_val} != {gt_val}",
+ error_message=f"Output mismatch: {sol_val} "
+ f"!= {gt_val}",
)
else:
# Fallback: string comparison
@@ -456,3 +479,64 @@ class PythonVerifier(BaseVerifier):
return True
except Exception:
return False
+
+ def _is_equal_with_tolerance(self, a: Any, b: Any) -> bool:
+ r"""Compares two Python objects for equality with optional float
+ tolerance.
+
+ This method recursively compares nested structures (lists, tuples,
+ sets, and dictionaries) and applies floating point tolerance when
+ comparing numerical values. If no float tolerance is set, a runtime
+ error is raised.
+
+ Args:
+ a (Any): First value to compare.
+ b (Any): Second value to compare.
+
+ Returns:
+ bool: True if the values are considered equal within the
+ specified float tolerance; False otherwise.
+
+ Raises:
+ RuntimeError: If float tolerance is not set (i.e., None).
+ """
+ if self.float_tolerance is None:
+ raise RuntimeError(
+ "Can't compare with tolerance if tolerance is None."
+ )
+ if isinstance(a, (int, float)) and isinstance(b, (int, float)):
+ return abs(float(a) - float(b)) <= self.float_tolerance
+ if isinstance(a, list) and isinstance(b, list):
+ return len(a) == len(b) and all(
+ self._is_equal_with_tolerance(x, y) for x, y in zip(a, b)
+ )
+ if isinstance(a, tuple) and isinstance(b, tuple):
+ return len(a) == len(b) and all(
+ self._is_equal_with_tolerance(x, y) for x, y in zip(a, b)
+ )
+ if isinstance(a, set) and isinstance(b, set):
+ if len(a) != len(b):
+ return False
+ # Need to check both directions to ensure proper matching
+ # Create a copy of b to track matched elements
+ b_copy = list(b)
+ for x in a:
+ found_match = False
+ for i, y in enumerate(b_copy):
+ if self._is_equal_with_tolerance(x, y):
+ found_match = True
+ # Remove the matched element to prevent double-matching
+ b_copy.pop(i)
+ break
+ if not found_match:
+ return False
+ return True
+ if isinstance(a, dict) and isinstance(b, dict):
+ if set(a.keys()) != set(b.keys()):
+ return False
+ return all(self._is_equal_with_tolerance(a[k], b[k]) for k in a)
+ logger.warning(
+ f"Falling back to simple comparison without "
+ f"tolerance for {a} and {b}."
+ )
+ return a == b # fallback
diff --git a/test/verifiers/test_python_verifier.py b/test/verifiers/test_python_verifier.py
index 394200fc8..2b548869a 100644
--- a/test/verifiers/test_python_verifier.py
+++ b/test/verifiers/test_python_verifier.py
@@ -151,3 +151,91 @@ async def test_python_verifier_with_numpy(python_verifier):
assert result.status == VerificationOutcome.SUCCESS
assert "[1 2 3]" in result.result
await python_verifier.cleanup()
+
+
+ "a,b,tol,expected",
+ [
+ (0.123456, 0.123457, 1e-5, True),
+ (0.123456, 0.123467, 1e-5, False),
+ ([1.0, 2.0], [1.0, 2.000001], 1e-5, True),
+ ([1.0, 2.0], [1.0, 2.1], 1e-5, False),
+ ({'a': 1.0, 'b': 2.0}, {'a': 1.0000001, 'b': 2.0}, 1e-5, True),
+ ({'a': 1.0, 'b': 2.0}, {'a': 1.0001, 'b': 2.0}, 1e-5, False),
+ ((1.0, 2.0), (1.0, 2.000001), 1e-5, True),
+ ((1.0, 2.0), (1.0, 2.01), 1e-5, False),
+ ({1.0, 2.0}, {2.000001, 1.0}, 1e-5, True),
+ ({1.0, 2.0}, {2.1, 1.0}, 1e-5, False),
+ ({1.0, 1.000001}, {1.000002, 2.0}, 1e-5, False),
+ ({"x": [1.0, 2.0]}, {"x": [1.000001, 2.0]}, 1e-5, True),
+ ({"x": [1.0, 2.0]}, {"x": [1.1, 2.0]}, 1e-5, False),
+ ],
+)
+def test_is_equal_with_tolerance(a, b, tol, expected):
+ r"""Test the function calculating whether two floats are equal
+ given tolerance"""
+ verifier = PythonVerifier(float_tolerance=tol)
+ assert verifier._is_equal_with_tolerance(a, b) == expected
+
+
+ "script,expected_output,tol,should_pass",
+ [
+ # Float scalar
+ ("0.123456", "0.123457", 1e-5, True),
+ ("0.123456", "0.123467", 1e-5, False),
+ # Lists
+ ("[1.0, 2.0]", "[1.0, 2.000001]", 1e-5, True),
+ ("[1.0, 2.0]", "[1.0, 2.1]", 1e-5, False),
+ # Dicts
+ ("{'a': 1.0, 'b': 2.0}", "{'a': 1.000001, 'b': 2.0}", 1e-5, True),
+ ("{'a': 1.0, 'b': 2.0}", "{'a': 1.1, 'b': 2.0}", 1e-5, False),
+ # Sets
+ ("{1.0, 2.0}", "{1.0, 2.000001}", 1e-5, True),
+ ("{1.0, 2.0}", "{1.0, 2.1}", 1e-5, False),
+ # Tuples
+ ("(1.0, 2.0)", "(1.0, 2.000001)", 1e-5, True),
+ ("(1.0, 2.0)", "(1.0, 2.1)", 1e-5, False),
+ # Mixed nesting
+ ("{'a': [1.0, 2.0]}", "{'a': [1.000001, 2.0]}", 1e-5, True),
+ ("{'a': [1.0, 2.0]}", "{'a': [1.1, 2.0]}", 1e-5, False),
+ # Same as above, but as code blocks
+ ("print(0.123456)", "0.123457", 1e-5, True),
+ ("print(0.123456)", "0.123467", 1e-5, False),
+ ("print([1.0, 2.0])", "[1.0, 2.000001]", 1e-5, True),
+ ("print([1.0, 2.0])", "[1.0, 2.1]", 1e-5, False),
+ (
+ "print({'a': 1.0, 'b': 2.0})",
+ "{'a': 1.000001, 'b': 2.0}",
+ 1e-5,
+ True,
+ ),
+ ("print({'a': 1.0, 'b': 2.0})", "{'a': 1.1, 'b': 2.0}", 1e-5, False),
+ ("print({1.0, 2.0})", "{1.0, 2.000001}", 1e-5, True),
+ ("print({1.0, 2.0})", "{1.0, 2.1}", 1e-5, False),
+ # This test should fail with the simplified all(any(...)) logic
+ ("print({1.0, 1.000001})", "{1.000002, 2.0}", 1e-5, False),
+ ],
+)
+async def test_verify_with_float_tolerance(
+ script, expected_output, tol, should_pass
+):
+ r"""Test verifier end-to-end with float tolerance"""
+ verifier = PythonVerifier(float_tolerance=tol)
+ await verifier.setup(uv=True)
+
+ result = await verifier.verify(
+ solution=script, reference_answer=expected_output
+ )
+
+ if should_pass:
+ assert (
+ result.status == VerificationOutcome.SUCCESS
+ ), f"Expected success, got {result}"
+ else:
+ assert (
+ result.status == VerificationOutcome.FAILURE
+ ), f"Expected failure, got {result}"
+
+ await verifier.cleanup()