🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: add math verify as verifier (#2044)
Rishabhce54002e73d0
diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.pyindex 36003c94f..3e49e3e13 100644--- a/camel/verifiers/__init__.py+++ b/camel/verifiers/__init__.py@@ -12,6 +12,7 @@# limitations under the License.# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========from .base import BaseVerifier+from .math_verifier import MathVerifierfrom .models import VerificationOutcomefrom .python_verifier import PythonVerifier@@ -19,4 +20,5 @@ __all__ = ["BaseVerifier","VerificationOutcome","PythonVerifier",+ "MathVerifier",]diff --git a/camel/verifiers/math_verifier.py b/camel/verifiers/math_verifier.pynew file mode 100644index 000000000..db3674648--- /dev/null+++ b/camel/verifiers/math_verifier.py@@ -0,0 +1,145 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++from typing import Optional++from camel.extractors.base import BaseExtractor+from camel.logger import get_logger+from camel.verifiers import BaseVerifier+from camel.verifiers.models import VerificationOutcome, VerificationResult++logger = get_logger(__name__)+++class MathVerifier(BaseVerifier):+ r"""Verifier for mathematical expressions using Math-Verify.++ Features:+ - Supports LaTeX and plain mathematical expressions+ - Handles complex numbers, matrices, and sets+ - Configurable precision for floating-point comparisons+ - Comprehensive error handling and logging+ """++ def __init__(+ self,+ extractor: Optional[BaseExtractor] = None,+ timeout: Optional[float] = 30.0,+ float_rounding: int = 6,+ numeric_precision: int = 15,+ **kwargs,+ ):+ r"""Initializes the MathVerifier.++ Args:+ extractor (Optional[BaseExtractor], optional): The extractor to use+ for extracting code from the solution. (default: :obj:`None`)+ timeout (Optional[float], optional): The execution timeout in+ seconds. (default: :obj:`30.0`)+ float_rounding (int, optional): The number of decimal places to+ round floating-point numbers. (default: :obj:`6`)+ numeric_precision (int, optional): The numeric precision for+ floating-point comparisons. (default: :obj:`15`)+ """+ super().__init__(extractor=extractor, timeout=timeout, **kwargs)+ self.float_rounding = float_rounding+ self.numeric_precision = numeric_precision++ async def _setup(self, **kwargs) -> None:+ r"""No special setup needed for math verification."""+ pass++ async def _cleanup(self) -> None:+ r"""No cleanup needed for math verification."""+ pass++ async def _verify_implementation(+ self, solution: str, ground_truth: Optional[str]+ ) -> VerificationResult:+ r"""Verify mathematical expressions using Math-Verify.++ Args:+ solution: The solution to verify+ ground_truth: The expected answer to compare against++ Returns:+ VerificationResult containing the verification status and details+ """+ from math_verify import parse, verify+ from math_verify.parser import (+ ExprExtractionConfig,+ LatexExtractionConfig,+ )++ if ground_truth is None:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=(+ "Ground truth is required for " "mathematical verification"+ ),+ )++ try:+ # Parse both expressions with LaTeX and plain expression support+ parsed_ground_truth = parse(+ ground_truth,+ extraction_config=[+ LatexExtractionConfig(boxed_match_priority=0),+ ExprExtractionConfig(),+ ],+ )+ parsed_solution = parse(+ solution,+ extraction_config=[+ LatexExtractionConfig(),+ ExprExtractionConfig(),+ ],+ )++ if not parsed_ground_truth or not parsed_solution:+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message="Failed to parse expressions",+ )++ # Order matters! ground_truth must be first argument+ is_correct = verify(+ parsed_ground_truth,+ parsed_solution,+ float_rounding=self.float_rounding,+ numeric_precision=self.numeric_precision,+ )++ if is_correct:+ logger.debug("Mathematical verification succeeded")+ return VerificationResult(+ status=VerificationOutcome.SUCCESS, result=solution+ )+ else:+ logger.debug("Mathematical verification failed")+ return VerificationResult(+ status=VerificationOutcome.FAILURE,+ result=solution,+ error_message="Solution does not match ground truth",+ )++ except Exception as error:+ logger.error(f"Mathematical verification error: {error!s}")+ return VerificationResult(+ status=VerificationOutcome.ERROR,+ result="",+ error_message=f"Mathematical verification error: {error!s}",+ )diff --git a/examples/verifier/math_verifier_example.py b/examples/verifier/math_verifier_example.pynew file mode 100644index 000000000..f5cf9e4c7--- /dev/null+++ b/examples/verifier/math_verifier_example.py@@ -0,0 +1,91 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import asyncio++from camel.logger import get_logger+from camel.verifiers import MathVerifier++logger = get_logger(__name__)++# Initialize verifier with configuration+verifier = MathVerifier(float_rounding=6, numeric_precision=15)+++async def main():+ r"""Run test cases demonstrating different verification scenarios."""++ print("\nStarting Math Verifier Examples\n")+ await verifier.setup()++ try:+ # Test case 1: Basic numerical equivalence (should succeed)+ print("=== Test 1: Basic Numerical ===")+ result = await verifier.verify(solution="0.333333", ground_truth="1/3")+ print("Input: 0.333333 ≈ 1/3")+ print(f"Result: {result.status}")+ print(+ f"err: {result.error_message if result.error_message else 'None'}"+ )++ # Test case 2: LaTeX expressions (should succeed)+ print("=== Test 2: LaTeX Expression ===")+ result = await verifier.verify(+ solution=r"$\frac{1}{2}$", ground_truth=r"0.5"+ )+ print("Input: \\frac{1}{2} = 0.5")+ print(f"Result: {result.status}")+ print(+ f"err: {result.error_message if result.error_message else 'None'}"+ )++ # Test case 3: Deliberate mismatch (should fail)+ print("=== Test 3: Expected Failure ===")+ result = await verifier.verify(solution="0.5", ground_truth="0.3333")+ print("Input: 0.5 ≠ 0.3333")+ print(f"Result: {result.status}")+ print(+ f"err: {result.error_message if result.error_message else 'None'}"+ )++ finally:+ await verifier.cleanup()+ print("Math Verifier Examples Completed")+++if __name__ == "__main__":+ asyncio.run(main())++"""+===============================================================================+Starting Math Verifier Examples++=== Test 1: Basic Numerical ===+Input: 0.333333 ≈ 1/3+Result: VerificationOutcome.SUCCESS+err: None++=== Test 2: LaTeX Expression ===+Input: \frac{1}{2} = 0.5+Result: VerificationOutcome.SUCCESS+err: None++=== Test 3: Expected Failure ===+Input: 0.5 ≠ 0.3333+Result: VerificationOutcome.FAILURE+err: Solution does not match ground truth++Math Verifier Examples Completed+===============================================================================+"""diff --git a/pyproject.toml b/pyproject.tomlindex 23e1192ad..290bbf29b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -137,6 +137,7 @@ data_tools = ["aiosqlite>=0.20.0,<0.21","stripe>=11.3.0,<12","networkx>=3.4.2,<4",+ "math-verify>=0.7.0,<0.8",]research_tools = ["arxiv>=2.1.3,<3",@@ -330,7 +331,8 @@ all = ["fpdf>=1.7.2","mcp>=1.3.0","typer>=0.15.2",- "mem0ai>=0.1.67"+ "mem0ai>=0.1.67",+ "math-verify>=0.7.0,<0.8",][project.urls]@@ -509,5 +511,6 @@ module = ["mcp","typer","mem0",+ "math_verify.*"]ignore_missing_imports = truediff --git a/tests/datasets/test_self_instruct_generator.py b/test/datasets/test_self_instruct_generator.pysimilarity index 100%rename from tests/datasets/test_self_instruct_generator.pyrename to test/datasets/test_self_instruct_generator.pydiff --git a/test/verifiers/test_math_verifier.py b/test/verifiers/test_math_verifier.pynew file mode 100644index 000000000..2cf560804--- /dev/null+++ b/test/verifiers/test_math_verifier.py@@ -0,0 +1,79 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import pytest++from camel.verifiers import MathVerifier+from camel.verifiers.models import VerificationOutcome+++async def test_math_verifier_basic():+ verifier = MathVerifier()+ try:+ # Test basic numerical equivalence+ result = await verifier.verify("0.5", "1/2")+ assert result.status == VerificationOutcome.SUCCESS+ assert result.error_message is None++ # Test LaTeX expressions+ result = await verifier.verify(r"$\frac{1}{2}$", "0.5")+ assert result.status == VerificationOutcome.SUCCESS+ assert result.error_message is None++ # Test deliberate mismatch+ result = await verifier.verify("0.5", "0.3333")+ assert result.status == VerificationOutcome.FAILURE+ assert result.error_message is not None++ finally:+ await verifier.cleanup()+++async def test_math_verifier_complex():+ verifier = MathVerifier()+ try:+ # Test complex expressions+ result = await verifier.verify(+ r"$\frac{1}{2} + \frac{1}{3}$", r"$\frac{5}{6}$"+ )+ assert result.status == VerificationOutcome.SUCCESS+ assert result.error_message is None++ # Test algebraic expressions+ result = await verifier.verify(r"$x^2 + 2x + 1$", r"$(x + 1)^2$")+ assert result.status == VerificationOutcome.SUCCESS+ assert result.error_message is None++ finally:+ await verifier.cleanup()+++async def test_math_verifier_precision():+ verifier = MathVerifier(float_rounding=3)+ try:+ # Test with custom precision+ result = await verifier.verify("0.3333", "1/3")+ assert result.status == VerificationOutcome.SUCCESS+ assert result.error_message is None++ # Test precision failure+ result = await verifier.verify("0.33", "1/3")+ assert result.status == VerificationOutcome.FAILURE+ assert result.error_message is not None++ finally:+ await verifier.cleanup()diff --git a/uv.lock b/uv.lockindex 674869741..f31500062 100644--- a/uv.lock+++ b/uv.lock@@ -231,6 +231,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/76/74/5d90ad14d55fbe3f9c474fdcb6e34b4bed99e3be8efac98734a5ddce88c1/anthropic-0.49.0-py3-none-any.whl", hash = "sha256:bbc17ad4e7094988d2fa86b87753ded8dce12498f4b85fe5810f208f454a8375", size = 243368 },]+[[package]]+name = "antlr4-python3-runtime"+version = "4.13.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/33/5f/2cdf6f7aca3b20d3f316e9f505292e1f256a32089bd702034c29ebde6242/antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916", size = 117467 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/89/03/a851e84fcbb85214dc637b6378121ef9a0dd61b4c65264675d8a5c9b1ae7/antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8", size = 144462 },+]+[[package]]name = "anyio"version = "4.8.0"@@ -585,6 +594,7 @@ all = [{ name = "jupyter-client" },{ name = "linkup-sdk" },{ name = "litellm" },+ { name = "math-verify" },{ name = "mcp" },{ name = "mem0ai" },{ name = "mistralai" },@@ -665,6 +675,7 @@ data-tools = [{ name = "aiosqlite" },{ name = "datacommons" },{ name = "datacommons-pandas" },+ { name = "math-verify" },{ name = "networkx" },{ name = "numpy" },{ name = "pandas" },@@ -942,6 +953,8 @@ requires-dist = [{ name = "linkup-sdk", marker = "extra == 'web-tools'", specifier = ">=0.2.1,<0.3" },{ name = "litellm", marker = "extra == 'all'", specifier = ">=1.38.1,<2" },{ name = "litellm", marker = "extra == 'model-platforms'", specifier = ">=1.38.1,<2" },+ { name = "math-verify", marker = "extra == 'all'", specifier = ">=0.7.0,<0.8" },+ { name = "math-verify", marker = "extra == 'data-tools'", specifier = ">=0.7.0,<0.8" },{ name = "mcp", marker = "extra == 'all'", specifier = ">=1.3.0" },{ name = "mcp", marker = "extra == 'dev-tools'", specifier = ">=1.3.0" },{ name = "mcp-server-fetch", marker = "extra == 'owl'", specifier = "==2025.1.17" },@@ -3072,6 +3085,19 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036 },]+[[package]]+name = "latex2sympy2-extended"+version = "1.10.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "antlr4-python3-runtime" },+ { name = "sympy" },+]+sdist = { url = "https://files.pythonhosted.org/packages/14/8d/de5cc53dcf8a98396094b81a6478eb7234ef2e5c5fa4900dcc335d3b7fc2/latex2sympy2_extended-1.10.1.tar.gz", hash = "sha256:e365f0cb8807296c9d1f1661be926859bebd922a6139ce4be1e6572ca183ca57", size = 205808 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/6d/eb/a5e8b06b924b4149cf498e1598116bad1e91ab23046c2dfc2c498154d393/latex2sympy2_extended-1.10.1-py3-none-any.whl", hash = "sha256:917a23e8f3b6edea88a56978fbbe87ed9fca4197f8277646be57b4660710347c", size = 207460 },+]+[[package]]name = "lazy-object-proxy"version = "1.10.0"@@ -3277,6 +3303,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878 },]+[[package]]+name = "math-verify"+version = "0.7.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "latex2sympy2-extended" },+]+sdist = { url = "https://files.pythonhosted.org/packages/ab/59/7daf53e52593af0f7f455ea97c91dc34682f1c2e72f493490259a640836c/math_verify-0.7.0.tar.gz", hash = "sha256:b7459c7b9f1c503b7df32566a95f1e6a33a6cad8cbb759eaa6f100d401112024", size = 57582 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/44/c5/e5de2e29f9db4c92956ce167e1aa534b1e1770ddc0b17600257cf5f8bd50/math_verify-0.7.0-py3-none-any.whl", hash = "sha256:bbdd491b511f6ceef27c5e08543affa1735807b6e3abb6f005bf493dc0eb485b", size = 28677 },+]+[[package]]name = "matplotlib"version = "3.10.1"