Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: add math verify as verifier (#2044)

Rishabhce54002e73d0
diff --git a/camel/verifiers/__init__.py b/camel/verifiers/__init__.py
index 36003c94f..3e49e3e13 100644
--- a/camel/verifiers/__init__.py
+++ b/camel/verifiers/__init__.py
@@ -12,6 +12,7 @@
# limitations under the License.
# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
from .base import BaseVerifier
+from .math_verifier import MathVerifier
from .models import VerificationOutcome
from .python_verifier import PythonVerifier
@@ -19,4 +20,5 @@ __all__ = [
"BaseVerifier",
"VerificationOutcome",
"PythonVerifier",
+ "MathVerifier",
]
diff --git a/camel/verifiers/math_verifier.py b/camel/verifiers/math_verifier.py
new file mode 100644
index 000000000..db3674648
--- /dev/null
+++ b/camel/verifiers/math_verifier.py
@@ -0,0 +1,145 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import Optional
+
+from camel.extractors.base import BaseExtractor
+from camel.logger import get_logger
+from camel.verifiers import BaseVerifier
+from camel.verifiers.models import VerificationOutcome, VerificationResult
+
+logger = get_logger(__name__)
+
+
+class MathVerifier(BaseVerifier):
+ r"""Verifier for mathematical expressions using Math-Verify.
+
+ Features:
+ - Supports LaTeX and plain mathematical expressions
+ - Handles complex numbers, matrices, and sets
+ - Configurable precision for floating-point comparisons
+ - Comprehensive error handling and logging
+ """
+
+ def __init__(
+ self,
+ extractor: Optional[BaseExtractor] = None,
+ timeout: Optional[float] = 30.0,
+ float_rounding: int = 6,
+ numeric_precision: int = 15,
+ **kwargs,
+ ):
+ r"""Initializes the MathVerifier.
+
+ Args:
+ extractor (Optional[BaseExtractor], optional): The extractor to use
+ for extracting code from the solution. (default: :obj:`None`)
+ timeout (Optional[float], optional): The execution timeout in
+ seconds. (default: :obj:`30.0`)
+ float_rounding (int, optional): The number of decimal places to
+ round floating-point numbers. (default: :obj:`6`)
+ numeric_precision (int, optional): The numeric precision for
+ floating-point comparisons. (default: :obj:`15`)
+ """
+ super().__init__(extractor=extractor, timeout=timeout, **kwargs)
+ self.float_rounding = float_rounding
+ self.numeric_precision = numeric_precision
+
+ async def _setup(self, **kwargs) -> None:
+ r"""No special setup needed for math verification."""
+ pass
+
+ async def _cleanup(self) -> None:
+ r"""No cleanup needed for math verification."""
+ pass
+
+ async def _verify_implementation(
+ self, solution: str, ground_truth: Optional[str]
+ ) -> VerificationResult:
+ r"""Verify mathematical expressions using Math-Verify.
+
+ Args:
+ solution: The solution to verify
+ ground_truth: The expected answer to compare against
+
+ Returns:
+ VerificationResult containing the verification status and details
+ """
+ from math_verify import parse, verify
+ from math_verify.parser import (
+ ExprExtractionConfig,
+ LatexExtractionConfig,
+ )
+
+ if ground_truth is None:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=(
+ "Ground truth is required for " "mathematical verification"
+ ),
+ )
+
+ try:
+ # Parse both expressions with LaTeX and plain expression support
+ parsed_ground_truth = parse(
+ ground_truth,
+ extraction_config=[
+ LatexExtractionConfig(boxed_match_priority=0),
+ ExprExtractionConfig(),
+ ],
+ )
+ parsed_solution = parse(
+ solution,
+ extraction_config=[
+ LatexExtractionConfig(),
+ ExprExtractionConfig(),
+ ],
+ )
+
+ if not parsed_ground_truth or not parsed_solution:
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message="Failed to parse expressions",
+ )
+
+ # Order matters! ground_truth must be first argument
+ is_correct = verify(
+ parsed_ground_truth,
+ parsed_solution,
+ float_rounding=self.float_rounding,
+ numeric_precision=self.numeric_precision,
+ )
+
+ if is_correct:
+ logger.debug("Mathematical verification succeeded")
+ return VerificationResult(
+ status=VerificationOutcome.SUCCESS, result=solution
+ )
+ else:
+ logger.debug("Mathematical verification failed")
+ return VerificationResult(
+ status=VerificationOutcome.FAILURE,
+ result=solution,
+ error_message="Solution does not match ground truth",
+ )
+
+ except Exception as error:
+ logger.error(f"Mathematical verification error: {error!s}")
+ return VerificationResult(
+ status=VerificationOutcome.ERROR,
+ result="",
+ error_message=f"Mathematical verification error: {error!s}",
+ )
diff --git a/examples/verifier/math_verifier_example.py b/examples/verifier/math_verifier_example.py
new file mode 100644
index 000000000..f5cf9e4c7
--- /dev/null
+++ b/examples/verifier/math_verifier_example.py
@@ -0,0 +1,91 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+
+from camel.logger import get_logger
+from camel.verifiers import MathVerifier
+
+logger = get_logger(__name__)
+
+# Initialize verifier with configuration
+verifier = MathVerifier(float_rounding=6, numeric_precision=15)
+
+
+async def main():
+ r"""Run test cases demonstrating different verification scenarios."""
+
+ print("\nStarting Math Verifier Examples\n")
+ await verifier.setup()
+
+ try:
+ # Test case 1: Basic numerical equivalence (should succeed)
+ print("=== Test 1: Basic Numerical ===")
+ result = await verifier.verify(solution="0.333333", ground_truth="1/3")
+ print("Input: 0.333333 ≈ 1/3")
+ print(f"Result: {result.status}")
+ print(
+ f"err: {result.error_message if result.error_message else 'None'}"
+ )
+
+ # Test case 2: LaTeX expressions (should succeed)
+ print("=== Test 2: LaTeX Expression ===")
+ result = await verifier.verify(
+ solution=r"$\frac{1}{2}$", ground_truth=r"0.5"
+ )
+ print("Input: \\frac{1}{2} = 0.5")
+ print(f"Result: {result.status}")
+ print(
+ f"err: {result.error_message if result.error_message else 'None'}"
+ )
+
+ # Test case 3: Deliberate mismatch (should fail)
+ print("=== Test 3: Expected Failure ===")
+ result = await verifier.verify(solution="0.5", ground_truth="0.3333")
+ print("Input: 0.5 ≠ 0.3333")
+ print(f"Result: {result.status}")
+ print(
+ f"err: {result.error_message if result.error_message else 'None'}"
+ )
+
+ finally:
+ await verifier.cleanup()
+ print("Math Verifier Examples Completed")
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
+
+"""
+===============================================================================
+Starting Math Verifier Examples
+
+=== Test 1: Basic Numerical ===
+Input: 0.333333 ≈ 1/3
+Result: VerificationOutcome.SUCCESS
+err: None
+
+=== Test 2: LaTeX Expression ===
+Input: \frac{1}{2} = 0.5
+Result: VerificationOutcome.SUCCESS
+err: None
+
+=== Test 3: Expected Failure ===
+Input: 0.5 ≠ 0.3333
+Result: VerificationOutcome.FAILURE
+err: Solution does not match ground truth
+
+Math Verifier Examples Completed
+===============================================================================
+"""
diff --git a/pyproject.toml b/pyproject.toml
index 23e1192ad..290bbf29b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -137,6 +137,7 @@ data_tools = [
"aiosqlite>=0.20.0,<0.21",
"stripe>=11.3.0,<12",
"networkx>=3.4.2,<4",
+ "math-verify>=0.7.0,<0.8",
]
research_tools = [
"arxiv>=2.1.3,<3",
@@ -330,7 +331,8 @@ all = [
"fpdf>=1.7.2",
"mcp>=1.3.0",
"typer>=0.15.2",
- "mem0ai>=0.1.67"
+ "mem0ai>=0.1.67",
+ "math-verify>=0.7.0,<0.8",
]
[project.urls]
@@ -509,5 +511,6 @@ module = [
"mcp",
"typer",
"mem0",
+ "math_verify.*"
]
ignore_missing_imports = true
diff --git a/tests/datasets/test_self_instruct_generator.py b/test/datasets/test_self_instruct_generator.py
similarity index 100%
rename from tests/datasets/test_self_instruct_generator.py
rename to test/datasets/test_self_instruct_generator.py
diff --git a/test/verifiers/test_math_verifier.py b/test/verifiers/test_math_verifier.py
new file mode 100644
index 000000000..2cf560804
--- /dev/null
+++ b/test/verifiers/test_math_verifier.py
@@ -0,0 +1,79 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import pytest
+
+from camel.verifiers import MathVerifier
+from camel.verifiers.models import VerificationOutcome
+
+
+async def test_math_verifier_basic():
+ verifier = MathVerifier()
+ try:
+ # Test basic numerical equivalence
+ result = await verifier.verify("0.5", "1/2")
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.error_message is None
+
+ # Test LaTeX expressions
+ result = await verifier.verify(r"$\frac{1}{2}$", "0.5")
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.error_message is None
+
+ # Test deliberate mismatch
+ result = await verifier.verify("0.5", "0.3333")
+ assert result.status == VerificationOutcome.FAILURE
+ assert result.error_message is not None
+
+ finally:
+ await verifier.cleanup()
+
+
+async def test_math_verifier_complex():
+ verifier = MathVerifier()
+ try:
+ # Test complex expressions
+ result = await verifier.verify(
+ r"$\frac{1}{2} + \frac{1}{3}$", r"$\frac{5}{6}$"
+ )
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.error_message is None
+
+ # Test algebraic expressions
+ result = await verifier.verify(r"$x^2 + 2x + 1$", r"$(x + 1)^2$")
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.error_message is None
+
+ finally:
+ await verifier.cleanup()
+
+
+async def test_math_verifier_precision():
+ verifier = MathVerifier(float_rounding=3)
+ try:
+ # Test with custom precision
+ result = await verifier.verify("0.3333", "1/3")
+ assert result.status == VerificationOutcome.SUCCESS
+ assert result.error_message is None
+
+ # Test precision failure
+ result = await verifier.verify("0.33", "1/3")
+ assert result.status == VerificationOutcome.FAILURE
+ assert result.error_message is not None
+
+ finally:
+ await verifier.cleanup()
diff --git a/uv.lock b/uv.lock
index 674869741..f31500062 100644
--- a/uv.lock
+++ b/uv.lock
@@ -231,6 +231,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/76/74/5d90ad14d55fbe3f9c474fdcb6e34b4bed99e3be8efac98734a5ddce88c1/anthropic-0.49.0-py3-none-any.whl", hash = "sha256:bbc17ad4e7094988d2fa86b87753ded8dce12498f4b85fe5810f208f454a8375", size = 243368 },
]
+[[package]]
+name = "antlr4-python3-runtime"
+version = "4.13.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/33/5f/2cdf6f7aca3b20d3f316e9f505292e1f256a32089bd702034c29ebde6242/antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916", size = 117467 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/89/03/a851e84fcbb85214dc637b6378121ef9a0dd61b4c65264675d8a5c9b1ae7/antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8", size = 144462 },
+]
+
[[package]]
name = "anyio"
version = "4.8.0"
@@ -585,6 +594,7 @@ all = [
{ name = "jupyter-client" },
{ name = "linkup-sdk" },
{ name = "litellm" },
+ { name = "math-verify" },
{ name = "mcp" },
{ name = "mem0ai" },
{ name = "mistralai" },
@@ -665,6 +675,7 @@ data-tools = [
{ name = "aiosqlite" },
{ name = "datacommons" },
{ name = "datacommons-pandas" },
+ { name = "math-verify" },
{ name = "networkx" },
{ name = "numpy" },
{ name = "pandas" },
@@ -942,6 +953,8 @@ requires-dist = [
{ name = "linkup-sdk", marker = "extra == 'web-tools'", specifier = ">=0.2.1,<0.3" },
{ name = "litellm", marker = "extra == 'all'", specifier = ">=1.38.1,<2" },
{ name = "litellm", marker = "extra == 'model-platforms'", specifier = ">=1.38.1,<2" },
+ { name = "math-verify", marker = "extra == 'all'", specifier = ">=0.7.0,<0.8" },
+ { name = "math-verify", marker = "extra == 'data-tools'", specifier = ">=0.7.0,<0.8" },
{ name = "mcp", marker = "extra == 'all'", specifier = ">=1.3.0" },
{ name = "mcp", marker = "extra == 'dev-tools'", specifier = ">=1.3.0" },
{ name = "mcp-server-fetch", marker = "extra == 'owl'", specifier = "==2025.1.17" },
@@ -3072,6 +3085,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036 },
]
+[[package]]
+name = "latex2sympy2-extended"
+version = "1.10.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "antlr4-python3-runtime" },
+ { name = "sympy" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/14/8d/de5cc53dcf8a98396094b81a6478eb7234ef2e5c5fa4900dcc335d3b7fc2/latex2sympy2_extended-1.10.1.tar.gz", hash = "sha256:e365f0cb8807296c9d1f1661be926859bebd922a6139ce4be1e6572ca183ca57", size = 205808 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6d/eb/a5e8b06b924b4149cf498e1598116bad1e91ab23046c2dfc2c498154d393/latex2sympy2_extended-1.10.1-py3-none-any.whl", hash = "sha256:917a23e8f3b6edea88a56978fbbe87ed9fca4197f8277646be57b4660710347c", size = 207460 },
+]
+
[[package]]
name = "lazy-object-proxy"
version = "1.10.0"
@@ -3277,6 +3303,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/34/75/51952c7b2d3873b44a0028b1bd26a25078c18f92f256608e8d1dc61b39fd/marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c", size = 50878 },
]
+[[package]]
+name = "math-verify"
+version = "0.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "latex2sympy2-extended" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ab/59/7daf53e52593af0f7f455ea97c91dc34682f1c2e72f493490259a640836c/math_verify-0.7.0.tar.gz", hash = "sha256:b7459c7b9f1c503b7df32566a95f1e6a33a6cad8cbb759eaa6f100d401112024", size = 57582 }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/44/c5/e5de2e29f9db4c92956ce167e1aa534b1e1770ddc0b17600257cf5f8bd50/math_verify-0.7.0-py3-none-any.whl", hash = "sha256:bbdd491b511f6ceef27c5e08543affa1735807b6e3abb6f005bf493dc0eb485b", size = 28677 },
+]
+
[[package]]
name = "matplotlib"
version = "3.10.1"