Skip to content

armgit / camel

Public Imported from GitHub
0

🐫 CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

4.3 KB
Raw Blame
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========= Copyright 2023-2026 @ CAMEL-AI.org. All Rights Reserved. =========

import json
from unittest.mock import MagicMock, mock_open, patch

from camel.benchmarks import APIBenchBenchmark

# Mock question, API, eval, and ast data
MOCK_QUESTION_DATA = [
    {
        "question_id": 1,
        "text": "How do I use the HuggingFace API for text classification?",
        "domain": "Natural Language Processing Text Classification",
    },
]
MOCK_API_DATA = [
    {
        "api_call": "huggingface_api_call_example",
        "domain": "Natural Language Processing Text Classification",
    }
]
MOCK_EVAL_DATA = [
    {
        "api_call": "huggingface_api_call_example",
        "eval": "huggingface_api_eval_example",
    }
]
MOCK_AST_DATA = [
    {"type": "ASTNode", "value": "some_ast_1"},
    {"type": "ASTNode", "value": "some_ast_2"},
]


def test_apibench_benchmark():
    # Patch necessary functions and classes to prevent real file system access
    with (
        patch("os.listdir", return_value=["huggingface_train.json"]),
        patch("pathlib.Path.mkdir"),
        patch("pathlib.Path.exists", return_value=True),
        patch("pathlib.Path.is_dir", return_value=True),
        patch("huggingface_hub.snapshot_download"),
        patch("builtins.open", mock_open()) as mock_file,
        patch(
            "camel.benchmarks.apibench.encode_question"
        ) as mock_encode_question,
        patch(
            "camel.benchmarks.apibench.evaluate_response"
        ) as mock_evaluate_response,
        patch.object(APIBenchBenchmark, 'load'),
    ):
        # Initialize APIBenchBenchmark with mock paths within the patch context
        data_dir = "/mock_data_dir"
        save_to = "mock_save_to.json"
        benchmark = APIBenchBenchmark(data_dir=data_dir, save_to=save_to)

        # Mock the encode_question function to return a fixed prompt
        mock_encode_question.return_value = (
            "How do I use the HuggingFace API for text classification?"
        )

        # Mock the evaluate_response function to return expected results
        mock_evaluate_response.return_value = (None, True, False)

        # Mock the agent's step method to return the expected response
        mock_agent = MagicMock()
        mock_agent.step.return_value.msgs = [
            MagicMock(content="HuggingFace API call here")
        ]

        # Manually set mock data for api, eval, and ast
        benchmark._data = {
            "api": MOCK_API_DATA,
            "ast": MOCK_AST_DATA,
            "questions": MOCK_QUESTION_DATA,
            "eval": MOCK_EVAL_DATA,
        }

        # Run the benchmark
        result = benchmark.run(
            agent=mock_agent,
            dataset_name="huggingface",
            randomize=False,
            subset=1,
        )

        # Assert the benchmark loaded the data correctly
        assert len(benchmark._data["questions"]) == 1
        assert (
            benchmark._data["questions"][0]["text"]
            == "How do I use the HuggingFace API for text classification?"
        )

        # Assert the benchmark results
        assert result["total"] == 1
        assert result["correct"] == 1
        assert result["hallucination"] == 0
        assert result["accuracy"] == 1.0
        assert result["hallucination rate"] == 0.0

        # Verify that the results were written correctly to the save_to file
        written_data = json.loads(mock_file().write.call_args[0][0])
        assert (
            written_data["question"]["text"]
            == "How do I use the HuggingFace API for text classification?"
        )
        assert written_data["agent_response"] == "HuggingFace API call here"
        assert written_data["correct"]
        assert not written_data["hallucination"]
        assert written_data["error"] is None