Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

30.0 KB
Raw Blame
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""CPU-only pytest coverage for `auto_round.eval.eval_cli`."""

import os
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest
import torch
import transformers

from auto_round.eval import eval_cli


class TestParseVllmArgs:
    """Tests for `parse_vllm_args`."""

    def test_empty_string_returns_empty_dict(self):
        assert eval_cli.parse_vllm_args("") == {}

    def test_none_returns_empty_dict(self):
        assert eval_cli.parse_vllm_args(None) == {}

    def test_simple_args_are_parsed(self):
        result = eval_cli.parse_vllm_args("tensor_parallel_size=2,gpu_memory_utilization=0.9")
        assert result["tensor_parallel_size"] == 2
        assert result["gpu_memory_utilization"] == 0.9

    def test_leading_dashes_are_stripped(self):
        result = eval_cli.parse_vllm_args("--tensor_parallel_size=2")
        assert result["tensor_parallel_size"] == 2

    def test_boolean_values_are_converted(self):
        result = eval_cli.parse_vllm_args("enable_chunked_prefill=true,disable_log_requests=false")
        assert result["enable_chunked_prefill"] is True
        assert result["disable_log_requests"] is False

    def test_space_separated_args_are_normalized(self):
        result = eval_cli.parse_vllm_args("tensor_parallel_size 2")
        assert result["tensor_parallel_size"] == 2

    def test_unknown_values_are_kept_as_strings(self):
        result = eval_cli.parse_vllm_args("model=facebook/opt-125m")
        assert result["model"] == "facebook/opt-125m"


class TestEvalArgumentParser:
    """Check parser defaults, aliases, and diffusion-specific options."""

    def test_positional_model_default_is_none(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args([])
        assert args.model is None

    def test_positional_model_is_accepted(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(["local-model-path"])
        assert args.model == "local-model-path"

    def test_model_name_default(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args([])
        # No default model: the CLI requires an explicit positional `model` or
        # `--model_name` (enforced by `run_eval`'s assertion).
        assert args.model_name is None

    def test_model_alias_updates_model_name(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(["--model_name", "custom-model"])
        assert args.model_name == "custom-model"

    def test_default_device_map(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args([])
        assert args.device_map == "0"

    def test_default_tasks(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args([])
        assert "mmlu" in args.tasks
        assert "hellaswag" in args.tasks

    def test_tasks_are_overridable(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(["--tasks", "mmlu,wikitext"])
        assert args.tasks == "mmlu,wikitext"

    def test_disable_trust_remote_code_flag(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(["--disable_trust_remote_code"])
        assert args.disable_trust_remote_code is True

    def test_diffusion_args_are_available(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(
            [
                "--prompt",
                "a cat",
                "--metrics",
                "clip-iqa",
                "--guidance_scale",
                "10.0",
                "--num_inference_steps",
                "10",
            ]
        )
        assert args.prompt == "a cat"
        assert args.metrics == "clip-iqa"
        assert args.guidance_scale == 10.0
        assert args.num_inference_steps == 10

    def test_eval_backend_defaults_to_hf(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args([])
        assert args.eval_backend == "hf"

    def test_vllm_args_are_accepted(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(["--vllm_args", "tensor_parallel_size=2,gpu_memory_utilization=0.9"])
        assert args.vllm_args == "tensor_parallel_size=2,gpu_memory_utilization=0.9"

    def test_fewshot_and_generation_args_are_accepted(self):
        parser = eval_cli.EvalArgumentParser()
        args = parser.parse_args(
            [
                "--num-fewshot",
                "3",
                "--eval-gen-kwargs",
                "temperature=0.1",
                "--fewshot-as-multiturn",
            ]
        )
        assert args.num_fewshot == 3
        assert args.eval_gen_kwargs == "temperature=0.1"
        assert args.fewshot_as_multiturn is True


class TestEvalInit:
    """Tests for `_eval_init` task normalization, device resolution, and dtype."""

    def test_cuda_visible_devices_is_set(self):
        with patch.object(eval_cli, "set_cuda_visible_devices") as mock_set, patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", None)
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            tasks, model_args, device_str = eval_cli._eval_init(
                "mmlu,wikitext", "/model", "0", disable_trust_remote_code=False, dtype="auto"
            )
        mock_set.assert_called_once_with("0")
        assert device_str == "cpu"

    def test_tasks_are_split_from_comma_string(self):
        with patch.object(eval_cli, "set_cuda_visible_devices"), patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", None)
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            tasks, _, _ = eval_cli._eval_init(
                "mmlu,wikitext", "/model", "0", disable_trust_remote_code=False, dtype="auto"
            )
        assert tasks == ["mmlu", "wikitext"]

    def test_list_tasks_are_passed_through(self):
        with patch.object(eval_cli, "set_cuda_visible_devices"), patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", None)
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            tasks, _, _ = eval_cli._eval_init(
                ["mmlu", "wikitext"], "/model", "0", disable_trust_remote_code=False, dtype="auto"
            )
        assert tasks == ["mmlu", "wikitext"]

    def test_parallelism_appended_to_model_args(self):
        with patch.object(eval_cli, "set_cuda_visible_devices"), patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", "p")
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            _, model_args, _ = eval_cli._eval_init("mmlu", "/model", "0", disable_trust_remote_code=False, dtype="auto")
        assert ",parallelize=True" in model_args

    def test_dtype_is_resolved_when_not_auto(self):
        with patch.object(eval_cli, "set_cuda_visible_devices"), patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", None)
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            _, model_args, _ = eval_cli._eval_init(
                "mmlu", "/model", "0", disable_trust_remote_code=False, dtype="bfloat16"
            )
        assert "dtype=auto" in model_args

    def test_model_args_contains_trust_remote_code(self):
        with patch.object(eval_cli, "set_cuda_visible_devices"), patch.object(
            eval_cli, "get_device_and_parallelism", return_value=("cpu", None)
        ), patch.object(eval_cli, "get_model_dtype", return_value="auto"):
            _, model_args, _ = eval_cli._eval_init("mmlu", "/model", "0", disable_trust_remote_code=True, dtype="auto")
        assert "trust_remote_code=False" in model_args
        assert ",parallelize=True" not in model_args


class TestEval:
    """Tests for the main `eval` entry point."""

    def test_diffusion_model_path_skips_lm_eval(self, monkeypatch):
        args = SimpleNamespace(model_name="diffusion-model", eval_backend="hf")
        captured = {}

        def fake_diffusion_eval(evaluation_args, pipe):
            captured["args"] = evaluation_args
            captured["pipe"] = pipe

        monkeypatch.setattr(eval_cli, "is_diffusion_model", lambda value: True)
        monkeypatch.setattr("auto_round.utils.diffusion_load_model", lambda value: (object(), object()))
        monkeypatch.setattr("auto_round.eval.evaluation.evaluate_diffusion_model", fake_diffusion_eval)
        eval_cli.eval(args)
        assert captured["args"] is args
        assert captured["pipe"] is not None

    def test_vllm_backend_delegates(self, monkeypatch):
        captured = {}

        def fake_eval_with_vllm(args):
            captured["args"] = args

        args = SimpleNamespace(model_name="vllm-model", eval_backend="vllm")
        monkeypatch.setattr(eval_cli, "is_diffusion_model", lambda value: False)
        monkeypatch.setattr(eval_cli, "eval_with_vllm", fake_eval_with_vllm)
        eval_cli.eval(args)
        assert captured["args"] is args

    def test_gguf_branch_uses_user_model(self, monkeypatch, capsys):
        args = SimpleNamespace(
            model_name="/model.gguf",
            eval_backend="hf",
            eval_bs=None,
            mllm=False,
            eval_model_dtype="auto",
            tasks="mmlu",
            device_map="cpu",
            disable_trust_remote_code=False,
            add_bos_token=False,
            limit=None,
        )

        fake_res = {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}

        monkeypatch.setattr(eval_cli, "is_diffusion_model", lambda value: False)
        monkeypatch.setattr(
            eval_cli,
            "_load_gguf_model_if_needed",
            lambda *args, **kwargs: (object(), object(), True, "model.gguf"),
        )

        monkeypatch.setattr(
            "auto_round.eval.evaluation.simple_evaluate_user_model",
            lambda *args, **kwargs: fake_res,
        )

        eval_cli.eval(args)
        captured = capsys.readouterr()
        assert "evaluation running time=" in captured.out

    def test_mllm_warning_when_auto_batch_size(self, monkeypatch, capsys):
        args = SimpleNamespace(
            model_name="/model",
            eval_backend="hf",
            eval_bs=None,
            mllm=True,
            eval_model_dtype="auto",
            tasks="mmlu",
            device_map="cpu",
            disable_trust_remote_code=False,
            add_bos_token=False,
            limit=None,
        )

        monkeypatch.setattr(eval_cli, "is_diffusion_model", lambda value: False)
        monkeypatch.setattr(
            eval_cli,
            "_load_gguf_model_if_needed",
            lambda *args, **kwargs: (object(), object(), False, None),
        )

        captured_calls = {}

        def fake_simple_evaluate(*args, **kwargs):
            captured_calls["kwargs"] = kwargs
            return {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}

        monkeypatch.setattr("auto_round.eval.evaluation.simple_evaluate", fake_simple_evaluate)

        eval_cli.eval(args)
        captured = capsys.readouterr()
        assert captured_calls["kwargs"]["batch_size"] == 16
        assert "evaluation running time=" in captured.out

    def test_non_mllm_uses_hf_model_name(self, monkeypatch, capsys):
        args = SimpleNamespace(
            model_name="/model",
            eval_backend="hf",
            eval_bs=8,
            mllm=False,
            eval_model_dtype="auto",
            tasks="mmlu",
            device_map="cpu",
            disable_trust_remote_code=False,
            add_bos_token=True,
            limit=None,
        )

        monkeypatch.setattr(eval_cli, "is_diffusion_model", lambda value: False)
        monkeypatch.setattr(
            eval_cli,
            "_load_gguf_model_if_needed",
            lambda *args, **kwargs: (object(), object(), False, None),
        )

        captured_calls = {}

        def fake_simple_evaluate(*args, **kwargs):
            captured_calls["kwargs"] = kwargs
            return {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}

        monkeypatch.setattr("auto_round.eval.evaluation.simple_evaluate", fake_simple_evaluate)

        eval_cli.eval(args)
        captured = capsys.readouterr()
        assert captured_calls["kwargs"]["model"] == "hf"
        assert "add_bos_token=True" in captured_calls["kwargs"]["model_args"]
        assert "evaluation running time=" in captured.out


class TestEvalWithVllm:
    """Tests for the vLLM evaluation backend."""

    def test_tensor_parallel_size_from_device_map(self, monkeypatch):
        args = SimpleNamespace(
            model_name="model-id",
            device_map="0,1",
            eval_bs=8,
            eval_model_dtype="auto",
            tasks="mmlu",
            mllm=False,
            disable_trust_remote_code=False,
            add_bos_token=False,
            limit=None,
            vllm_args=None,
        )

        captured_kwargs = {}
        captured_evaluate_kwargs = {}
        task_manager = object()
        fake_res = {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}}

        class FakeVLLM:
            def __init__(self, **kwargs):
                captured_kwargs.update(kwargs)

        fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
        fake_vllm_causallms.VLLM = FakeVLLM
        fake_vllm_vlms = type(sys)("lm_eval.models.vllm_vlms")
        fake_vllm_vlms.VLLM_VLM = type("VLLM_VLM", (), {"__init__": lambda *args, **kwargs: None})

        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_vlms", fake_vllm_vlms)

        monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
        monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
        monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
        monkeypatch.setattr(eval_cli, "_get_lm_eval_task_manager", lambda tasks: task_manager)

        def fake_simple_evaluate(**kwargs):
            captured_evaluate_kwargs.update(kwargs)
            return fake_res

        monkeypatch.setattr("lm_eval.evaluator.simple_evaluate", fake_simple_evaluate)
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
        monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false")

        with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
            eval_cli.eval_with_vllm(args)

        assert captured_kwargs["pretrained"] == "model-id"
        assert captured_kwargs["tensor_parallel_size"] == 2
        assert captured_evaluate_kwargs["task_manager"] is task_manager

    def test_task_manager_finds_nested_yaml_task(self, monkeypatch, tmp_path):
        tasks_root = tmp_path / "lm_eval" / "tasks"
        task_dir = tasks_root / "lambada"
        task_dir.mkdir(parents=True)
        (task_dir / "lambada_openai.yaml").write_text("task: lambada_openai\n")

        fake_lm_eval = type(sys)("lm_eval")
        fake_lm_eval.__file__ = str(tmp_path / "lm_eval" / "__init__.py")
        captured = {}

        class FakeTaskManager:
            def __init__(self, **kwargs):
                captured.update(kwargs)

        fake_tasks = type(sys)("lm_eval.tasks")
        fake_tasks.TaskManager = FakeTaskManager
        monkeypatch.setitem(sys.modules, "lm_eval", fake_lm_eval)
        monkeypatch.setitem(sys.modules, "lm_eval.tasks", fake_tasks)

        task_manager = eval_cli._get_lm_eval_task_manager(["lambada_openai"])

        assert isinstance(task_manager, FakeTaskManager)
        assert captured["include_defaults"] is False
        assert captured["include_path"] == [str(task_dir)]

    def test_mllm_uses_vllm_vlm(self, monkeypatch):
        args = SimpleNamespace(
            model_name="model-id",
            device_map="0",
            eval_bs=8,
            eval_model_dtype="auto",
            tasks="mmlu",
            mllm=True,
            disable_trust_remote_code=False,
            add_bos_token=False,
            limit=None,
            vllm_args=None,
        )

        captured_class = {}

        class FakeVLLM_VLM:
            def __init__(self, **kwargs):
                captured_class["kwargs"] = kwargs

        fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
        fake_vllm_causallms.VLLM = object
        fake_vllm_vlms = type(sys)("lm_eval.models.vllm_vlms")
        fake_vllm_vlms.VLLM_VLM = FakeVLLM_VLM

        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_vlms", fake_vllm_vlms)

        monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
        monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
        monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
        monkeypatch.setattr(
            "lm_eval.evaluator.simple_evaluate",
            lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
        )
        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")

        with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
            eval_cli.eval_with_vllm(args)

        assert captured_class["kwargs"]["pretrained"] == "model-id"

    def test_existing_device_env_is_not_overwritten(self, monkeypatch):
        args = SimpleNamespace(
            model_name="model-id",
            device_map="0",
            eval_bs=8,
            eval_model_dtype="auto",
            tasks="mmlu",
            mllm=False,
            disable_trust_remote_code=False,
            add_bos_token=False,
            limit=None,
            vllm_args=None,
        )

        fake_vllm_causallms = type(sys)("lm_eval.models.vllm_causallms")
        fake_vllm_causallms.VLLM = type("VLLM", (), {"__init__": lambda *args, **kwargs: None})
        fake_vllm_vlms = type(sys)("lm_eval.models.vllm_vlms")
        fake_vllm_vlms.VLLM_VLM = type("VLLM_VLM", (), {"__init__": lambda *args, **kwargs: None})

        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_causallms", fake_vllm_causallms)
        monkeypatch.setitem(sys.modules, "lm_eval.models.vllm_vlms", fake_vllm_vlms)

        monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "fake-env")
        monkeypatch.setattr(eval_cli, "get_major_device", lambda: "cuda")
        monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cuda", False))
        monkeypatch.setattr(eval_cli, "get_model_dtype", lambda dtype, default="auto": "auto")
        monkeypatch.setattr(
            "lm_eval.evaluator.simple_evaluate",
            lambda **kwargs: {"results": {"mmlu": {}}, "versions": {}, "n-shot": {}, "higher_is_better": {}},
        )

        with patch("auto_round.utils.DEVICE_ENVIRON_VARIABLE_MAPPING", {"cuda": "CUDA_VISIBLE_DEVICES"}):
            eval_cli.eval_with_vllm(args)

        assert os.environ.get("CUDA_VISIBLE_DEVICES") == "fake-env"


class TestEvalTaskByTask:
    """Tests for `eval_task_by_task`, focusing on CPU-safe branches."""

    def test_non_parallel_string_model_path(self, monkeypatch):
        fake_hflm = object()

        monkeypatch.setattr(eval_cli, "set_cuda_visible_devices", lambda device: None)
        monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cpu", None))
        monkeypatch.setattr(
            "auto_round.eval.eval_cli._load_gguf_model_if_needed",
            lambda *args, **kwargs: ("hf-causallm-model", None, False, None),
        )
        monkeypatch.setattr(
            "lm_eval.models.huggingface.HFLM",
            lambda **kwargs: fake_hflm,
        )
        monkeypatch.setattr(
            "auto_round.eval.eval_cli.dispatch_model_block_wise",
            lambda model, device_map: None,
        )

        captured = {}

        def fake_evaluate(*args, **kwargs):
            captured["tasks"] = kwargs.get("tasks", args[0] if args else None)
            captured["hflm"] = kwargs.get("hflm", args[1] if len(args) > 1 else None)

        monkeypatch.setattr(eval_cli, "_evaluate_tasks_with_retry", fake_evaluate)

        eval_cli.eval_task_by_task(
            model="hf-causallm-model",
            device="cpu",
            tasks="mmlu",
            batch_size=4,
            limit=2,
        )

        assert captured["hflm"] is fake_hflm

    def test_non_parallel_non_string_model_skips_gguf(self, monkeypatch):
        fake_model = object()
        fake_hflm = object()

        monkeypatch.setattr(eval_cli, "set_cuda_visible_devices", lambda device: None)
        monkeypatch.setattr(eval_cli, "get_device_and_parallelism", lambda device: ("cpu", None))
        monkeypatch.setattr(
            "auto_round.eval.eval_cli._load_gguf_model_if_needed",
            lambda *args, **kwargs: (fake_model, None, False, None),
        )
        monkeypatch.setattr(
            "lm_eval.models.huggingface.HFLM",
            lambda **kwargs: fake_hflm,
        )
        monkeypatch.setattr(
            "auto_round.eval.eval_cli.dispatch_model_block_wise",
            lambda model, device_map: None,
        )
        monkeypatch.setattr(eval_cli, "_evaluate_tasks_with_retry", lambda *args, **kwargs: None)

        eval_cli.eval_task_by_task(
            model=fake_model,
            device="cpu",
            tasks="mmlu",
        )


class TestEvaluateTasksWithRetry:
    """Tests for `_evaluate_tasks_with_retry` retry and aggregation behavior."""

    def test_successful_task_is_recorded(self, monkeypatch):
        fake_hflm = object()
        fake_res = {
            "results": {"mmlu": {"accuracy": 0.5}},
            "versions": {"mmlu": "1.0"},
            "n-shot": {"mmlu": 5},
            "higher_is_better": {"mmlu": True},
        }

        monkeypatch.setattr(
            "lm_eval.simple_evaluate",
            lambda **kwargs: fake_res,
        )
        monkeypatch.setattr(
            "lm_eval.utils.make_table",
            lambda res: "",
        )

        eval_cli._evaluate_tasks_with_retry(
            tasks=["mmlu"],
            hflm=fake_hflm,
            device_str="cpu",
            batch_size=8,
            limit=None,
            retry_times=3,
        )

    def test_string_tasks_are_split(self, monkeypatch):
        fake_hflm = object()
        fake_res = {
            "results": {"mmlu": {"accuracy": 0.5}},
            "versions": {"mmlu": "1.0"},
            "n-shot": {"mmlu": 5},
            "higher_is_better": {"mmlu": True},
        }
        captured = []

        def fake_simple_evaluate(**kwargs):
            captured.append(kwargs["tasks"])
            return fake_res

        monkeypatch.setattr("lm_eval.simple_evaluate", fake_simple_evaluate)
        monkeypatch.setattr("lm_eval.utils.make_table", lambda res: "")

        eval_cli._evaluate_tasks_with_retry(
            tasks="mmlu,wikitext",
            hflm=fake_hflm,
            device_str="cpu",
            batch_size=8,
            limit=None,
            retry_times=1,
        )

        assert captured == ["mmlu", "wikitext"]

    def test_oom_retry_reduces_batch_size(self, monkeypatch):
        fake_hflm = type("FakeHFLM", (), {"batch_sizes": None})()
        fake_res = {
            "results": {"mmlu": {"accuracy": 0.5}},
            "versions": {"mmlu": "1.0"},
            "n-shot": {"mmlu": 5},
            "higher_is_better": {"mmlu": True},
        }
        calls = []

        def fake_simple_evaluate(**kwargs):
            calls.append(kwargs.get("batch_size"))
            if len(calls) <= 2 and calls[-1] == 8:
                raise torch.OutOfMemoryError("CUDA out of memory")
            return fake_res

        monkeypatch.setattr("lm_eval.simple_evaluate", fake_simple_evaluate)
        monkeypatch.setattr("lm_eval.utils.make_table", lambda res: "")

        eval_cli._evaluate_tasks_with_retry(
            tasks=["mmlu"],
            hflm=fake_hflm,
            device_str="cpu",
            batch_size=8,
            limit=None,
            retry_times=1,
        )

        assert calls == [8, 1]

    def test_non_oom_error_is_raised_without_retry(self, monkeypatch):
        calls = []

        def fake_simple_evaluate(**kwargs):
            calls.append(kwargs)
            raise RuntimeError("permanent failure")

        monkeypatch.setattr(
            "lm_eval.simple_evaluate",
            fake_simple_evaluate,
        )

        with pytest.raises(RuntimeError, match="permanent failure"):
            eval_cli._evaluate_tasks_with_retry(
                tasks=["bad-task"],
                hflm=object(),
                device_str="cpu",
                batch_size=8,
                limit=None,
                retry_times=2,
            )

        assert len(calls) == 1

    def test_multiple_tasks_are_aggregated(self, monkeypatch):
        fake_hflm = object()
        res_a = {
            "results": {"mmlu": {"accuracy": 0.5}},
            "versions": {"mmlu": "1.0"},
            "n-shot": {"mmlu": 5},
            "higher_is_better": {"mmlu": True},
        }
        res_b = {
            "results": {"wikitext": {"word_perplexity": 10.0}},
            "versions": {"wikitext": "1.0"},
            "n-shot": {"wikitext": 0},
            "higher_is_better": {"wikitext": False},
        }

        monkeypatch.setattr("lm_eval.simple_evaluate", lambda **kwargs: res_a if kwargs["tasks"] == "mmlu" else res_b)
        monkeypatch.setattr("lm_eval.utils.make_table", lambda res: "")

        eval_cli._evaluate_tasks_with_retry(
            tasks=["mmlu", "wikitext"],
            hflm=fake_hflm,
            device_str="cpu",
            batch_size=8,
            limit=None,
            retry_times=1,
        )


class TestLoadGgufModelIfNeeded:
    """CPU-only tests for GGUF detection using temporary files/directories."""

    def test_parent_export_dir_resolves_to_single_nested_artifact_dir(self, tmp_path):
        export_root = tmp_path / "aware_topk_tune_shared"
        nested_model_dir = export_root / "Qwen3-30B-A3B-w4g128"
        nested_model_dir.mkdir(parents=True)
        (nested_model_dir / "config.json").write_text('{"model_type": "qwen3_moe"}', encoding="utf-8")

        resolved = eval_cli._resolve_exported_model_path(str(export_root))

        assert resolved == str(nested_model_dir)

    def test_parent_export_dir_stays_unchanged_when_multiple_nested_artifacts_exist(self, tmp_path):
        export_root = tmp_path / "aware_topk_tune_shared"
        first_nested = export_root / "first-export"
        second_nested = export_root / "second-export"
        first_nested.mkdir(parents=True)
        second_nested.mkdir(parents=True)
        (first_nested / "config.json").write_text('{"model_type": "llama"}', encoding="utf-8")
        (second_nested / "config.json").write_text('{"model_type": "llama"}', encoding="utf-8")

        resolved = eval_cli._resolve_exported_model_path(str(export_root))

        assert resolved == str(export_root)

    def test_gguf_file_detected_at_file_path(self, monkeypatch, tmp_path):
        gguf_path = tmp_path / "model.gguf"
        # GGUF readers validate the four-byte magic even though the model
        # loading calls below are mocked.  Keep the fixture structurally valid.
        gguf_path.write_bytes(b"GGUF")
        fake_model = MagicMock()

        monkeypatch.setattr(eval_cli.os.path, "isfile", lambda value: value == str(gguf_path))
        monkeypatch.setattr(eval_cli.os.path, "exists", lambda value: value == str(tmp_path) or value == str(gguf_path))
        monkeypatch.setattr(eval_cli.os, "listdir", lambda value: ["model.gguf"] if value == str(tmp_path) else [])
        monkeypatch.setattr(eval_cli, "get_model_dtype", lambda value="auto": "auto")
        monkeypatch.setitem(
            transformers.__dict__, "AutoTokenizer", SimpleNamespace(from_pretrained=lambda *args, **kwargs: object())
        )
        monkeypatch.setitem(
            transformers.__dict__,
            "AutoModelForCausalLM",
            SimpleNamespace(from_pretrained=lambda *args, **kwargs: fake_model),
        )

        model, tokenizer, is_gguf, gguf_file = eval_cli._load_gguf_model_if_needed(
            str(gguf_path), eval_model_dtype="auto"
        )

        assert is_gguf is True
        assert gguf_file == "model.gguf"
        assert tokenizer is not None
        assert model is fake_model

    def test_gguf_file_detected_inside_model_dir(self, monkeypatch, tmp_path):
        model_dir = tmp_path / "model"
        model_dir.mkdir()
        (model_dir / "model.gguf").write_bytes(b"GGUF")
        fake_model = MagicMock()

        monkeypatch.setattr(eval_cli.os.path, "isfile", lambda value: value == str(model_dir))
        monkeypatch.setattr(
            eval_cli.os.path, "exists", lambda value: value == str(model_dir) or value == str(model_dir / "model.gguf")
        )
        monkeypatch.setattr(eval_cli.os, "listdir", lambda value: ["model.gguf"])
        monkeypatch.setattr(eval_cli, "get_model_dtype", lambda value="auto": "auto")
        monkeypatch.setitem(
            transformers.__dict__, "AutoTokenizer", SimpleNamespace(from_pretrained=lambda *args, **kwargs: object())
        )
        monkeypatch.setitem(
            transformers.__dict__,
            "AutoModelForCausalLM",
            SimpleNamespace(from_pretrained=lambda *args, **kwargs: fake_model),
        )

        model, tokenizer, is_gguf, gguf_file = eval_cli._load_gguf_model_if_needed(
            str(model_dir), eval_model_dtype="auto"
        )

        assert is_gguf is True
        assert gguf_file == "model.gguf"
        assert tokenizer is not None
        assert model is fake_model