Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

7.7 KB
Raw Blame
import dataclasses
import json
import multiprocessing
import multiprocessing.resource_tracker
import shutil
import traceback
from pathlib import Path
from test.helpers import get_model_path, qwen_name_or_path

import pytest
import torch

from auto_round import AutoRound

# A patch to fix the Python `multiprocessing.ResourceTracker` [Errno 10] error.
_original_stop = multiprocessing.resource_tracker.ResourceTracker._stop


def _patched_stop(self, *args, _orig=_original_stop, **kwargs):
    if _orig is not None:
        try:
            _orig(self, *args, **kwargs)
        except ChildProcessError:
            pass


multiprocessing.resource_tracker.ResourceTracker._stop = _patched_stop


def _run_sglang_worker(model_path, connection):
    try:
        import sglang as sgl
        from sglang.srt.server_args import ServerArgs

        fields = set(ServerArgs.__annotations__.keys())
        engine_kwargs = {"model_path": model_path, "mem_fraction_static": 0.5}
        if "disable_piecewise_cuda_graph" in fields:
            engine_kwargs["disable_piecewise_cuda_graph"] = True
            if "cuda_graph_bs" in fields:
                engine_kwargs["cuda_graph_bs"] = [1]
        elif "disable_cuda_graph" in fields:
            engine_kwargs["disable_cuda_graph"] = True
            if "cuda_graph_bs_decode" in fields:
                engine_kwargs["cuda_graph_bs_decode"] = [1]

        llm = sgl.Engine(**engine_kwargs)
        try:
            outputs = llm.generate(["Hello, my name is"], {"temperature": 0.6, "top_p": 0.95})
            connection.send((True, outputs[0]["text"]))
        finally:
            llm.shutdown()
    except BaseException:
        connection.send((False, traceback.format_exc()))
    finally:
        connection.close()


class TestAutoRound:
    model_name = qwen_name_or_path

    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    @pytest.fixture(autouse=True, scope="class")
    def setup_and_teardown_class(self):
        # ===== SETUP (setup_class) =====
        print("[Setup] Running before any test in class")

        # Yield to hand control to the test methods
        yield

        # ===== TEARDOWN (teardown_class) =====
        print("[Teardown] Running after all tests in class")
        shutil.rmtree("runs", ignore_errors=True)

    def _run_sglang_inference(self, model_path: Path):
        # SM 12.x (Blackwell) GPUs require CUDA >= 12.9 for sglang's gptq_marlin_repack JIT kernel.
        # Skip inference when the environment is known to be incompatible.
        if torch.cuda.is_available():
            try:
                major, minor = torch.cuda.get_device_capability()
                if major >= 12:
                    cuda_ver = tuple(int(x) for x in (torch.version.cuda or "0.0").split(".")[:2])
                    if cuda_ver < (12, 9):
                        pytest.skip(
                            f"SM {major}.{minor} GPU requires CUDA >= 12.9 for sglang GPTQ JIT kernels "
                            f"(installed: CUDA {torch.version.cuda})"
                        )
            except Exception:
                pass

        context = multiprocessing.get_context("spawn")
        parent_connection, child_connection = context.Pipe(duplex=False)
        process = context.Process(target=_run_sglang_worker, args=(str(model_path), child_connection))
        process.start()
        child_connection.close()
        try:
            if not parent_connection.poll(600):
                process.terminate()
                pytest.fail("SGLang inference timed out after 600 seconds")
            succeeded, result = parent_connection.recv()
        finally:
            parent_connection.close()
            process.join(timeout=30)
            if process.is_alive():
                process.kill()
                process.join()

        if not succeeded:
            pytest.fail(f"SGLang worker failed:\n{result}")
        return result

    def test_ar_format_sglang(self, dataloader):
        autoround = AutoRound(
            self.model_name,
            scheme="W4A16",
            iters=2,
            seqlen=2,
            dataset=dataloader,
        )

        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=self.save_dir,
            inplace=True,
            format="auto_round",
        )

        generated_text = self._run_sglang_inference(quantized_model_path)
        print(generated_text)

        assert "!!!" not in generated_text

    def test_mixed_ar_format_sglang(self, dataloader):
        layer_config = {
            "self_attn": {"bits": 8},
            "lm_head": {"bits": 16},
            "mlp": {"bits": 16, "act_bits": 16},
        }

        autoround = AutoRound(
            self.model_name,
            scheme="W4A16",
            iters=2,
            seqlen=2,
            dataset=dataloader,
            layer_config=layer_config,
        )

        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=self.save_dir,
            inplace=True,
            format="auto_round",
        )
        config_file = Path(quantized_model_path) / "config.json"
        with open(config_file, "r", encoding="utf-8") as f:
            config = json.load(f)
        quant_config = config.get("quantization_config", {})
        extra_config = quant_config.get("extra_config", {})
        # check extra_config only saved attributes differing from Scheme values
        assert "act_bits" not in extra_config[".*mlp.*"].keys()
        assert "group_size" not in extra_config[".*mlp.*"].keys()
        assert "bits" in extra_config[".*mlp.*"].keys() and extra_config[".*mlp.*"]["bits"] == 16
        assert "bits" in extra_config[".*self_attn.*"].keys() and extra_config[".*self_attn.*"]["bits"] == 8
        generated_text = self._run_sglang_inference(quantized_model_path)
        print(generated_text)

        assert "!!!" not in generated_text

        shutil.rmtree(self.save_dir, ignore_errors=True)

    # TODO: transformers already fixed this bug, need to upgrade sglang
    @pytest.mark.xfail(reason="sglang is not upgraded with latest transformers")
    def test_qwen2_5_vl_loading(self, tiny_qwen_2_5_vl_model_path):
        from auto_round.utils import mllm_load_model

        layer_config = {
            "self_attn": {"bits": 8},
            "lm_head": {"bits": 16},
            "mlp": {"bits": 16, "act_bits": 16},
        }

        model, processor, tokenizer, image_processor = mllm_load_model(tiny_qwen_2_5_vl_model_path)

        autoround = AutoRound(
            model,
            tokenizer,
            scheme="W4A16",
            iters=1,
            nsamples=1,
            seqlen=32,
            processor=processor,
            image_processor=image_processor,
            layer_config=layer_config,
        )

        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=self.save_dir,
            inplace=True,
            format="auto_round",
        )

        generated_text = self._run_sglang_inference(quantized_model_path)
        print(generated_text)

        assert "!!!" not in generated_text

    @pytest.mark.skip_ci(reason="Third-party: Cannot work well in CI env")
    def test_awq_format_sglang(self, dataloader):
        autoround = AutoRound(
            self.model_name,
            scheme="W4A16",
            iters=2,
            seqlen=2,
            dataset=dataloader,
        )

        _, quantized_model_path = autoround.quantize_and_save(
            output_dir=self.save_dir,
            inplace=True,
            format="auto_round:auto_awq",
        )

        generated_text = self._run_sglang_inference(quantized_model_path)
        print(generated_text)

        assert "!!!" not in generated_text