Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add inc integration unit test (#1820)

Xin He948b482c237a
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 759f0ed7c..c54f63d94 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -67,7 +67,7 @@ function run_unit_test() {
auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
# Split test files into 5 parts
- find ./test_cpu -name "test*.py" | grep -Ev "llmc" | sort > all_tests.txt
+ find ./test_cpu -name "test*.py" | grep -Ev "test_llmc|test_inc" | sort > all_tests.txt
total_lines=$(wc -l < all_tests.txt)
NUM_CHUNKS=5
q=$(( total_lines / NUM_CHUNKS ))
@@ -95,10 +95,33 @@ function run_unit_test() {
done
}
+function run_inc_unit_test() {
+ echo "##[group]set up INC UT env..."
+ INC_PT_ONLY=1 uv pip install -r /auto-round/test/test_cpu/requirements_inc.txt
+ echo "##[endgroup]"
+
+ cd /auto-round/test || exit 1
+ auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
+
+ for test_file in $(find ./test_cpu -name "test_inc*.py" | sort); do
+ echo "##[group]Running ${test_file}..."
+ local test_basename=$(basename ${test_file} .py)
+ local ut_log_name=${LOG_DIR}/unittest_${test_basename}.log
+
+ numactl --physcpubind="${NUMA_CPUSET:-0-15}" --membind="${NUMA_NODE:-0}" \
+ python -m pytest --cov="${auto_round_path}" --cov-report term --html=report.html --self-contained-html \
+ --cov-report xml:coverage.xml --cov-append \
+ -vs --disable-warnings ${test_file} 2>&1 | tee ${ut_log_name}
+ echo "##[endgroup]"
+ done
+}
+
function run_llmc_unit_test() {
+ echo "##[group]set up LLMC UT env..."
BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt
uv pip uninstall auto-round
cd /auto-round && uv pip install .
+ echo "##[endgroup]"
cd /auto-round/test || exit 1
auto_round_path=$(python -c 'import auto_round; print(auto_round.__path__[0])')
@@ -127,6 +150,7 @@ function main() {
setup_environment
run_unit_test
if [ "$test_part" -eq 5 ]; then
+ run_inc_unit_test
run_llmc_unit_test
fi
collect_log
diff --git a/test/test_cpu/integrations/test_inc_integration.py b/test/test_cpu/integrations/test_inc_integration.py
new file mode 100644
index 000000000..d195a6638
--- /dev/null
+++ b/test/test_cpu/integrations/test_inc_integration.py
@@ -0,0 +1,461 @@
+import copy
+import os
+import shutil
+
+import pytest
+import torch
+import transformers
+from neural_compressor.torch.quantization import (
+ AutoRoundConfig,
+ convert,
+ prepare,
+ quantize,
+)
+from neural_compressor.torch.utils import logger
+from packaging.version import Version
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+torch.backends.__allow_nonbracketed_mutation_flag = True
+
+try:
+ import compressed_tensors
+
+ ct_installed = True
+except ImportError:
+ ct_installed = False
+
+
+target_modules = ["QuantLinear", "QuantLinearGPTQ", "QuantLinearAWQ", "WQLinear_GEMM", "AwqTorchQuantLinear"]
+
+
+def run_fn(model, dataloader):
+ for data in dataloader:
+ if isinstance(data, tuple) or isinstance(data, list):
+ model(*data)
+ elif isinstance(data, dict):
+ model(**data)
+ else:
+ model(data)
+
+
+class TestAutoRoundCPU:
+ @pytest.fixture(autouse=True, scope="class")
+ def setup_class_fixture(self, tiny_opt_model_path, request):
+ cls = request.cls
+ cls.opt_model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_opt_model_path,
+ ).to("cpu")
+ cls.inp = torch.ones([1, 10], dtype=torch.long, device="cpu")
+ cls.tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+ from neural_compressor.torch.algorithms.autoround import get_dataloader
+
+ cls.dataloader = get_dataloader(
+ cls.tokenizer, 32, dataset_name="NeelNanda/pile-10k", seed=42, bs=8, nsamples=10
+ )
+ yield
+ shutil.rmtree("saved_results", ignore_errors=True)
+ shutil.rmtree("tmp_auto_round", ignore_errors=True)
+
+ def setup_method(self, method):
+ logger.info(f"Running TestAutoRound test: {method.__name__}")
+
+ def test_quant_lm_head(self):
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ # "trl-internal-testing/tiny-Phi3ForCausalLM",
+ "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM"
+ )
+ tokenizer = AutoTokenizer.from_pretrained(
+ "optimum-intel-internal-testing/tiny-random-Phi3ForCausalLM", trust_remote_code=True
+ )
+
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ iters=0,
+ amp=False,
+ disable_opt_rtn=True,
+ scale_dtype="fp32",
+ quant_lm_head=True,
+ group_size=32,
+ )
+ model = prepare(model=model, quant_config=quant_config)
+ q_model = convert(model)
+ assert q_model.lm_head.__class__.__name__ in target_modules, "packing model failed."
+
+ def test_int4_dtype(self):
+ fp32_model = copy.deepcopy(self.opt_model)
+ quant_config = AutoRoundConfig(dtype="int4", nsamples=32, seqlen=10, iters=1, amp=False, scale_dtype="fp32")
+ logger.info(f"Test AutoRound with config {quant_config}")
+
+ # prepare + convert API
+ model = prepare(model=fp32_model, quant_config=quant_config)
+
+ run_fn(model, self.dataloader)
+ q_model = convert(model)
+ _ = q_model(self.inp) # inference
+ assert (
+ q_model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__ in target_modules
+ ), "packing model failed."
+
+ def test_autoround_with_quantize_API(self):
+ fp32_model = copy.deepcopy(self.opt_model)
+
+ quant_config = AutoRoundConfig(
+ scheme="W4A16", seqlen=10, iters=1, use_sym=False, amp=False, scale_dtype="fp32", reloading=False
+ )
+ logger.info(f"Test AutoRound with config {quant_config}")
+
+ # quantize API
+ q_model = quantize(
+ model=fp32_model,
+ quant_config=quant_config,
+ run_fn=run_fn,
+ run_args=(self.dataloader,),
+ )
+ assert (
+ q_model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__ in target_modules
+ ), "packing model failed."
+
+ def test_conv1d(self, tiny_lamini_model_path):
+ model = AutoModelForCausalLM.from_pretrained(tiny_lamini_model_path, device_map="cpu", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(tiny_lamini_model_path, trust_remote_code=True)
+ text = "Replace me by any text you'd like."
+ encoded_input = tokenizer(text, return_tensors="pt")
+ quant_config = AutoRoundConfig(
+ nsamples=32,
+ seqlen=10,
+ iters=0,
+ amp=False,
+ tokenizer=tokenizer,
+ export_format="auto_round",
+ device_map="cpu",
+ )
+ model = prepare(model=model, quant_config=quant_config)
+ q_model = convert(model)
+ output = tokenizer.decode(q_model.generate(**encoded_input, max_new_tokens=10)[0])
+ print(output)
+ assert output is not None
+ assert not isinstance(
+ q_model.transformer.h[0].attn.c_attn, transformers.pytorch_utils.Conv1D
+ ), "loading compressed model failed."
+
+ def test_mllm(self, tiny_qwen_vl_model_path):
+ input = torch.randn(1, 32)
+ from neural_compressor.torch.algorithms.autoround import get_mllm_dataloader
+ from transformers import AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration
+
+ model_name = tiny_qwen_vl_model_path
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
+ model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="cpu")
+ dataloader, template, truncation, batch_size, gradient_accumulate_steps, seqlen, nsamples = get_mllm_dataloader(
+ template=None,
+ model=model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=None,
+ dataset="NeelNanda/pile-10k",
+ extra_data_dir=None,
+ seqlen=32,
+ batch_size=1,
+ split=None,
+ apply_template=None,
+ truncation=False,
+ seed=42,
+ nsamples=1,
+ gradient_accumulate_steps=1,
+ quant_nontext_module=True,
+ )
+ quant_config = AutoRoundConfig(
+ bits=4,
+ group_size=128,
+ nsamples=1,
+ batch_size=batch_size,
+ iters=1,
+ seqlen=seqlen,
+ quant_nontext_module=True,
+ truncation=truncation,
+ gradient_accumulate_steps=gradient_accumulate_steps,
+ device_map="cpu",
+ tokenizer=tokenizer,
+ processor=processor,
+ )
+
+ model = prepare(model=model, quant_config=quant_config)
+ run_fn(model, dataloader)
+ q_model = convert(model)
+ assert (
+ q_model.model.language_model.layers[0].mlp.up_proj.__class__.__name__ in target_modules
+ ), "model quantization failed."
+
+ def test_set_local(self, tiny_opt_model_path, tmp_path):
+ fp32_model = AutoModelForCausalLM.from_pretrained(
+ tiny_opt_model_path,
+ device_map="cpu",
+ )
+ output_dir = tmp_path
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ output_dir=output_dir,
+ dtype="int4",
+ iters=0,
+ amp=False,
+ disable_opt_rtn=True,
+ scale_dtype="fp32",
+ export_format="auto_round",
+ device_map="cpu",
+ )
+ logger.info(f"Test AutoRound with config {quant_config}")
+ quant_config.set_local("self_attn", AutoRoundConfig(bits=16, data_type="float", act_bits=16))
+
+ # prepare + convert API
+ model = prepare(model=fp32_model, quant_config=quant_config)
+ q_model = convert(model)
+
+ # Autoround applied subfolder for formats during saving, such as, './saved_inc/opt-125m-w4g128'.
+ output_dir = q_model.name_or_path
+ model = AutoModelForCausalLM.from_pretrained(
+ output_dir,
+ torch_dtype="auto",
+ device_map="cpu",
+ )
+ out = model(self.inp)[0]
+ assert isinstance(q_model.model.decoder.layers[0].self_attn.v_proj, torch.nn.Linear), "set_local failed."
+
+ @pytest.mark.skipif(not ct_installed, reason="The compressed-tensors module is not installed.")
+ @pytest.mark.parametrize("scheme", ["W3A16", "W8A16", "MXFP4", "NVFP4", "FP8_STATIC"])
+ def test_scheme(self, scheme, tiny_opt_model_path, tmp_path):
+ # INC API
+ fp32_model = copy.deepcopy(self.opt_model)
+ inp = torch.ones([1, 10], dtype=torch.long, device="cpu")
+ tokenizer = copy.deepcopy(self.tokenizer)
+
+ output_dir = tmp_path
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ nsamples=32,
+ seqlen=10,
+ iters=1,
+ amp=False,
+ scale_dtype="fp16",
+ scheme=scheme,
+ export_format="auto_round",
+ output_dir=output_dir, # default is "tmp_auto_round"
+ device_map="cpu",
+ )
+
+ # quantizer execute
+ model = prepare(model=fp32_model, quant_config=quant_config)
+ inc_model = convert(model)
+ # Autoround applied subfolder for formats during saving, such as, './saved_inc/opt-125m-w4g128'.
+ output_dir = inc_model.name_or_path
+ inc_model = AutoModelForCausalLM.from_pretrained(
+ output_dir,
+ torch_dtype="auto",
+ device_map="cpu",
+ )
+ out = inc_model(inp)[0]
+
+ @pytest.mark.skipif(not ct_installed, reason="The compressed-tensors module is not installed.")
+ def test_target_bits(self, tiny_opt_model_path, tmp_path):
+ fp32_model = AutoModelForCausalLM.from_pretrained(
+ tiny_opt_model_path,
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+
+ output_dir = tmp_path
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ nsamples=32,
+ seqlen=10,
+ iters=1,
+ target_bits=5,
+ options=("MXFP4", "MXFP8"),
+ enable_torch_compile=True,
+ low_gpu_mem_usage=True,
+ export_format="auto_round",
+ device_map="cpu",
+ )
+ # quantizer execute
+ model = prepare(model=fp32_model, quant_config=quant_config)
+ model = convert(model)
+ # mxfp4/8 model inference relies on autoround extension for vLLM.
+ target_modules = ["MXFP4QuantLinear", "MXFP8QuantLinear"]
+ assert (
+ model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__ in target_modules
+ and model.model.decoder.layers[1].fc1.__class__.__name__ in target_modules
+ ), "model is not quantized correctly, please check."
+
+ def test_static_attention_dtype(self, tiny_opt_model_path, tmp_path):
+ fp32_model = AutoModelForCausalLM.from_pretrained(
+ tiny_opt_model_path,
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+
+ output_dir = tmp_path
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ iters=0,
+ nsamples=2,
+ seqlen=2,
+ scheme="FP8_STATIC",
+ static_attention_dtype="fp8",
+ output_dir=output_dir,
+ export_format="auto_round",
+ device_map="cpu",
+ )
+ # quantizer execute
+ model = prepare(model=fp32_model, quant_config=quant_config)
+ model = convert(model)
+ output_dir = model.name_or_path
+
+ from safetensors import safe_open
+
+ f = safe_open(os.path.join(output_dir, "model.safetensors"), framework="pt")
+ assert "model.decoder.layers.1.self_attn.k_proj.input_scale" in f.keys()
+ assert "model.decoder.layers.1.self_attn.k_proj.weight_scale" in f.keys()
+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])
+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn
+ check_attrs = ["k_scale", "v_scale", "q_scale"]
+
+ for attr in check_attrs:
+ weight_name = f"model.decoder.layers.1.self_attn.{attr}"
+ assert weight_name in f.keys()
+ assert f.get_tensor(weight_name).shape == torch.Size([1])
+ assert f.get_tensor(weight_name).dtype == torch.float32
+
+ @pytest.mark.parametrize("static_kv_dtype", [None, "fp8", "float16"])
+ def test_static_afp8_export(self, static_kv_dtype, tiny_opt_model_path, tmp_path):
+ fp32_model = AutoModelForCausalLM.from_pretrained(
+ tiny_opt_model_path,
+ device_map="cpu",
+ )
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+
+ output_dir = tmp_path
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ bits=8,
+ group_size=-1,
+ iters=0,
+ act_bits=8,
+ nsamples=2,
+ seqlen=2,
+ data_type="fp8",
+ act_data_type="fp8",
+ act_dynamic=False,
+ act_group_size=0,
+ static_kv_dtype=static_kv_dtype,
+ export_format="auto_round",
+ output_dir=output_dir,
+ device_map="cpu",
+ )
+
+ # quantizer execute
+ model = prepare(model=fp32_model, quant_config=quant_config)
+ model = convert(model)
+ output_dir = model.name_or_path
+
+ from safetensors import safe_open
+
+ f = safe_open(os.path.join(output_dir, "model.safetensors"), framework="pt")
+ assert "model.decoder.layers.1.self_attn.k_proj.input_scale" in f.keys()
+ assert "model.decoder.layers.1.self_attn.k_proj.weight_scale" in f.keys()
+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])
+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn
+ if static_kv_dtype is None:
+ with torch.no_grad():
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ output_dir,
+ torch_dtype="auto",
+ device_map="cpu",
+ low_cpu_mem_usage=True,
+ trust_remote_code=True,
+ )
+ model.eval()
+ assert (
+ model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__
+ == "WeightFP8ActFP8StaticQuantLinear"
+ ), f"Expected WeightFP8ActFP8StaticQuantLinear, got {model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__}"
+ tokenizer = transformers.AutoTokenizer.from_pretrained(output_dir)
+ prompt = "AI is "
+ encode = tokenizer.encode(prompt, return_tensors="pt")
+ with torch.no_grad():
+ output_tokens = model.generate(
+ encode,
+ max_length=10,
+ )
+ output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+ print(f"Prompt: {prompt}")
+ print(f"Output: {output}")
+ assert output is not None, "Output should not be None"
+
+ @pytest.mark.parametrize(
+ "scheme, static_kv_dtype, static_attention_dtype",
+ [
+ ("MXFP4", None, "fp8"),
+ ("MXFP4", "fp8", None),
+ ("MXFP8", None, "fp8"),
+ ("MXFP8", "fp8", None),
+ ("NVFP4", None, "fp8"),
+ ("NVFP4", "fp8", None),
+ ],
+ )
+ def test_fp8_kv_attn(self, scheme, static_kv_dtype, static_attention_dtype, tiny_opt_model_path, tmp_path):
+
+ from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
+ from transformers.models.opt.modeling_opt import OPTForCausalLM
+
+ model_name = tiny_opt_model_path
+ config = AutoConfig.from_pretrained(model_name)
+ config.num_hidden_layers = 1
+ model = OPTForCausalLM(config)
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+ output_dir = tmp_path
+ quant_config = AutoRoundConfig(
+ tokenizer=tokenizer,
+ scheme=scheme,
+ iters=0,
+ seqlen=2,
+ static_kv_dtype=static_kv_dtype,
+ static_attention_dtype=static_attention_dtype,
+ export_format="auto_round",
+ output_dir=output_dir,
+ reloading=False,
+ device_map="cpu",
+ )
+
+ # quantizer execute
+ model = prepare(model=model, quant_config=quant_config)
+ compressed_model = convert(model)
+
+ attn = compressed_model.model.decoder.layers[0].self_attn
+ q_proj = attn.q_proj
+
+ # weight_scale should exist for all quantized schemes
+ assert hasattr(q_proj, "weight_scale"), f"Missing weight_scale in q_proj for scheme={scheme}"
+ if static_kv_dtype == "fp8":
+ assert (
+ compressed_model.config.quantization_config["static_kv_dtype"] == "fp8"
+ ), f"Invalid static_kv_dtype in config for scheme={scheme}, static_kv_dtype={static_kv_dtype}"
+
+ # Only when static_kv_dtype / static_attention_dtype are fp8 do we expect FP8 KV scales
+ if static_kv_dtype == "fp8" or static_attention_dtype == "fp8":
+ assert attn.k_scale is not None and attn.v_scale is not None, (
+ f"Missing k_scale/v_scale in attention for scheme={scheme}, "
+ f"static_kv_dtype={static_kv_dtype}, static_attention_dtype={static_attention_dtype}"
+ )
+
+ if static_attention_dtype == "fp8":
+ assert (
+ compressed_model.config.quantization_config["static_attention_dtype"] == "fp8"
+ ), f"Invalid static_attention_dtype in config for scheme={scheme}, static_attention_dtype={static_attention_dtype}"
+ assert (
+ getattr(attn, "q_scale", None) is not None
+ ), f"Missing q_scale in attention for scheme={scheme}, static_attention_dtype={static_attention_dtype}"
diff --git a/test/test_cpu/requirements_inc.txt b/test/test_cpu/requirements_inc.txt
new file mode 100644
index 000000000..cb2222096
--- /dev/null
+++ b/test/test_cpu/requirements_inc.txt
@@ -0,0 +1,2 @@
+neural_compressor_pt @ git+https://github.com/intel/[email protected]
+compressed-tensors