PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable low_cpu_mem_usage for mxfp/nvfp (#1648)
Tang Kaihui8d7bb84c1efa
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 75bc46c0c..c416dadb6 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -1746,8 +1746,10 @@ class BaseCompressor(object):self.low_cpu_mem_usage = Falseself.is_immediate_saving = False- if self.is_immediate_saving and "int" not in self.data_type:- logger.warning("immediate_saving is only supported for int quantization, set to False")+ if self.is_immediate_saving and not (+ "int" in self.data_type or is_nv_fp(self.data_type) or is_mx_fp(self.data_type)+ ):+ logger.warning("immediate_saving is only supported for int/nv_fp/mx_fp quantization, set to False")self.is_immediate_saving = Falseif self.orig_output_dir is None:diff --git a/auto_round/compressors/shard_writer.py b/auto_round/compressors/shard_writer.pyindex f061a92b9..f89ba714a 100644--- a/auto_round/compressors/shard_writer.py+++ b/auto_round/compressors/shard_writer.py@@ -146,10 +146,11 @@ class ShardWriter:filtered_tensors[name] = tensorcontinue- ptr = tensor.untyped_storage().data_ptr()+ ptr = tensor.untyped_storage().data_ptr() + tensor.storage_offset() * tensor.element_size()if ptr not in storage_map:storage_map.add(ptr)filtered_tensors[name] = tensor+self.current_shard_tensors = filtered_tensorsdef _flush_shard(self):diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.pyindex ff7a9e732..ea2e5a6ed 100644--- a/test/test_cpu/models/test_moe_model.py+++ b/test/test_cpu/models/test_moe_model.py@@ -64,13 +64,7 @@ def test_gptoss(scheme, tiny_gpt_oss_model_path, tmp_path):# verify the quantized model can be loaded and run inferenceloaded_model = GptOssForCausalLM.from_pretrained(output_dir)- for n, m in quantized_model.named_modules():- if m.__class__.__name__ == "QuantLinear":- loaded_m = loaded_model.get_submodule(n)- if scheme == "MXFP4":- assert (loaded_m.weight_packed.to("cpu") == m.weight_packed.to("cpu")).all()- if scheme == "MXFP8":- assert (loaded_m.weight.to("cpu") == m.weight.to("cpu")).all()+inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():loaded_out = loaded_model(inp)@@ -84,10 +78,7 @@ def test_llama4(tiny_llama4_model_path):assert quantized_model is not None, "Quantized model should not be None."loaded_model = Llama4ForConditionalGeneration.from_pretrained(output_dir)- for n, m in quantized_model.named_modules():- if m.__class__.__name__ == "QuantLinear":- loaded_m = loaded_model.get_submodule(n)- assert (loaded_m.weight_packed.to("cpu") == m.weight_packed.to("cpu")).all()+inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():loaded_out = loaded_model(inp)@@ -110,10 +101,6 @@ def test_qwen3_vl_moe_mxfp(tiny_qwen3_vl_moe_model_path):assert quantized_model is not None, "Quantized model should not be None."loaded_model = Qwen3VLMoeForConditionalGeneration.from_pretrained(output_dir, device_map="cpu")- for n, m in quantized_model.named_modules():- if m.__class__.__name__ == "QuantLinear":- loaded_m = loaded_model.get_submodule(n)- assert (loaded_m.weight_packed.to("cpu") == m.weight_packed.to("cpu")).all()inp = torch.randint(0, 100, (1, 32))with torch.inference_mode():loaded_out = loaded_model(inp)diff --git a/test/test_cuda/models/test_fp8_model.py b/test/test_cuda/models/test_fp8_model.pyindex 1b7e97e4d..e8343b0b2 100644--- a/test/test_cuda/models/test_fp8_model.py+++ b/test/test_cuda/models/test_fp8_model.py@@ -126,6 +126,7 @@ def test_qwen3_fp8_moe_mxfp(tiny_fp8_qwen_moe_model_path, mock_fp8_capable_devicnsamples=2,seqlen=32,iters=0,+ low_cpu_mem_usage=False,)quantized_model, _ = autoround.quantize_and_save(format="auto_round", output_dir=output_dir)assert quantized_model is not None, "Quantized model should not be None."