PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
release CUDA memory in WeightConverter and avoid meaningless print (#1498)
Xin Hecd10eab0201e
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.pyindex 85b294759..44b4f1c3f 100644--- a/auto_round/modeling/fused_moe/moe_experts_interface.py+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py@@ -494,6 +494,8 @@ def _unfuse_experts_weights_inplace(logger.debug(f"Skipping unfuse for {module.__class__.__name__}: does not support @use_experts_implementation")return False+ memory_monitor.update()+ memory_monitor.log_summary("Before applying custom replacements")# Get first projection to determine num_experts and layoutfirst_proj_name = next(iter(detected_projections))first_param = getattr(module, first_proj_name)@@ -592,6 +594,9 @@ def _unfuse_experts_weights_inplace(# Install compact repr to collapse identical expert containers in print output_install_compact_expert_repr(module)+ memory_monitor.update()+ memory_monitor.log_summary("After applying custom replacements")+return Truediff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.pyindex f84515348..6938250fe 100644--- a/auto_round/modeling/fused_moe/replace_modules.py+++ b/auto_round/modeling/fused_moe/replace_modules.py@@ -48,7 +48,6 @@ if not is_transformers_version_greater_or_equal_5():BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")-@dump_mem_usage("Applying general replacements")def _handle_moe_modules(model: torch.nn.Module) -> list[str]:"""Handle fused MOE modules using transformers' linear_loop backend.@@ -291,20 +290,20 @@ def apply_replacements("""_import_required_replacements(model)- _log_first_moe_block(model, "before replacement")-# Custom replacements firstif is_custom_model(model):+ _log_first_moe_block(model, "before replacement")_apply_custom_replacements(model)+ _log_first_moe_block(model, "after replacement")if auto_detect_moe and is_transformers_version_greater_or_equal_5():+ _log_first_moe_block(model, "before replacement")_handle_moe_modules(model)-- _log_first_moe_block(model, "after replacement")+ _log_first_moe_block(model, "after replacement")return model-@dump_mem_usage("Applying custom replacements")+@dump_mem_usage("applying custom replacements")def _apply_custom_replacements(model: torch.nn.Module) -> list:"""Scan model and replace registered modules with custom implementations.diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex ec7c08e2c..11aaf452d 100644--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -543,6 +543,10 @@ class FP8Handler(WeightTypeHandler):new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))+ # Free intermediate CUDA tensors to avoid memory buildup+ del dq_weight+ layer.to("meta")+if to_cpu:new_layer = new_layer.to("cpu")@@ -615,6 +619,10 @@ class MXFP4Handler(WeightTypeHandler):)new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))+ # Free intermediate CUDA tensors to avoid memory buildup+ del dq_weight+ layer.to("meta")+if to_cpu:new_layer = new_layer.to("cpu")@@ -683,6 +691,10 @@ class MXFP8Handler(WeightTypeHandler):)new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))+ # Free intermediate CUDA tensors to avoid memory buildup+ del dq_weight+ layer.to("meta")+if to_cpu:new_layer = new_layer.to("cpu")@@ -735,6 +747,10 @@ class NVFP4Handler(WeightTypeHandler):dq_weight = layer.compressor.decompress_module(layer)new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))+ # Free intermediate CUDA tensors to avoid memory buildup+ del dq_weight+ layer.to("meta")+if to_cpu:new_layer = new_layer.to("cpu")diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.pyindex 0a2847d0a..382f3ab51 100644--- a/test/test_cpu/core/test_autoround.py+++ b/test/test_cpu/core/test_autoround.py@@ -632,16 +632,16 @@ class TestAutoRound:with patch("auto_round.utils.device.is_gaudi2", return_value=True):check_and_mark_quantized_module(mock_layer)convert_module_to_hp_if_necessary(mock_layer, device="hpu")- # Verify it was moved to CPU- mock_layer.to.assert_called_with("cpu")+ # Verify it was moved to meta+ mock_layer.to.assert_called_with("meta")with patch("auto_round.utils.device.is_gaudi2", return_value=False):# Reset mockmock_layer.to.reset_mock()check_and_mark_quantized_module(mock_layer)convert_module_to_hp_if_necessary(mock_layer, device="hpu")- # Verify it was moved to HPU (as requested in device arg)- mock_layer.to.assert_called_with("hpu")+ # Verify it was moved to meta+ mock_layer.to.assert_called_with("meta")def test_mixed_bit_setting(self, tiny_opt_model_path):model_name = tiny_opt_model_pathdiff --git a/test/test_cpu/utils/test_fp8_re_quant.py b/test/test_cpu/utils/test_fp8_re_quant.pyindex 21596f2de..0ddd1addd 100644--- a/test/test_cpu/utils/test_fp8_re_quant.py+++ b/test/test_cpu/utils/test_fp8_re_quant.py@@ -36,11 +36,12 @@ class TestFP8ReQuant(unittest.TestCase):in_features, out_features = 128, 64layer = MockFP8Layer(in_features, out_features)- dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None)+ tmp_weight = layer.weight.clone()+ dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None) # in-place operationself.assertEqual(dq_weight.shape, (out_features, in_features))self.assertEqual(dq_weight.dtype, torch.bfloat16)- expected = layer.weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)+ expected = tmp_weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)torch.testing.assert_close(dq_weight, expected)def test_per_channel_dequant(self):@@ -50,14 +51,18 @@ class TestFP8ReQuant(unittest.TestCase):# Per-channel scale (out_features)scale = torch.randn(out_features)- dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)- expected = weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)+ tmp_weight = weight.clone()+ # in-place operation+ dq_weight = _dequant_fp8_linear_weight(tmp_weight, scale, block_size=None)+ expected = tmp_weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)torch.testing.assert_close(dq_weight, expected)# Per-channel scale (in_features)scale = torch.randn(in_features)- dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)- expected = weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)+ tmp_weight = weight.clone()+ # in-place operation+ dq_weight = _dequant_fp8_linear_weight(tmp_weight, scale, block_size=None)+ expected = tmp_weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)torch.testing.assert_close(dq_weight, expected)def test_devstral_model_structure(self):@@ -87,6 +92,9 @@ class TestFP8ReQuant(unittest.TestCase):self.data_type = "fp8"mock_layer = FP8Linear(in_features, out_features)+ # mock_layer will to meta after converting.+ mock_weight = mock_layer.weight.clone()+ mock_weight_scale = mock_layer.weight_scale.clone()# This should now pass without AttributeError for 'block_size'check_and_mark_quantized_module(mock_layer)@@ -97,6 +105,6 @@ class TestFP8ReQuant(unittest.TestCase):self.assertEqual(new_layer.out_features, out_features)self.assertEqual(new_layer.weight.dtype, torch.bfloat16)- expected = mock_layer.weight.to(torch.bfloat16) * mock_layer.weight_scale.to(torch.bfloat16)+ expected = mock_weight.to(torch.bfloat16) * mock_weight_scale.to(torch.bfloat16)torch.testing.assert_close(new_layer.weight, expected)print("Devstral layer conversion successful!")