Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

release CUDA memory in WeightConverter and avoid meaningless print (#1498)

Xin Hecd10eab0201e
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index 85b294759..44b4f1c3f 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -494,6 +494,8 @@ def _unfuse_experts_weights_inplace(
logger.debug(f"Skipping unfuse for {module.__class__.__name__}: does not support @use_experts_implementation")
return False
+ memory_monitor.update()
+ memory_monitor.log_summary("Before applying custom replacements")
# Get first projection to determine num_experts and layout
first_proj_name = next(iter(detected_projections))
first_param = getattr(module, first_proj_name)
@@ -592,6 +594,9 @@ def _unfuse_experts_weights_inplace(
# Install compact repr to collapse identical expert containers in print output
_install_compact_expert_repr(module)
+ memory_monitor.update()
+ memory_monitor.log_summary("After applying custom replacements")
+
return True
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index f84515348..6938250fe 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -48,7 +48,6 @@ if not is_transformers_version_greater_or_equal_5():
BUILTIN_MODULES["gpt_oss"] = LazyImport("auto_round.modeling.fused_moe.gpt_oss")
-@dump_mem_usage("Applying general replacements")
def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
"""Handle fused MOE modules using transformers' linear_loop backend.
@@ -291,20 +290,20 @@ def apply_replacements(
"""
_import_required_replacements(model)
- _log_first_moe_block(model, "before replacement")
-
# Custom replacements first
if is_custom_model(model):
+ _log_first_moe_block(model, "before replacement")
_apply_custom_replacements(model)
+ _log_first_moe_block(model, "after replacement")
if auto_detect_moe and is_transformers_version_greater_or_equal_5():
+ _log_first_moe_block(model, "before replacement")
_handle_moe_modules(model)
-
- _log_first_moe_block(model, "after replacement")
+ _log_first_moe_block(model, "after replacement")
return model
-@dump_mem_usage("Applying custom replacements")
+@dump_mem_usage("applying custom replacements")
def _apply_custom_replacements(model: torch.nn.Module) -> list:
"""Scan model and replace registered modules with custom implementations.
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index ec7c08e2c..11aaf452d 100644
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -543,6 +543,10 @@ class FP8Handler(WeightTypeHandler):
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+ # Free intermediate CUDA tensors to avoid memory buildup
+ del dq_weight
+ layer.to("meta")
+
if to_cpu:
new_layer = new_layer.to("cpu")
@@ -615,6 +619,10 @@ class MXFP4Handler(WeightTypeHandler):
)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+ # Free intermediate CUDA tensors to avoid memory buildup
+ del dq_weight
+ layer.to("meta")
+
if to_cpu:
new_layer = new_layer.to("cpu")
@@ -683,6 +691,10 @@ class MXFP8Handler(WeightTypeHandler):
)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+ # Free intermediate CUDA tensors to avoid memory buildup
+ del dq_weight
+ layer.to("meta")
+
if to_cpu:
new_layer = new_layer.to("cpu")
@@ -735,6 +747,10 @@ class NVFP4Handler(WeightTypeHandler):
dq_weight = layer.compressor.decompress_module(layer)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
+ # Free intermediate CUDA tensors to avoid memory buildup
+ del dq_weight
+ layer.to("meta")
+
if to_cpu:
new_layer = new_layer.to("cpu")
diff --git a/test/test_cpu/core/test_autoround.py b/test/test_cpu/core/test_autoround.py
index 0a2847d0a..382f3ab51 100644
--- a/test/test_cpu/core/test_autoround.py
+++ b/test/test_cpu/core/test_autoround.py
@@ -632,16 +632,16 @@ class TestAutoRound:
with patch("auto_round.utils.device.is_gaudi2", return_value=True):
check_and_mark_quantized_module(mock_layer)
convert_module_to_hp_if_necessary(mock_layer, device="hpu")
- # Verify it was moved to CPU
- mock_layer.to.assert_called_with("cpu")
+ # Verify it was moved to meta
+ mock_layer.to.assert_called_with("meta")
with patch("auto_round.utils.device.is_gaudi2", return_value=False):
# Reset mock
mock_layer.to.reset_mock()
check_and_mark_quantized_module(mock_layer)
convert_module_to_hp_if_necessary(mock_layer, device="hpu")
- # Verify it was moved to HPU (as requested in device arg)
- mock_layer.to.assert_called_with("hpu")
+ # Verify it was moved to meta
+ mock_layer.to.assert_called_with("meta")
def test_mixed_bit_setting(self, tiny_opt_model_path):
model_name = tiny_opt_model_path
diff --git a/test/test_cpu/utils/test_fp8_re_quant.py b/test/test_cpu/utils/test_fp8_re_quant.py
index 21596f2de..0ddd1addd 100644
--- a/test/test_cpu/utils/test_fp8_re_quant.py
+++ b/test/test_cpu/utils/test_fp8_re_quant.py
@@ -36,11 +36,12 @@ class TestFP8ReQuant(unittest.TestCase):
in_features, out_features = 128, 64
layer = MockFP8Layer(in_features, out_features)
- dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None)
+ tmp_weight = layer.weight.clone()
+ dq_weight = _dequant_fp8_linear_weight(layer.weight, layer.weight_scale, block_size=None) # in-place operation
self.assertEqual(dq_weight.shape, (out_features, in_features))
self.assertEqual(dq_weight.dtype, torch.bfloat16)
- expected = layer.weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)
+ expected = tmp_weight.to(torch.bfloat16) * layer.weight_scale.to(torch.bfloat16)
torch.testing.assert_close(dq_weight, expected)
def test_per_channel_dequant(self):
@@ -50,14 +51,18 @@ class TestFP8ReQuant(unittest.TestCase):
# Per-channel scale (out_features)
scale = torch.randn(out_features)
- dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)
- expected = weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)
+ tmp_weight = weight.clone()
+ # in-place operation
+ dq_weight = _dequant_fp8_linear_weight(tmp_weight, scale, block_size=None)
+ expected = tmp_weight.to(torch.bfloat16) * scale.view(-1, 1).to(torch.bfloat16)
torch.testing.assert_close(dq_weight, expected)
# Per-channel scale (in_features)
scale = torch.randn(in_features)
- dq_weight = _dequant_fp8_linear_weight(weight, scale, block_size=None)
- expected = weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)
+ tmp_weight = weight.clone()
+ # in-place operation
+ dq_weight = _dequant_fp8_linear_weight(tmp_weight, scale, block_size=None)
+ expected = tmp_weight.to(torch.bfloat16) * scale.view(1, -1).to(torch.bfloat16)
torch.testing.assert_close(dq_weight, expected)
def test_devstral_model_structure(self):
@@ -87,6 +92,9 @@ class TestFP8ReQuant(unittest.TestCase):
self.data_type = "fp8"
mock_layer = FP8Linear(in_features, out_features)
+ # mock_layer will to meta after converting.
+ mock_weight = mock_layer.weight.clone()
+ mock_weight_scale = mock_layer.weight_scale.clone()
# This should now pass without AttributeError for 'block_size'
check_and_mark_quantized_module(mock_layer)
@@ -97,6 +105,6 @@ class TestFP8ReQuant(unittest.TestCase):
self.assertEqual(new_layer.out_features, out_features)
self.assertEqual(new_layer.weight.dtype, torch.bfloat16)
- expected = mock_layer.weight.to(torch.bfloat16) * mock_layer.weight_scale.to(torch.bfloat16)
+ expected = mock_weight.to(torch.bfloat16) * mock_weight_scale.to(torch.bfloat16)
torch.testing.assert_close(new_layer.weight, expected)
print("Devstral layer conversion successful!")