PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[Feature] Register_block_output to support GLM5.2 tuning (#2104)
Xin Hed05fb638a813
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.pyindex 60f9b87ab..b2771011d 100644--- a/auto_round/algorithms/block_runner.py+++ b/auto_round/algorithms/block_runner.py@@ -42,6 +42,9 @@ if TYPE_CHECKING:#: Register new diffusion architectures with :func:`register_diffusion_output`._DIFFUSION_OUTPUT_REGISTRY: dict[str, list[str]] = {}+#: Maps decoder block class name to output keys that become the next block's inputs.+_BLOCK_OUTPUT_REGISTRY: dict[str, list[str]] = {}+def register_diffusion_output(block_cls_name: str, output_keys: list[str]) -> None:"""Register the output key order for a diffusion transformer block class.@@ -60,6 +63,11 @@ def register_diffusion_output(block_cls_name: str, output_keys: list[str]) -> No_DIFFUSION_OUTPUT_REGISTRY[block_cls_name] = output_keys+def register_block_output(block_cls_name: str, output_keys: list[str]) -> None:+ """Register tuple output keys that should be forwarded to the next block."""+ _BLOCK_OUTPUT_REGISTRY[block_cls_name] = output_keys++# Built-in diffusion block registrations.# Add new architectures here instead of editing BlockRunner internals.register_diffusion_output("FluxTransformerBlock", ["encoder_hidden_states", "hidden_states"])@@ -69,6 +77,9 @@ register_diffusion_output("OvisImageSingleTransformerBlock", ["encoder_hidden_stregister_diffusion_output("StableAudioDiTBlock", ["hidden_states"])register_diffusion_output("WanTransformerBlock", ["hidden_states"])+# GLM DSA full indexer layers return top-k indices for subsequent shared layers.+register_block_output("GlmMoeDsaDecoderLayer", ["hidden_states", "prev_topk_indices"])+# ---------------------------------------------------------------------------# BlockForwardRunner@@ -208,7 +219,7 @@ class BlockForwardRunner:batch_indices = indices[i : i + self.batch_size]batch_inputs, batch_others = self._select_batch(inputs, input_others, batch_indices)raw_output = self._forward_one_batch(block, batch_inputs, batch_others)- batch_output_dict = self._get_diffusion_output_dict(raw_output, block)+ batch_output_dict = self._get_output_dict(raw_output, block)output = self._normalize_output(raw_output, block)if is_returned_list and self.batch_size != 1: # split it to 1if batch_output_dict:@@ -292,7 +303,7 @@ class BlockForwardRunner:self.amp,self.amp_dtype,self.device,- 0,+ None,)def _count_samples(self, inputs: Any) -> int:@@ -337,15 +348,20 @@ class BlockForwardRunner:return firstraise TypeError(f"Block output[0] must be tensor, got {type(first).__name__}.")- def _get_diffusion_output_dict(- self, output: Any, block: "torch.nn.Module" = None- ) -> dict[str, torch.Tensor] | None:- if not self.is_diffusion or isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):+ def _get_output_dict(self, output: Any, block: "torch.nn.Module" = None) -> dict[str, torch.Tensor] | None:+ if isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):return Noneblock_cls_name = block.__class__.__name__ if block is not None else None- output_config = (- _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config) if block_cls_name else self.output_config- )+ if self.is_diffusion:+ output_config = (+ _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config)+ if block_cls_name+ else self.output_config+ )+ else:+ output_config = _BLOCK_OUTPUT_REGISTRY.get(block_cls_name)+ if output_config is None:+ return Noneoutput_dict = {}for idx, key in enumerate(output_config):if idx >= len(output):diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex c0b809388..dcf2bf6d2 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -813,15 +813,14 @@ def get_moe_memory_ratio(block: torch.nn.Module) -> float:return 1.0, False # Default ratio for non-MoE models-def estimate_tuning_block_mem(- block: torch.nn.Module, input_ids: list[torch.Tensor], batch_size: int-) -> tuple[dict, float]:+def estimate_tuning_block_mem(block: torch.nn.Module, input_ids: Any, batch_size: int) -> tuple[dict, float]:"""Calculates the memory consumption of a specific block in the model.Args:block (torch.nn.Module): The block of the model to analyze.- input_ids (list[torch.Tensor]): A list of input tensors for the block.+ input_ids: Cached block inputs as a tensor, a list of tensors, or a dictionary containing+ ``hidden_states`` and auxiliary block inputs.batch_size (int): Number of samples to consider for memory estimation.Returns:@@ -837,9 +836,23 @@ def estimate_tuning_block_mem(layer_memory_dict = {}- # Calculate batch_size and sequence_length from input_ids for output memory estimation- seq_len = input_ids[0].shape[1] if input_ids and len(input_ids[0].shape) >= 2 else 1- element_size = input_ids[0].element_size() if input_ids else 2 # Default to 2 bytes (fp16/bf16)+ def iter_input_tensors(value):+ if isinstance(value, torch.Tensor):+ yield value+ elif isinstance(value, dict):+ for nested_value in value.values():+ yield from iter_input_tensors(nested_value)+ elif isinstance(value, (list, tuple)):+ for nested_value in value:+ yield from iter_input_tensors(nested_value)++ input_tensors = list(iter_input_tensors(input_ids))+ hidden_states = input_ids.get("hidden_states") if isinstance(input_ids, dict) else input_ids+ reference_tensor = next(iter_input_tensors(hidden_states), input_tensors[0] if input_tensors else None)++ # Calculate sequence length and dtype from hidden states for output memory estimation.+ seq_len = reference_tensor.shape[1] if reference_tensor is not None and reference_tensor.ndim >= 2 else 1+ element_size = reference_tensor.element_size() if reference_tensor is not None else 2moe_ratio, has_moe = get_moe_memory_ratio(block) # Get MoE memory ratio (1.0 for non-MoE models)@@ -880,7 +893,7 @@ def estimate_tuning_block_mem(}# Assuming bfloat16 or float32, input and output- block_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_ids) / 1024**3+ block_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_tensors) / 1024**3# Roughly estimate additional memory for attention and other operations# For MoE expert layers, multiply activation memory by the ratio of active experts@@ -986,6 +999,12 @@ def set_auto_device_map_for_block_with_tuning(card_1_left_memory = max(0, device_1_memory - loss_memory) if card_0_in_high_risk else device_1_memoryloss_device = device_1 if card_0_in_high_risk else output_device+ if not layer_memory_dict:+ output_device = device_0 if output_device is None else output_device+ block.to(output_device)+ logger.debug(f"No layers require tuning; moved the block to {output_device}")+ return card_0_in_high_risk, loss_device+# Calculate total available memory across all devicestotal_available_memory = card_0_left_memory + card_1_left_memoryfor i in range(2, len(gpu_devices)):diff --git a/test/test_cpu/algorithms/test_block_runner.py b/test/test_cpu/algorithms/test_block_runner.pynew file mode 100644index 000000000..394799e99--- /dev/null+++ b/test/test_cpu/algorithms/test_block_runner.py@@ -0,0 +1,66 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import torch++from auto_round.algorithms.block_runner import BlockForwardRunner+++class GlmMoeDsaDecoderLayer(torch.nn.Module):+ def __init__(self, shared=False):+ super().__init__()+ self.shared = shared++ def forward(self, hidden_states, prev_topk_indices=None):+ if self.shared:+ assert prev_topk_indices is not None+ expected_indices = (hidden_states[..., :1] - 1).to(torch.long)+ torch.testing.assert_close(prev_topk_indices, expected_indices)+ topk_indices = prev_topk_indices+ else:+ topk_indices = hidden_states[..., :1].to(torch.long)+ return hidden_states + 1, topk_indices+++class TupleOutputBlock(torch.nn.Module):+ def forward(self, hidden_states):+ return hidden_states + 1, hidden_states + 2+++def test_glm_dsa_topk_indices_propagate_between_blocks():+ runner = BlockForwardRunner(batch_size=2, device="cpu", cache_device="cpu", amp=False)+ inputs = [torch.full((1, 3, 2), value, dtype=torch.float32) for value in range(3)]++ reference_outputs = runner(GlmMoeDsaDecoderLayer(), inputs, {})+ next_inputs = runner.last_output_dict++ assert next_inputs is not None+ assert set(next_inputs) == {"hidden_states", "prev_topk_indices"}+ assert next_inputs["hidden_states"] is reference_outputs++ shared_outputs = runner(GlmMoeDsaDecoderLayer(shared=True), next_inputs, {})++ assert len(shared_outputs) == len(inputs)+ assert runner.last_output_dict is not None+++def test_unregistered_tuple_output_keeps_first_tensor_behavior():+ runner = BlockForwardRunner(batch_size=2, device="cpu", cache_device="cpu", amp=False)+ inputs = [torch.zeros((1, 2, 2)), torch.ones((1, 2, 2))]++ outputs = runner(TupleOutputBlock(), inputs, {})++ assert runner.last_output_dict is None+ for output, input_tensor in zip(outputs, inputs):+ torch.testing.assert_close(output, input_tensor + 1)