Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

[Feature] Register_block_output to support GLM5.2 tuning (#2104)

Xin Hed05fb638a813
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.py
index 60f9b87ab..b2771011d 100644
--- a/auto_round/algorithms/block_runner.py
+++ b/auto_round/algorithms/block_runner.py
@@ -42,6 +42,9 @@ if TYPE_CHECKING:
#: Register new diffusion architectures with :func:`register_diffusion_output`.
_DIFFUSION_OUTPUT_REGISTRY: dict[str, list[str]] = {}
+#: Maps decoder block class name to output keys that become the next block's inputs.
+_BLOCK_OUTPUT_REGISTRY: dict[str, list[str]] = {}
+
def register_diffusion_output(block_cls_name: str, output_keys: list[str]) -> None:
"""Register the output key order for a diffusion transformer block class.
@@ -60,6 +63,11 @@ def register_diffusion_output(block_cls_name: str, output_keys: list[str]) -> No
_DIFFUSION_OUTPUT_REGISTRY[block_cls_name] = output_keys
+def register_block_output(block_cls_name: str, output_keys: list[str]) -> None:
+ """Register tuple output keys that should be forwarded to the next block."""
+ _BLOCK_OUTPUT_REGISTRY[block_cls_name] = output_keys
+
+
# Built-in diffusion block registrations.
# Add new architectures here instead of editing BlockRunner internals.
register_diffusion_output("FluxTransformerBlock", ["encoder_hidden_states", "hidden_states"])
@@ -69,6 +77,9 @@ register_diffusion_output("OvisImageSingleTransformerBlock", ["encoder_hidden_st
register_diffusion_output("StableAudioDiTBlock", ["hidden_states"])
register_diffusion_output("WanTransformerBlock", ["hidden_states"])
+# GLM DSA full indexer layers return top-k indices for subsequent shared layers.
+register_block_output("GlmMoeDsaDecoderLayer", ["hidden_states", "prev_topk_indices"])
+
# ---------------------------------------------------------------------------
# BlockForwardRunner
@@ -208,7 +219,7 @@ class BlockForwardRunner:
batch_indices = indices[i : i + self.batch_size]
batch_inputs, batch_others = self._select_batch(inputs, input_others, batch_indices)
raw_output = self._forward_one_batch(block, batch_inputs, batch_others)
- batch_output_dict = self._get_diffusion_output_dict(raw_output, block)
+ batch_output_dict = self._get_output_dict(raw_output, block)
output = self._normalize_output(raw_output, block)
if is_returned_list and self.batch_size != 1: # split it to 1
if batch_output_dict:
@@ -292,7 +303,7 @@ class BlockForwardRunner:
self.amp,
self.amp_dtype,
self.device,
- 0,
+ None,
)
def _count_samples(self, inputs: Any) -> int:
@@ -337,15 +348,20 @@ class BlockForwardRunner:
return first
raise TypeError(f"Block output[0] must be tensor, got {type(first).__name__}.")
- def _get_diffusion_output_dict(
- self, output: Any, block: "torch.nn.Module" = None
- ) -> dict[str, torch.Tensor] | None:
- if not self.is_diffusion or isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):
+ def _get_output_dict(self, output: Any, block: "torch.nn.Module" = None) -> dict[str, torch.Tensor] | None:
+ if isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):
return None
block_cls_name = block.__class__.__name__ if block is not None else None
- output_config = (
- _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config) if block_cls_name else self.output_config
- )
+ if self.is_diffusion:
+ output_config = (
+ _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config)
+ if block_cls_name
+ else self.output_config
+ )
+ else:
+ output_config = _BLOCK_OUTPUT_REGISTRY.get(block_cls_name)
+ if output_config is None:
+ return None
output_dict = {}
for idx, key in enumerate(output_config):
if idx >= len(output):
diff --git a/auto_round/utils/device.py b/auto_round/utils/device.py
index c0b809388..dcf2bf6d2 100644
--- a/auto_round/utils/device.py
+++ b/auto_round/utils/device.py
@@ -813,15 +813,14 @@ def get_moe_memory_ratio(block: torch.nn.Module) -> float:
return 1.0, False # Default ratio for non-MoE models
-def estimate_tuning_block_mem(
- block: torch.nn.Module, input_ids: list[torch.Tensor], batch_size: int
-) -> tuple[dict, float]:
+def estimate_tuning_block_mem(block: torch.nn.Module, input_ids: Any, batch_size: int) -> tuple[dict, float]:
"""
Calculates the memory consumption of a specific block in the model.
Args:
block (torch.nn.Module): The block of the model to analyze.
- input_ids (list[torch.Tensor]): A list of input tensors for the block.
+ input_ids: Cached block inputs as a tensor, a list of tensors, or a dictionary containing
+ ``hidden_states`` and auxiliary block inputs.
batch_size (int): Number of samples to consider for memory estimation.
Returns:
@@ -837,9 +836,23 @@ def estimate_tuning_block_mem(
layer_memory_dict = {}
- # Calculate batch_size and sequence_length from input_ids for output memory estimation
- seq_len = input_ids[0].shape[1] if input_ids and len(input_ids[0].shape) >= 2 else 1
- element_size = input_ids[0].element_size() if input_ids else 2 # Default to 2 bytes (fp16/bf16)
+ def iter_input_tensors(value):
+ if isinstance(value, torch.Tensor):
+ yield value
+ elif isinstance(value, dict):
+ for nested_value in value.values():
+ yield from iter_input_tensors(nested_value)
+ elif isinstance(value, (list, tuple)):
+ for nested_value in value:
+ yield from iter_input_tensors(nested_value)
+
+ input_tensors = list(iter_input_tensors(input_ids))
+ hidden_states = input_ids.get("hidden_states") if isinstance(input_ids, dict) else input_ids
+ reference_tensor = next(iter_input_tensors(hidden_states), input_tensors[0] if input_tensors else None)
+
+ # Calculate sequence length and dtype from hidden states for output memory estimation.
+ seq_len = reference_tensor.shape[1] if reference_tensor is not None and reference_tensor.ndim >= 2 else 1
+ element_size = reference_tensor.element_size() if reference_tensor is not None else 2
moe_ratio, has_moe = get_moe_memory_ratio(block) # Get MoE memory ratio (1.0 for non-MoE models)
@@ -880,7 +893,7 @@ def estimate_tuning_block_mem(
}
# Assuming bfloat16 or float32, input and output
- block_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_ids) / 1024**3
+ block_input_output_memory = 2 * sum(tensor.nbytes for tensor in input_tensors) / 1024**3
# Roughly estimate additional memory for attention and other operations
# For MoE expert layers, multiply activation memory by the ratio of active experts
@@ -986,6 +999,12 @@ def set_auto_device_map_for_block_with_tuning(
card_1_left_memory = max(0, device_1_memory - loss_memory) if card_0_in_high_risk else device_1_memory
loss_device = device_1 if card_0_in_high_risk else output_device
+ if not layer_memory_dict:
+ output_device = device_0 if output_device is None else output_device
+ block.to(output_device)
+ logger.debug(f"No layers require tuning; moved the block to {output_device}")
+ return card_0_in_high_risk, loss_device
+
# Calculate total available memory across all devices
total_available_memory = card_0_left_memory + card_1_left_memory
for i in range(2, len(gpu_devices)):
diff --git a/test/test_cpu/algorithms/test_block_runner.py b/test/test_cpu/algorithms/test_block_runner.py
new file mode 100644
index 000000000..394799e99
--- /dev/null
+++ b/test/test_cpu/algorithms/test_block_runner.py
@@ -0,0 +1,66 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+
+from auto_round.algorithms.block_runner import BlockForwardRunner
+
+
+class GlmMoeDsaDecoderLayer(torch.nn.Module):
+ def __init__(self, shared=False):
+ super().__init__()
+ self.shared = shared
+
+ def forward(self, hidden_states, prev_topk_indices=None):
+ if self.shared:
+ assert prev_topk_indices is not None
+ expected_indices = (hidden_states[..., :1] - 1).to(torch.long)
+ torch.testing.assert_close(prev_topk_indices, expected_indices)
+ topk_indices = prev_topk_indices
+ else:
+ topk_indices = hidden_states[..., :1].to(torch.long)
+ return hidden_states + 1, topk_indices
+
+
+class TupleOutputBlock(torch.nn.Module):
+ def forward(self, hidden_states):
+ return hidden_states + 1, hidden_states + 2
+
+
+def test_glm_dsa_topk_indices_propagate_between_blocks():
+ runner = BlockForwardRunner(batch_size=2, device="cpu", cache_device="cpu", amp=False)
+ inputs = [torch.full((1, 3, 2), value, dtype=torch.float32) for value in range(3)]
+
+ reference_outputs = runner(GlmMoeDsaDecoderLayer(), inputs, {})
+ next_inputs = runner.last_output_dict
+
+ assert next_inputs is not None
+ assert set(next_inputs) == {"hidden_states", "prev_topk_indices"}
+ assert next_inputs["hidden_states"] is reference_outputs
+
+ shared_outputs = runner(GlmMoeDsaDecoderLayer(shared=True), next_inputs, {})
+
+ assert len(shared_outputs) == len(inputs)
+ assert runner.last_output_dict is not None
+
+
+def test_unregistered_tuple_output_keeps_first_tensor_behavior():
+ runner = BlockForwardRunner(batch_size=2, device="cpu", cache_device="cpu", amp=False)
+ inputs = [torch.zeros((1, 2, 2)), torch.ones((1, 2, 2))]
+
+ outputs = runner(TupleOutputBlock(), inputs, {})
+
+ assert runner.last_output_dict is None
+ for output, input_tensor in zip(outputs, inputs):
+ torch.testing.assert_close(output, input_tensor + 1)