Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix Flux diffusion state propagation during block replay (#2086)

Wang, Chang3cae4b68addc
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.py
index ed0b46665..0e88ff3b4 100644
--- a/auto_round/algorithms/block_runner.py
+++ b/auto_round/algorithms/block_runner.py
@@ -123,6 +123,7 @@ class BlockForwardRunner:
self.shared_cache_keys = shared_cache_keys
self.output_config = output_config if output_config is not None else ["hidden_states"]
self.enable_torch_compile = enable_torch_compile
+ self.last_output_dict = None
self.block_forward = block_forward
if self.enable_torch_compile:
from auto_round.utils import compile_func
@@ -192,6 +193,8 @@ class BlockForwardRunner:
else:
device = inputs.device
+ self.last_output_dict = None
+ output_dict = {}
if indices is None:
indices = torch.arange(num_samples, dtype=torch.long, device=device)
elif not isinstance(indices, torch.Tensor):
@@ -205,10 +208,17 @@ class BlockForwardRunner:
batch_indices = indices[i : i + self.batch_size]
batch_inputs, batch_others = self._select_batch(inputs, input_others, batch_indices)
raw_output = self._forward_one_batch(block, batch_inputs, batch_others)
+ batch_output_dict = self._get_diffusion_output_dict(raw_output, block)
output = self._normalize_output(raw_output, block)
if is_returned_list and self.batch_size != 1: # split it to 1
+ if batch_output_dict:
+ for key, value in batch_output_dict.items():
+ output_dict.setdefault(key, []).extend(self.split_outputs(value))
output = self.split_outputs(output)
else:
+ if batch_output_dict:
+ for key, value in batch_output_dict.items():
+ output_dict.setdefault(key, []).append(value)
output = [output]
outputs.extend(output)
@@ -216,12 +226,26 @@ class BlockForwardRunner:
raise RuntimeError("BlockForwardRunner.forward: no outputs collected.")
if is_returned_list:
- return [o.to(out_device) for o in outputs]
+ result = [o.to(out_device) for o in outputs]
+ if output_dict:
+ self.last_output_dict = {key: [o.to(out_device) for o in values] for key, values in output_dict.items()}
+ self.last_output_dict["hidden_states"] = result
+ return result
else:
if self.batch_size == 1:
outputs = [output.unsqueeze(dim=self.batch_dim).to(out_device) for output in outputs]
+ if output_dict:
+ output_dict = {
+ key: [value.unsqueeze(dim=self.batch_dim).to(out_device) for value in values]
+ for key, values in output_dict.items()
+ }
outputs = torch.cat(outputs, dim=self.batch_dim).to(out_device)
+ if output_dict:
+ self.last_output_dict = {
+ key: torch.cat(values, dim=self.batch_dim).to(out_device) for key, values in output_dict.items()
+ }
+ self.last_output_dict["hidden_states"] = outputs
return outputs.to(out_device)
@@ -313,6 +337,23 @@ class BlockForwardRunner:
return first
raise TypeError(f"Block output[0] must be tensor, got {type(first).__name__}.")
+ def _get_diffusion_output_dict(
+ self, output: Any, block: "torch.nn.Module" = None
+ ) -> dict[str, torch.Tensor] | None:
+ if not self.is_diffusion or isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):
+ return None
+ block_cls_name = block.__class__.__name__ if block is not None else None
+ output_config = (
+ _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config) if block_cls_name else self.output_config
+ )
+ output_dict = {}
+ for idx, key in enumerate(output_config):
+ if idx >= len(output):
+ break
+ if isinstance(output[idx], torch.Tensor):
+ output_dict[key] = output[idx]
+ return output_dict or None
+
def _select_batch(self, inputs, input_others, indices):
"""Select a subset of inputs by indices."""
batch_dim = self.batch_dim
diff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.py
index 1ec3905be..bf399f814 100644
--- a/auto_round/algorithms/composer.py
+++ b/auto_round/algorithms/composer.py
@@ -394,11 +394,13 @@ class AlgorithmComposer:
pre.pre_quantize_block(block_ctx)
reference_output = None
+ reference_next_input = None
# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────
if fp_inputs is not None:
with torch.no_grad():
quant_hooks = self._get_fp_act_hooks(block)
reference_output = block_forward_fn(block, fp_inputs, input_others)
+ reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_output
for h in quant_hooks:
h.remove()
@@ -450,10 +452,11 @@ class AlgorithmComposer:
if self.block_quantizer.enable_quanted_input:
with torch.no_grad():
new_q_input = block_forward_fn(block, effective_input, input_others)
+ new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_input
else:
new_q_input = None
- return new_q_input, reference_output
+ return new_q_input, reference_next_input
def compress_layer_outside_block(
self,
diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.py
index 7f04b72e3..25306213a 100644
--- a/auto_round/calibration/inputs.py
+++ b/auto_round/calibration/inputs.py
@@ -34,10 +34,10 @@ def split_inputs(
Mirrors the original ``Compressor._split_inputs`` exactly:
- - For diffusion models, every key containing ``"hidden_state"`` is pulled
- out into a dict and returned as ``input_ids``; the remaining kwargs are
- returned as ``input_others``. Keys in *shared_cache_keys* are kept in
- ``input_others`` even if they match ``"hidden_state"``.
+ - For diffusion models, the block's primary ``hidden_states`` tensor is
+ pulled out and returned as ``input_ids``; auxiliary tensors such as
+ ``encoder_hidden_states`` remain in ``input_others`` so replayed block
+ forwards receive the same kwargs as the original model call.
- Otherwise, ``inputs[first_input_name]`` is popped and returned as
``input_ids`` (may be ``None``); the remainder is ``input_others``.
@@ -45,7 +45,7 @@ def split_inputs(
behaviour that downstream code relies on.
"""
if is_diffusion:
- input_id_str = [key for key in inputs.keys() if "hidden_state" in key and key not in shared_cache_keys]
+ input_id_str = [key for key in ("hidden_states",) if key in inputs and key not in shared_cache_keys]
input_ids = {k: inputs.pop(k, None) for k in input_id_str}
input_others = inputs
return input_ids, input_others
diff --git a/test/test_cpu/utils/test_calibration_inputs.py b/test/test_cpu/utils/test_calibration_inputs.py
index 87009c253..0dac59082 100644
--- a/test/test_cpu/utils/test_calibration_inputs.py
+++ b/test/test_cpu/utils/test_calibration_inputs.py
@@ -42,10 +42,10 @@ def test_split_inputs_diffusion():
input_ids, input_others = split_inputs(inputs, "ignored", is_diffusion=True)
- assert set(input_ids) == {"hidden_states", "encoder_hidden_states"}
+ assert set(input_ids) == {"hidden_states"}
assert "hidden_states" not in input_others
- assert "encoder_hidden_states" not in input_others
- assert set(input_others) == {"timestep"}
+ assert "encoder_hidden_states" in input_others
+ assert set(input_others) == {"encoder_hidden_states", "timestep"}
def test_preprocess_block_inputs_casts_tensors(monkeypatch):
@@ -54,6 +54,7 @@ def test_preprocess_block_inputs_casts_tensors(monkeypatch):
"auto_round.calibration.inputs.clear_memory",
lambda device_list: calls.append(tuple(device_list)),
)
+ monkeypatch.setattr("auto_round.calibration.inputs.device_manager", SimpleNamespace(device_list=["cpu"]))
model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)
compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])