PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Fix Flux diffusion state propagation during block replay (#2086)
Wang, Chang3cae4b68addc
diff --git a/auto_round/algorithms/block_runner.py b/auto_round/algorithms/block_runner.pyindex ed0b46665..0e88ff3b4 100644--- a/auto_round/algorithms/block_runner.py+++ b/auto_round/algorithms/block_runner.py@@ -123,6 +123,7 @@ class BlockForwardRunner:self.shared_cache_keys = shared_cache_keysself.output_config = output_config if output_config is not None else ["hidden_states"]self.enable_torch_compile = enable_torch_compile+ self.last_output_dict = Noneself.block_forward = block_forwardif self.enable_torch_compile:from auto_round.utils import compile_func@@ -192,6 +193,8 @@ class BlockForwardRunner:else:device = inputs.device+ self.last_output_dict = None+ output_dict = {}if indices is None:indices = torch.arange(num_samples, dtype=torch.long, device=device)elif not isinstance(indices, torch.Tensor):@@ -205,10 +208,17 @@ class BlockForwardRunner:batch_indices = indices[i : i + self.batch_size]batch_inputs, batch_others = self._select_batch(inputs, input_others, batch_indices)raw_output = self._forward_one_batch(block, batch_inputs, batch_others)+ batch_output_dict = self._get_diffusion_output_dict(raw_output, block)output = self._normalize_output(raw_output, block)if is_returned_list and self.batch_size != 1: # split it to 1+ if batch_output_dict:+ for key, value in batch_output_dict.items():+ output_dict.setdefault(key, []).extend(self.split_outputs(value))output = self.split_outputs(output)else:+ if batch_output_dict:+ for key, value in batch_output_dict.items():+ output_dict.setdefault(key, []).append(value)output = [output]outputs.extend(output)@@ -216,12 +226,26 @@ class BlockForwardRunner:raise RuntimeError("BlockForwardRunner.forward: no outputs collected.")if is_returned_list:- return [o.to(out_device) for o in outputs]+ result = [o.to(out_device) for o in outputs]+ if output_dict:+ self.last_output_dict = {key: [o.to(out_device) for o in values] for key, values in output_dict.items()}+ self.last_output_dict["hidden_states"] = result+ return resultelse:if self.batch_size == 1:outputs = [output.unsqueeze(dim=self.batch_dim).to(out_device) for output in outputs]+ if output_dict:+ output_dict = {+ key: [value.unsqueeze(dim=self.batch_dim).to(out_device) for value in values]+ for key, values in output_dict.items()+ }outputs = torch.cat(outputs, dim=self.batch_dim).to(out_device)+ if output_dict:+ self.last_output_dict = {+ key: torch.cat(values, dim=self.batch_dim).to(out_device) for key, values in output_dict.items()+ }+ self.last_output_dict["hidden_states"] = outputsreturn outputs.to(out_device)@@ -313,6 +337,23 @@ class BlockForwardRunner:return firstraise TypeError(f"Block output[0] must be tensor, got {type(first).__name__}.")+ def _get_diffusion_output_dict(+ self, output: Any, block: "torch.nn.Module" = None+ ) -> dict[str, torch.Tensor] | None:+ if not self.is_diffusion or isinstance(output, torch.Tensor) or not isinstance(output, (tuple, list)):+ return None+ block_cls_name = block.__class__.__name__ if block is not None else None+ output_config = (+ _DIFFUSION_OUTPUT_REGISTRY.get(block_cls_name, self.output_config) if block_cls_name else self.output_config+ )+ output_dict = {}+ for idx, key in enumerate(output_config):+ if idx >= len(output):+ break+ if isinstance(output[idx], torch.Tensor):+ output_dict[key] = output[idx]+ return output_dict or None+def _select_batch(self, inputs, input_others, indices):"""Select a subset of inputs by indices."""batch_dim = self.batch_dimdiff --git a/auto_round/algorithms/composer.py b/auto_round/algorithms/composer.pyindex 1ec3905be..bf399f814 100644--- a/auto_round/algorithms/composer.py+++ b/auto_round/algorithms/composer.py@@ -394,11 +394,13 @@ class AlgorithmComposer:pre.pre_quantize_block(block_ctx)reference_output = None+ reference_next_input = None# ── Step 3: Quantizer calibration (act_max, imatrix, etc.) ─────────────if fp_inputs is not None:with torch.no_grad():quant_hooks = self._get_fp_act_hooks(block)reference_output = block_forward_fn(block, fp_inputs, input_others)+ reference_next_input = getattr(block_forward_fn, "last_output_dict", None) or reference_outputfor h in quant_hooks:h.remove()@@ -450,10 +452,11 @@ class AlgorithmComposer:if self.block_quantizer.enable_quanted_input:with torch.no_grad():new_q_input = block_forward_fn(block, effective_input, input_others)+ new_q_input = getattr(block_forward_fn, "last_output_dict", None) or new_q_inputelse:new_q_input = None- return new_q_input, reference_output+ return new_q_input, reference_next_inputdef compress_layer_outside_block(self,diff --git a/auto_round/calibration/inputs.py b/auto_round/calibration/inputs.pyindex 7f04b72e3..25306213a 100644--- a/auto_round/calibration/inputs.py+++ b/auto_round/calibration/inputs.py@@ -34,10 +34,10 @@ def split_inputs(Mirrors the original ``Compressor._split_inputs`` exactly:- - For diffusion models, every key containing ``"hidden_state"`` is pulled- out into a dict and returned as ``input_ids``; the remaining kwargs are- returned as ``input_others``. Keys in *shared_cache_keys* are kept in- ``input_others`` even if they match ``"hidden_state"``.+ - For diffusion models, the block's primary ``hidden_states`` tensor is+ pulled out and returned as ``input_ids``; auxiliary tensors such as+ ``encoder_hidden_states`` remain in ``input_others`` so replayed block+ forwards receive the same kwargs as the original model call.- Otherwise, ``inputs[first_input_name]`` is popped and returned as``input_ids`` (may be ``None``); the remainder is ``input_others``.@@ -45,7 +45,7 @@ def split_inputs(behaviour that downstream code relies on."""if is_diffusion:- input_id_str = [key for key in inputs.keys() if "hidden_state" in key and key not in shared_cache_keys]+ input_id_str = [key for key in ("hidden_states",) if key in inputs and key not in shared_cache_keys]input_ids = {k: inputs.pop(k, None) for k in input_id_str}input_others = inputsreturn input_ids, input_othersdiff --git a/test/test_cpu/utils/test_calibration_inputs.py b/test/test_cpu/utils/test_calibration_inputs.pyindex 87009c253..0dac59082 100644--- a/test/test_cpu/utils/test_calibration_inputs.py+++ b/test/test_cpu/utils/test_calibration_inputs.py@@ -42,10 +42,10 @@ def test_split_inputs_diffusion():input_ids, input_others = split_inputs(inputs, "ignored", is_diffusion=True)- assert set(input_ids) == {"hidden_states", "encoder_hidden_states"}+ assert set(input_ids) == {"hidden_states"}assert "hidden_states" not in input_others- assert "encoder_hidden_states" not in input_others- assert set(input_others) == {"timestep"}+ assert "encoder_hidden_states" in input_others+ assert set(input_others) == {"encoder_hidden_states", "timestep"}def test_preprocess_block_inputs_casts_tensors(monkeypatch):@@ -54,6 +54,7 @@ def test_preprocess_block_inputs_casts_tensors(monkeypatch):"auto_round.calibration.inputs.clear_memory",lambda device_list: calls.append(tuple(device_list)),)+ monkeypatch.setattr("auto_round.calibration.inputs.device_manager", SimpleNamespace(device_list=["cpu"]))model_context = SimpleNamespace(is_diffusion=False, amp=True, amp_dtype=torch.bfloat16)compress_context = SimpleNamespace(cache_device=torch.device("cpu"), device_list=["cpu"])