PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add opt-in diffusion tuning cache prefetch (#2340)
Wang, Chang585d20653215
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.pyindex 37bd72aeb..df70764db 100644--- a/auto_round/algorithms/quantization/sign_round/quantizer.py+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py@@ -473,61 +473,107 @@ class SignRoundQuantizer(BaseQuantizer):else None)- for i in range(self.iters):- if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):- for n, m in block.named_modules():- m.cur_iter = i- total_loss = 0- global_indices = index_sampler.next_batch()- if valid_token_mask:- num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)-- for batch_start in range(0, len(global_indices), batch_size):- indices = global_indices[batch_start : batch_start + batch_size]- ref_output = torch.cat([fp_outputs[i] for i in indices], dim=0).to(loss_device)- pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)- if loss_device is not None:- pred_output = pred_output.to(loss_device)- if (- block_ctx.block_index == block_ctx.block_cnt - 1- and self.enable_lfq- and input_ids is not None- and self._is_text_decoder_block(block_ctx.block_name)- ):- loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))- else:- loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)- num_elm = 1 if num_elm <= 0 else num_elm- total_loss += loss.item() / num_elm-- if mid_iter_mem_check:- # clear memory to avoid OOM due to memory fragmentation- clear_memory_if_reached_threshold(threshold=0.5, device_list=device_manager.device_list)-- self._scale_loss_and_backward(scaler, loss)-- if mid_iter_mem_check:- # clear memory to avoid OOM due to memory fragmentation- clear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)+ tuning_cache = None+ # Only opt-in diffusion tuning can enter the CUDA staging path.+ cache_budget = getattr(self.model_context, "diffusion_tuning_cache_size", 0)+ use_tuning_cache = (+ getattr(self.model_context, "is_diffusion", False)+ and (cache_budget == "auto" or cache_budget > 0)+ and self.compress_context.low_gpu_mem_usage+ and str(device).startswith("cuda")+ and len(device_manager.device_list) == 1+ and (loss_device is None or torch.device(loss_device) == torch.device(device))+ )- if i == 0:- init_loss = total_loss- current_lr = optimizer.param_groups[0]["lr"]- logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)+ try:+ for i in range(self.iters):+ # Auto observes a complete forward/backward/optimizer iteration+ # on the legacy path before allocating any extra GPU buffers.+ if use_tuning_cache and i == (1 if cache_budget == "auto" else 0):+ from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache++ tuning_cache = DiffusionTuningCache.create(+ block,+ block_fwd,+ active_inputs,+ input_others,+ fp_outputs,+ index_sampler,+ self.iters - i,+ cache_budget,+ device,+ )+ if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):+ for n, m in block.named_modules():+ m.cur_iter = i+ total_loss = 0+ global_indices = index_sampler.next_batch()+ if valid_token_mask:+ num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)++ for batch_start in range(0, len(global_indices), batch_size):+ indices = global_indices[batch_start : batch_start + batch_size]+ staged = tuning_cache.get(indices) if tuning_cache is not None else None+ if staged is None:+ ref_output = torch.cat([fp_outputs[i] for i in indices], dim=0).to(loss_device)+ pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)+ else:+ ref_output = staged[2]+ pred_output = tuning_cache.forward(block, staged, _fwd_cache_device)+ if loss_device is not None:+ pred_output = pred_output.to(loss_device)+ if (+ block_ctx.block_index == block_ctx.block_cnt - 1+ and self.enable_lfq+ and input_ids is not None+ and self._is_text_decoder_block(block_ctx.block_name)+ ):+ loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))+ else:+ loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)+ num_elm = 1 if num_elm <= 0 else num_elm+ total_loss += loss.item() / num_elm++ if mid_iter_mem_check:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.5, device_list=device_manager.device_list)++ self._scale_loss_and_backward(scaler, loss)++ if mid_iter_mem_check:+ # clear memory to avoid OOM due to memory fragmentation+ clear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)++ if i == 0:+ init_loss = total_loss+ current_lr = optimizer.param_groups[0]["lr"]+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)++ if total_loss < best_loss:+ best_loss = total_loss+ if not self.not_use_best_mse:+ best_params = (+ tuning_cache.collect_best_params()+ if tuning_cache is not None and tuning_cache.best is not None+ else collect_best_params(block, self.compress_context.cache_device)+ )+ last_best_iter = i+ if self.not_use_best_mse and i == self.iters - 1:+ best_params = (+ tuning_cache.collect_best_params()+ if tuning_cache is not None and tuning_cache.best is not None+ else collect_best_params(block, self.compress_context.cache_device)+ )- if total_loss < best_loss:- best_loss = total_lossif not self.not_use_best_mse:- best_params = collect_best_params(block, self.compress_context.cache_device)- last_best_iter = i- if self.not_use_best_mse and i == self.iters - 1:- best_params = collect_best_params(block, self.compress_context.cache_device)-- if not self.not_use_best_mse:- if 0 < self.dynamic_max_gap <= i - last_best_iter:- break- sync_gradients()- self._step(scaler, optimizer, lr_schedule)+ if 0 < self.dynamic_max_gap <= i - last_best_iter:+ break+ sync_gradients()+ self._step(scaler, optimizer, lr_schedule)++ finally:+ if tuning_cache is not None:+ tuning_cache.close()last_loss = total_lossbest_iter = self.itersdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 6b96ceff4..203966cb6 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -355,6 +355,7 @@ _ENTRY_KWARG_OWNERS = {"num_inference_steps": "diffusion","calib_num_inference_steps": "diffusion","generator_seed": "diffusion",+ "diffusion_tuning_cache_size": "diffusion",}_SCHEME_FIELDS = set(QuantizationScheme.get_attributes())diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex 0e601547f..8441bf5bb 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -99,6 +99,7 @@ def _build_entry_model_type_kwargs(args) -> dict:"num_inference_steps": args.num_inference_steps,"calib_num_inference_steps": args.calib_num_inference_steps,"generator_seed": args.generator_seed,+ "diffusion_tuning_cache_size": args.diffusion_tuning_cache_size,}diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex dbab03181..40f0f4559 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -296,6 +296,14 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Arguhelp="Number of denoising steps for diffusion generation/evaluation.",)diff.add_argument("--generator_seed", default=None, type=int, help="Random seed used for diffusion generation.")+ diff.add_argument(+ "--diffusion_tuning_cache_size",+ default=0,+ type=lambda value: value if value == "auto" else float(value),+ help="Extra GPU buffer budget in GiB for single-CUDA diffusion SignRound prefetch with low_gpu_mem_usage. "+ "Use 'auto' to select a budget after the first tuning iteration; 0 preserves the existing path. "+ "This is not a limit on total GPU memory.",+ )# ---- Common Quantization Arguments ----quant_group = parser.add_argument_group("Common Quantization Arguments")diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.mdindex 4d2dcc388..91a177bf1 100644--- a/auto_round/compressors/diffusion/README.md+++ b/auto_round/compressors/diffusion/README.md@@ -42,6 +42,9 @@ autoround.quantize_and_save(output_dir, format="fake", inplace=True)control the calibration schedule.- `guidance_scale`: controls how much the image generation process follows the text prompt.- `generator_seed`: a seed that controls the initial noise from which an image is generated.+- `diffusion_tuning_cache_size`: opt-in extra GPU buffer budget in GiB for diffusion SignRound prefetch.+ It only takes effect with `low_gpu_mem_usage=True` on single-GPU SignRound flow (`enable_quanted_input=False`+ and no custom `layer_config`). This value reserves additional temporary buffers and is not a cap on total VRAM.For more hyperparameters, refer to [Homepage Detailed Hyperparameters](../../../README.md#quantization-scheme--configuration).@@ -57,9 +60,15 @@ auto-round \--batch_size 1 \--dataset coco2014 \--calib_num_inference_steps 8 \+ --low_gpu_mem_usage \+ --diffusion_tuning_cache_size auto \--output_dir ./tmp_autoround```+Use `--diffusion_tuning_cache_size` only when `--low_gpu_mem_usage` is enabled on single-GPU SignRound+(`--enable_quanted_input` disabled and no custom `--layer_config`). The GiB value controls extra prefetch buffers,+not total GPU memory.+### Diffusion Support MatrixFor diffusion models, currently we validate quantization on the following models, which involves quantizing the transformer component of the pipeline.diff --git a/auto_round/compressors/diffusion/tuning_cache.py b/auto_round/compressors/diffusion/tuning_cache.pynew file mode 100644index 000000000..a3f9b55f4--- /dev/null+++ b/auto_round/compressors/diffusion/tuning_cache.py@@ -0,0 +1,236 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Opt-in, bounded CUDA staging for diffusion SignRound batches."""++import queue+import random+import threading++import torch+from torch.utils._pytree import tree_flatten, tree_map++from auto_round.logger import logger+++def _batch_plan(sampler, iters, batch_size):+ """Forecast the cyclic sampler without advancing it or the global RNG."""+ rng = random.Random()+ rng.setstate(random.getstate())+ indices, position = list(sampler.indices), sampler.index+ for _ in range(iters):+ if position + sampler.batch_size > sampler.nsamples:+ rng.shuffle(indices)+ position = 0+ selected = indices[position : position + sampler.batch_size]+ position += sampler.batch_size+ for start in range(0, len(selected), batch_size):+ yield selected[start : start + batch_size]+++def _signature(batch):+ leaves, spec = tree_flatten(batch)+ signature = []+ for value in leaves:+ if isinstance(value, torch.Tensor):+ if value.device.type != "cpu" or value.layout != torch.strided:+ return None+ signature.append((value.shape, value.dtype))+ elif value is None or isinstance(value, (str, bool, int, float)):+ signature.append((type(value), value))+ else:+ return None+ return spec, signature+++def _nbytes(batch):+ return sum(t.numel() * t.element_size() for t in tree_flatten(batch)[0] if isinstance(t, torch.Tensor))+++def _auto_budget_bytes(free_bytes):+ # Keep the warmed allocator pool for activations/workspace, and leave at+ # least half the unreserved memory (and at least 1 GiB) as additional margin.+ return max(0, min(free_bytes // 2, free_bytes - 2**30))+++class DiffusionTuningCache:+ """Two reusable host/device slots, with optional in-place best snapshots.++ The budget covers these persistent CUDA buffers only, not the training+ activations/workspace. Pinned host memory is bounded by the two batch slots.+ The caller must close the cache after backward, including on early exit.+ """++ @classmethod+ def create(cls, block, runner, inputs, others, outputs, sampler, iters, budget_gib, device):+ if runner.batch_size != 1 or not isinstance(inputs, dict) or iters <= 0:+ return None+ cache = cls()+ cache.runner, cache.inputs, cache.others, cache.outputs = runner, inputs, others, outputs+ cache.device = torch.device(device)+ cache.plan = iter(_batch_plan(sampler, iters, runner.batch_size))+ # Start the generator now, before the original sampler advances.+ first = next(cache.plan)+ template = cache._select(first)+ cache.signature = _signature(template)+ if cache.signature is None:+ return None+ cache.sources = (+ block.params+ if hasattr(block, "orig_layer")+ else {name: module.params for name, module in block.named_modules() if hasattr(module, "orig_layer")}+ )+ staging_bytes = 2 * _nbytes(template)+ best_bytes = _nbytes(cache.sources)+ if budget_gib == "auto":+ torch.cuda.synchronize(cache.device)+ free_bytes = torch.cuda.mem_get_info(cache.device)[0]+ budget = (+ _auto_budget_bytes(free_bytes) if budget_gib == "auto" else int(min(budget_gib, free_bytes / 2**30) * 2**30)+ )+ if budget_gib == "auto":+ logger.info("Diffusion tuning auto cache budget after warmup: %.2f GiB.", budget / 2**30)+ if staging_bytes > budget:+ logger.info("Diffusion tuning prefetch skipped: two batch buffers exceed the cache budget.")+ return None+ cache.stop = threading.Event()+ cache.free, cache.ready = queue.Queue(), queue.Queue()+ cache.slots, cache.current, cache.thread = [], None, None+ cache.best = None+ cache.stream = torch.cuda.Stream(device=cache.device)+ try:+ for slot_id in range(2):+ host = tree_map(+ lambda t: torch.empty_like(t, device="cpu", pin_memory=True) if isinstance(t, torch.Tensor) else t,+ template,+ )+ gpu = tree_map(+ lambda t: torch.empty_like(t, device=cache.device) if isinstance(t, torch.Tensor) else t, template+ )+ cache.slots.append(dict(host=host, gpu=gpu, ready=torch.cuda.Event(), finished=None))+ cache.free.put(slot_id)+ if staging_bytes + best_bytes <= budget:+ cache.best = tree_map(lambda t: torch.empty_like(t, device=cache.device), cache.sources)+ # Allocations happen on the compute stream; hand them to the copy stream.+ cache.stream.wait_stream(torch.cuda.current_stream(cache.device))+ cache.thread = threading.Thread(target=cache._produce, args=(first,), daemon=True)+ cache.thread.start()+ except torch.OutOfMemoryError:+ cache.close()+ logger.info("Diffusion tuning prefetch skipped: buffer allocation ran out of memory.")+ return None+ logger.info(+ "Diffusion tuning prefetch enabled with %.2f GiB of GPU buffers; best parameters on %s.",+ (staging_bytes + (best_bytes if cache.best is not None else 0)) / 2**30,+ "GPU" if cache.best is not None else "CPU",+ )+ return cache++ def _select(self, indices):+ inputs, others = self.runner.select_batch(self.inputs, self.others, torch.tensor(indices, dtype=torch.long))+ return inputs, others, self.outputs[indices[0]]++ def _produce(self, first):+ try:+ with torch.cuda.device(self.device):+ indices = first+ while not self.stop.is_set():+ try:+ slot_id = self.free.get(timeout=0.1)+ except queue.Empty:+ continue+ slot = self.slots[slot_id]+ if slot["finished"] is not None:+ slot["finished"].synchronize()+ batch = self._select(indices)+ if _signature(batch) != self.signature:+ self.ready.put(None)+ return+ with torch.cuda.stream(self.stream):+ source = tree_flatten(batch)[0]+ host = tree_flatten(slot["host"])[0]+ gpu = tree_flatten(slot["gpu"])[0]+ for src, pinned, dest in zip(source, host, gpu):+ if isinstance(src, torch.Tensor):+ pinned.copy_(src)+ dest.copy_(pinned, non_blocking=True)+ slot["ready"].record(self.stream)+ self.ready.put((list(indices), slot_id))+ indices = next(self.plan, None)+ if indices is None:+ return+ except Exception as error:+ self.ready.put(error)++ def get(self, indices):+ if self.stop.is_set():+ return None+ if self.current is not None:+ slot = self.slots[self.current]+ slot["finished"] = torch.cuda.Event()+ slot["finished"].record(torch.cuda.current_stream(self.device))+ self.free.put(self.current)+ self.current = None+ try:+ item = self.ready.get(timeout=60)+ except queue.Empty:+ # Prefetch is an optional optimization. If the producer cannot+ # keep up, release its buffers and let the caller use the legacy+ # synchronous path for the current batch.+ self.close()+ logger.info("Diffusion tuning prefetch timed out; falling back to the legacy path.")+ return None+ if isinstance(item, Exception):+ raise item+ if item is None or item[0] != list(indices):+ # Dynamic shapes or a custom forward using Python RNG can invalidate+ # the forecast. Consume the actual sampler batch on the original path.+ self.close()+ logger.info("Diffusion tuning prefetch stopped: batch structure or sample order changed.")+ return None+ _, self.current = item+ slot = self.slots[self.current]+ torch.cuda.current_stream(self.device).wait_event(slot["ready"])+ return slot["gpu"]++ def forward(self, block, batch, cache_device):+ inputs, others, _ = batch+ shared = self.runner.shared_cache_keys+ inputs = {+ key: [value] if isinstance(value, torch.Tensor) or key in shared else value for key, value in inputs.items()+ }+ others = {+ key: (+ [value]+ if key != "positional_inputs" and (isinstance(value, (torch.Tensor, list)) or key in shared)+ else value+ )+ for key, value in others.items()+ }+ return self.runner.forward(block, inputs, others, [0], cache_device)++ def collect_best_params(self):+ for dest, source in zip(tree_flatten(self.best)[0], tree_flatten(self.sources)[0]):+ dest.copy_(source.detach())+ return self.best++ def close(self):+ self.stop.set()+ if self.thread is not None:+ self.thread.join()+ # Both streams can still reference buffers on exceptions/early stopping.+ self.stream.synchronize()+ torch.cuda.current_stream(self.device).synchronize()+ self.slots.clear()+ self.current = Nonediff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.pyindex 5228c80a6..73e5a06e7 100644--- a/auto_round/compressors/diffusion_mixin.py+++ b/auto_round/compressors/diffusion_mixin.py@@ -12,6 +12,7 @@# See the License for the specific language governing permissions and# limitations under the License.import json+import mathimport osfrom typing import Any, Optional, Union@@ -44,6 +45,10 @@ class DiffusionMixin:num_inference_steps: Number of denoising steps for diffusion generation or evaluationcalib_num_inference_steps: Number of denoising steps used to collect calibration inputsgenerator_seed: Seed for initial noise generation+ diffusion_tuning_cache_size: Extra persistent GPU buffer budget in GiB for+ single-CUDA SignRound prefetch with low_gpu_mem_usage; 0 disables it.+ "auto" selects a conservative budget after the first tuning iteration.+ Training activations/workspace are not included in this budget.Design note:``ModelContext._load_model()`` loads the diffusion pipeline and sets@@ -59,12 +64,22 @@ class DiffusionMixin:num_inference_steps: int = 50,calib_num_inference_steps: int = 8,generator_seed: Optional[int] = None,+ diffusion_tuning_cache_size: Union[float, str] = 0,**kwargs,) -> None:if num_inference_steps < 1:raise ValueError("num_inference_steps must be a positive integer.")if calib_num_inference_steps < 1:raise ValueError("calib_num_inference_steps must be a positive integer.")+ if diffusion_tuning_cache_size != "auto":+ try:+ diffusion_tuning_cache_size = float(diffusion_tuning_cache_size)+ if not math.isfinite(diffusion_tuning_cache_size) or diffusion_tuning_cache_size < 0:+ raise ValueError+ except (TypeError, ValueError):+ raise ValueError(+ "diffusion_tuning_cache_size must be 'auto' or finite and non-negative (GiB)."+ ) from None# Store diffusion-specific attributesself.guidance_scale = guidance_scale@@ -119,6 +134,7 @@ class DiffusionMixin:# Call parent class __init__ (will be Compressor, ImatrixCompressor, etc)super().__init__(*args, **kwargs)+ self.model_context.diffusion_tuning_cache_size = diffusion_tuning_cache_sizepipe = getattr(self.model_context, "pipe", None)model = getattr(self.model_context, "model", None)diff --git a/test/unit/common/compressors/test_diffusion_mixin.py b/test/unit/common/compressors/test_diffusion_mixin.pyindex b0edac3e2..c05a51117 100644--- a/test/unit/common/compressors/test_diffusion_mixin.py+++ b/test/unit/common/compressors/test_diffusion_mixin.py@@ -28,8 +28,10 @@ class TestDiffusionMixinProperties:assert params.get("num_inference_steps") == 50assert params.get("calib_num_inference_steps") == 8assert params.get("generator_seed") is None+ assert params.get("diffusion_tuning_cache_size") == 0- def test_generation_and_calibration_steps_are_independent(self):+ @pytest.mark.parametrize("budget", [2, "auto"])+ def test_generation_and_calibration_steps_are_independent(self, budget):class Parent:def __init__(self, *args, **kwargs):self.model_context = SimpleNamespace(pipe=None, model=None)@@ -37,16 +39,21 @@ class TestDiffusionMixinProperties:class MockCompressor(DiffusionMixin, Parent):pass- comp = MockCompressor(num_inference_steps=20, calib_num_inference_steps=7)+ comp = MockCompressor(num_inference_steps=20, calib_num_inference_steps=7, diffusion_tuning_cache_size=budget)assert comp.num_inference_steps == 20assert comp.calib_num_inference_steps == 7+ assert comp.model_context.diffusion_tuning_cache_size == budget@pytest.mark.parametrize(("kwargs", "match"),[({"calib_num_inference_steps": 0}, "calib_num_inference_steps"),({"num_inference_steps": 0}, "num_inference_steps"),+ ({"diffusion_tuning_cache_size": -1}, "diffusion_tuning_cache_size"),+ ({"diffusion_tuning_cache_size": float("nan")}, "diffusion_tuning_cache_size"),+ ({"diffusion_tuning_cache_size": float("inf")}, "diffusion_tuning_cache_size"),+ ({"diffusion_tuning_cache_size": "invalid"}, "diffusion_tuning_cache_size"),],)def test_rejects_non_positive_inference_steps(self, kwargs, match):diff --git a/test/unit/common/compressors/test_diffusion_tuning_cache.py b/test/unit/common/compressors/test_diffusion_tuning_cache.pynew file mode 100644index 000000000..4f4fd019b--- /dev/null+++ b/test/unit/common/compressors/test_diffusion_tuning_cache.py@@ -0,0 +1,64 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import random++import pytest+import torch++from auto_round.algorithms.block_runner import BlockForwardRunner+from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache, _batch_plan+from auto_round.compressors.utils import IndexSampler++[email protected]("global_batch", [1, 2])+def test_prefetch_plan_preserves_sampler_and_rng(global_batch):+ state = random.getstate()+ try:+ sampler = IndexSampler(5, global_batch)+ before = random.getstate()+ predicted = list(_batch_plan(sampler, 8, 1))+ assert random.getstate() == before+ assert sampler.index == 0+ assert predicted == [[i] for _ in range(8) for i in sampler.next_batch()]+ finally:+ random.setstate(state)++[email protected]("shared", [False, True])+def test_prefetched_batch_preserves_input_structure(shared):+ class Block(torch.nn.Module):+ def forward(self, hidden_states, scale, temb, freqs):+ return hidden_states * scale + temb + freqs[0] + freqs[1]++ cache = DiffusionTuningCache()+ cache.runner = BlockForwardRunner(+ batch_size=1,+ device="cpu",+ cache_device="cpu",+ amp=False,+ is_diffusion=True,+ shared_cache_keys=("freqs",) if shared else (),+ )+ cache.inputs = {"hidden_states": torch.arange(24).reshape(3, 2, 4), "scale": 2.0}+ cache.others = {+ "temb": torch.ones(3, 2, 4),+ "freqs": [[torch.ones(1, 2, 4), torch.ones(1, 2, 4) * 2] for _ in range(3)],+ "positional_inputs": [],+ }+ cache.outputs = [torch.ones(1, 2, 4) * i for i in range(3)]+ batch = cache._select([1])+ expected = cache.runner.forward(Block(), cache.inputs, cache.others, [1], "cpu")+ torch.testing.assert_close(cache.forward(Block(), batch, "cpu"), expected)+ torch.testing.assert_close(batch[2], cache.outputs[1])diff --git a/test/unit/common/core/test_entry_normalization.py b/test/unit/common/core/test_entry_normalization.pyindex 350260a2e..c8e560f0d 100644--- a/test/unit/common/core/test_entry_normalization.py+++ b/test/unit/common/core/test_entry_normalization.py@@ -14,6 +14,16 @@ def _types(configs):return [type(config).__name__ for config in configs][email protected]("value,expected", [("auto", "auto"), ("2", 2.0), ("0", 0.0)])+def test_diffusion_cache_cli_and_entry_routing(value, expected):+ from auto_round.cli.main import _build_entry_model_type_kwargs+ from auto_round.cli.parser import build_quantize_parser++ args = build_quantize_parser().parse_args(["--diffusion_tuning_cache_size", value])+ grouped = _split_entry_kwargs(_build_entry_model_type_kwargs(args))+ assert grouped["diffusion"]["diffusion_tuning_cache_size"] == expected++def test_split_entry_kwargs_partitions_owned_fields():processor = object()@@ -25,6 +35,7 @@ def test_split_entry_kwargs_partitions_owned_fields():"model_free": False,"num_inference_steps": 20,"calib_num_inference_steps": 8,+ "diffusion_tuning_cache_size": 2,})@@ -34,6 +45,7 @@ def test_split_entry_kwargs_partitions_owned_fields():assert grouped["route"]["model_free"] is Falseassert grouped["diffusion"]["num_inference_steps"] == 20assert grouped["diffusion"]["calib_num_inference_steps"] == 8+ assert grouped["diffusion"]["diffusion_tuning_cache_size"] == 2def test_split_entry_kwargs_ignores_unknown_fields(monkeypatch):diff --git a/test/unit/test_cuda/models/test_diffusion.py b/test/unit/test_cuda/models/test_diffusion.pyindex 3fa81319b..ba0919ffe 100644--- a/test/unit/test_cuda/models/test_diffusion.py+++ b/test/unit/test_cuda/models/test_diffusion.py@@ -61,9 +61,22 @@ class TestAutoRound:autoround.quantize_and_save(tmp_path)@require_optimum- def test_diffusion_tune(self, tiny_flux_model_path, tmp_path):+ @pytest.mark.parametrize("low_gpu_mem_usage,cache_size", [(False, 0), (True, "auto"), (True, 1e-9)])+ def test_diffusion_tune(self, tiny_flux_model_path, tmp_path, monkeypatch, low_gpu_mem_usage, cache_size):from diffusers import AutoPipelineForText2Image+ from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache++ caches = []+ create_cache = DiffusionTuningCache.create++ def capture_cache(*args, **kwargs):+ cache = create_cache(*args, **kwargs)+ caches.append(cache)+ return cache++ monkeypatch.setattr(DiffusionTuningCache, "create", capture_cache)+## load the modelpipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)model = pipe.transformer@@ -86,15 +99,25 @@ class TestAutoRound:pipe,tokenizer=None,scheme="MXFP4",- iters=1,+ iters=2,nsamples=1,calib_num_inference_steps=2,layer_config=layer_config,dataset="coco2014",+ low_gpu_mem_usage=low_gpu_mem_usage,+ diffusion_tuning_cache_size=cache_size,)# skip model saving since it takes much timeautoround.quantize_and_save(tmp_path)+ if cache_size == "auto":+ assert any(cache is not None for cache in caches)+ assert all(not cache.thread.is_alive() and not cache.slots for cache in caches if cache is not None)+ elif cache_size:+ assert caches and all(cache is None for cache in caches)+ else:+ assert not caches+@pytest.mark.skip_ci(reason="Architecture: Download large model; Time-consuming")def test_diffusion_model_checker(self):from auto_round.utils import is_diffusion_model