Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add opt-in diffusion tuning cache prefetch (#2340)

Wang, Chang585d20653215
diff --git a/auto_round/algorithms/quantization/sign_round/quantizer.py b/auto_round/algorithms/quantization/sign_round/quantizer.py
index 37bd72aeb..df70764db 100644
--- a/auto_round/algorithms/quantization/sign_round/quantizer.py
+++ b/auto_round/algorithms/quantization/sign_round/quantizer.py
@@ -473,61 +473,107 @@ class SignRoundQuantizer(BaseQuantizer):
else None
)
- for i in range(self.iters):
- if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):
- for n, m in block.named_modules():
- m.cur_iter = i
- total_loss = 0
- global_indices = index_sampler.next_batch()
- if valid_token_mask:
- num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)
-
- for batch_start in range(0, len(global_indices), batch_size):
- indices = global_indices[batch_start : batch_start + batch_size]
- ref_output = torch.cat([fp_outputs[i] for i in indices], dim=0).to(loss_device)
- pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)
- if loss_device is not None:
- pred_output = pred_output.to(loss_device)
- if (
- block_ctx.block_index == block_ctx.block_cnt - 1
- and self.enable_lfq
- and input_ids is not None
- and self._is_text_decoder_block(block_ctx.block_name)
- ):
- loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))
- else:
- loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
- num_elm = 1 if num_elm <= 0 else num_elm
- total_loss += loss.item() / num_elm
-
- if mid_iter_mem_check:
- # clear memory to avoid OOM due to memory fragmentation
- clear_memory_if_reached_threshold(threshold=0.5, device_list=device_manager.device_list)
-
- self._scale_loss_and_backward(scaler, loss)
-
- if mid_iter_mem_check:
- # clear memory to avoid OOM due to memory fragmentation
- clear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)
+ tuning_cache = None
+ # Only opt-in diffusion tuning can enter the CUDA staging path.
+ cache_budget = getattr(self.model_context, "diffusion_tuning_cache_size", 0)
+ use_tuning_cache = (
+ getattr(self.model_context, "is_diffusion", False)
+ and (cache_budget == "auto" or cache_budget > 0)
+ and self.compress_context.low_gpu_mem_usage
+ and str(device).startswith("cuda")
+ and len(device_manager.device_list) == 1
+ and (loss_device is None or torch.device(loss_device) == torch.device(device))
+ )
- if i == 0:
- init_loss = total_loss
- current_lr = optimizer.param_groups[0]["lr"]
- logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)
+ try:
+ for i in range(self.iters):
+ # Auto observes a complete forward/backward/optimizer iteration
+ # on the legacy path before allocating any extra GPU buffers.
+ if use_tuning_cache and i == (1 if cache_budget == "auto" else 0):
+ from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache
+
+ tuning_cache = DiffusionTuningCache.create(
+ block,
+ block_fwd,
+ active_inputs,
+ input_others,
+ fp_outputs,
+ index_sampler,
+ self.iters - i,
+ cache_budget,
+ device,
+ )
+ if self.enable_alg_ext and self.scheme.data_type.endswith("dq"):
+ for n, m in block.named_modules():
+ m.cur_iter = i
+ total_loss = 0
+ global_indices = index_sampler.next_batch()
+ if valid_token_mask:
+ num_elm = self._get_non_zero_cnt(valid_token_mask, global_indices)
+
+ for batch_start in range(0, len(global_indices), batch_size):
+ indices = global_indices[batch_start : batch_start + batch_size]
+ staged = tuning_cache.get(indices) if tuning_cache is not None else None
+ if staged is None:
+ ref_output = torch.cat([fp_outputs[i] for i in indices], dim=0).to(loss_device)
+ pred_output = block_fwd.forward(block, active_inputs, input_others, indices, _fwd_cache_device)
+ else:
+ ref_output = staged[2]
+ pred_output = tuning_cache.forward(block, staged, _fwd_cache_device)
+ if loss_device is not None:
+ pred_output = pred_output.to(loss_device)
+ if (
+ block_ctx.block_index == block_ctx.block_cnt - 1
+ and self.enable_lfq
+ and input_ids is not None
+ and self._is_text_decoder_block(block_ctx.block_name)
+ ):
+ loss = self.lfq_loss(pred_output, torch.cat([input_ids[i] for i in indices], dim=0))
+ else:
+ loss = self._get_loss(pred_output, ref_output, indices, mse_loss, device, valid_token_mask)
+ num_elm = 1 if num_elm <= 0 else num_elm
+ total_loss += loss.item() / num_elm
+
+ if mid_iter_mem_check:
+ # clear memory to avoid OOM due to memory fragmentation
+ clear_memory_if_reached_threshold(threshold=0.5, device_list=device_manager.device_list)
+
+ self._scale_loss_and_backward(scaler, loss)
+
+ if mid_iter_mem_check:
+ # clear memory to avoid OOM due to memory fragmentation
+ clear_memory_if_reached_threshold(threshold=0.8, device_list=device_manager.device_list)
+
+ if i == 0:
+ init_loss = total_loss
+ current_lr = optimizer.param_groups[0]["lr"]
+ logger.debug("iter %d loss: %.3e lr: %s", i, total_loss, current_lr)
+
+ if total_loss < best_loss:
+ best_loss = total_loss
+ if not self.not_use_best_mse:
+ best_params = (
+ tuning_cache.collect_best_params()
+ if tuning_cache is not None and tuning_cache.best is not None
+ else collect_best_params(block, self.compress_context.cache_device)
+ )
+ last_best_iter = i
+ if self.not_use_best_mse and i == self.iters - 1:
+ best_params = (
+ tuning_cache.collect_best_params()
+ if tuning_cache is not None and tuning_cache.best is not None
+ else collect_best_params(block, self.compress_context.cache_device)
+ )
- if total_loss < best_loss:
- best_loss = total_loss
if not self.not_use_best_mse:
- best_params = collect_best_params(block, self.compress_context.cache_device)
- last_best_iter = i
- if self.not_use_best_mse and i == self.iters - 1:
- best_params = collect_best_params(block, self.compress_context.cache_device)
-
- if not self.not_use_best_mse:
- if 0 < self.dynamic_max_gap <= i - last_best_iter:
- break
- sync_gradients()
- self._step(scaler, optimizer, lr_schedule)
+ if 0 < self.dynamic_max_gap <= i - last_best_iter:
+ break
+ sync_gradients()
+ self._step(scaler, optimizer, lr_schedule)
+
+ finally:
+ if tuning_cache is not None:
+ tuning_cache.close()
last_loss = total_loss
best_iter = self.iters
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 6b96ceff4..203966cb6 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -355,6 +355,7 @@ _ENTRY_KWARG_OWNERS = {
"num_inference_steps": "diffusion",
"calib_num_inference_steps": "diffusion",
"generator_seed": "diffusion",
+ "diffusion_tuning_cache_size": "diffusion",
}
_SCHEME_FIELDS = set(QuantizationScheme.get_attributes())
diff --git a/auto_round/cli/main.py b/auto_round/cli/main.py
index 0e601547f..8441bf5bb 100644
--- a/auto_round/cli/main.py
+++ b/auto_round/cli/main.py
@@ -99,6 +99,7 @@ def _build_entry_model_type_kwargs(args) -> dict:
"num_inference_steps": args.num_inference_steps,
"calib_num_inference_steps": args.calib_num_inference_steps,
"generator_seed": args.generator_seed,
+ "diffusion_tuning_cache_size": args.diffusion_tuning_cache_size,
}
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index dbab03181..40f0f4559 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -296,6 +296,14 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
help="Number of denoising steps for diffusion generation/evaluation.",
)
diff.add_argument("--generator_seed", default=None, type=int, help="Random seed used for diffusion generation.")
+ diff.add_argument(
+ "--diffusion_tuning_cache_size",
+ default=0,
+ type=lambda value: value if value == "auto" else float(value),
+ help="Extra GPU buffer budget in GiB for single-CUDA diffusion SignRound prefetch with low_gpu_mem_usage. "
+ "Use 'auto' to select a budget after the first tuning iteration; 0 preserves the existing path. "
+ "This is not a limit on total GPU memory.",
+ )
# ---- Common Quantization Arguments ----
quant_group = parser.add_argument_group("Common Quantization Arguments")
diff --git a/auto_round/compressors/diffusion/README.md b/auto_round/compressors/diffusion/README.md
index 4d2dcc388..91a177bf1 100644
--- a/auto_round/compressors/diffusion/README.md
+++ b/auto_round/compressors/diffusion/README.md
@@ -42,6 +42,9 @@ autoround.quantize_and_save(output_dir, format="fake", inplace=True)
control the calibration schedule.
- `guidance_scale`: controls how much the image generation process follows the text prompt.
- `generator_seed`: a seed that controls the initial noise from which an image is generated.
+- `diffusion_tuning_cache_size`: opt-in extra GPU buffer budget in GiB for diffusion SignRound prefetch.
+ It only takes effect with `low_gpu_mem_usage=True` on single-GPU SignRound flow (`enable_quanted_input=False`
+ and no custom `layer_config`). This value reserves additional temporary buffers and is not a cap on total VRAM.
For more hyperparameters, refer to [Homepage Detailed Hyperparameters](../../../README.md#quantization-scheme--configuration).
@@ -57,9 +60,15 @@ auto-round \
--batch_size 1 \
--dataset coco2014 \
--calib_num_inference_steps 8 \
+ --low_gpu_mem_usage \
+ --diffusion_tuning_cache_size auto \
--output_dir ./tmp_autoround
```
+Use `--diffusion_tuning_cache_size` only when `--low_gpu_mem_usage` is enabled on single-GPU SignRound
+(`--enable_quanted_input` disabled and no custom `--layer_config`). The GiB value controls extra prefetch buffers,
+not total GPU memory.
+
### Diffusion Support Matrix
For diffusion models, currently we validate quantization on the following models, which involves quantizing the transformer component of the pipeline.
diff --git a/auto_round/compressors/diffusion/tuning_cache.py b/auto_round/compressors/diffusion/tuning_cache.py
new file mode 100644
index 000000000..a3f9b55f4
--- /dev/null
+++ b/auto_round/compressors/diffusion/tuning_cache.py
@@ -0,0 +1,236 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Opt-in, bounded CUDA staging for diffusion SignRound batches."""
+
+import queue
+import random
+import threading
+
+import torch
+from torch.utils._pytree import tree_flatten, tree_map
+
+from auto_round.logger import logger
+
+
+def _batch_plan(sampler, iters, batch_size):
+ """Forecast the cyclic sampler without advancing it or the global RNG."""
+ rng = random.Random()
+ rng.setstate(random.getstate())
+ indices, position = list(sampler.indices), sampler.index
+ for _ in range(iters):
+ if position + sampler.batch_size > sampler.nsamples:
+ rng.shuffle(indices)
+ position = 0
+ selected = indices[position : position + sampler.batch_size]
+ position += sampler.batch_size
+ for start in range(0, len(selected), batch_size):
+ yield selected[start : start + batch_size]
+
+
+def _signature(batch):
+ leaves, spec = tree_flatten(batch)
+ signature = []
+ for value in leaves:
+ if isinstance(value, torch.Tensor):
+ if value.device.type != "cpu" or value.layout != torch.strided:
+ return None
+ signature.append((value.shape, value.dtype))
+ elif value is None or isinstance(value, (str, bool, int, float)):
+ signature.append((type(value), value))
+ else:
+ return None
+ return spec, signature
+
+
+def _nbytes(batch):
+ return sum(t.numel() * t.element_size() for t in tree_flatten(batch)[0] if isinstance(t, torch.Tensor))
+
+
+def _auto_budget_bytes(free_bytes):
+ # Keep the warmed allocator pool for activations/workspace, and leave at
+ # least half the unreserved memory (and at least 1 GiB) as additional margin.
+ return max(0, min(free_bytes // 2, free_bytes - 2**30))
+
+
+class DiffusionTuningCache:
+ """Two reusable host/device slots, with optional in-place best snapshots.
+
+ The budget covers these persistent CUDA buffers only, not the training
+ activations/workspace. Pinned host memory is bounded by the two batch slots.
+ The caller must close the cache after backward, including on early exit.
+ """
+
+ @classmethod
+ def create(cls, block, runner, inputs, others, outputs, sampler, iters, budget_gib, device):
+ if runner.batch_size != 1 or not isinstance(inputs, dict) or iters <= 0:
+ return None
+ cache = cls()
+ cache.runner, cache.inputs, cache.others, cache.outputs = runner, inputs, others, outputs
+ cache.device = torch.device(device)
+ cache.plan = iter(_batch_plan(sampler, iters, runner.batch_size))
+ # Start the generator now, before the original sampler advances.
+ first = next(cache.plan)
+ template = cache._select(first)
+ cache.signature = _signature(template)
+ if cache.signature is None:
+ return None
+ cache.sources = (
+ block.params
+ if hasattr(block, "orig_layer")
+ else {name: module.params for name, module in block.named_modules() if hasattr(module, "orig_layer")}
+ )
+ staging_bytes = 2 * _nbytes(template)
+ best_bytes = _nbytes(cache.sources)
+ if budget_gib == "auto":
+ torch.cuda.synchronize(cache.device)
+ free_bytes = torch.cuda.mem_get_info(cache.device)[0]
+ budget = (
+ _auto_budget_bytes(free_bytes) if budget_gib == "auto" else int(min(budget_gib, free_bytes / 2**30) * 2**30)
+ )
+ if budget_gib == "auto":
+ logger.info("Diffusion tuning auto cache budget after warmup: %.2f GiB.", budget / 2**30)
+ if staging_bytes > budget:
+ logger.info("Diffusion tuning prefetch skipped: two batch buffers exceed the cache budget.")
+ return None
+ cache.stop = threading.Event()
+ cache.free, cache.ready = queue.Queue(), queue.Queue()
+ cache.slots, cache.current, cache.thread = [], None, None
+ cache.best = None
+ cache.stream = torch.cuda.Stream(device=cache.device)
+ try:
+ for slot_id in range(2):
+ host = tree_map(
+ lambda t: torch.empty_like(t, device="cpu", pin_memory=True) if isinstance(t, torch.Tensor) else t,
+ template,
+ )
+ gpu = tree_map(
+ lambda t: torch.empty_like(t, device=cache.device) if isinstance(t, torch.Tensor) else t, template
+ )
+ cache.slots.append(dict(host=host, gpu=gpu, ready=torch.cuda.Event(), finished=None))
+ cache.free.put(slot_id)
+ if staging_bytes + best_bytes <= budget:
+ cache.best = tree_map(lambda t: torch.empty_like(t, device=cache.device), cache.sources)
+ # Allocations happen on the compute stream; hand them to the copy stream.
+ cache.stream.wait_stream(torch.cuda.current_stream(cache.device))
+ cache.thread = threading.Thread(target=cache._produce, args=(first,), daemon=True)
+ cache.thread.start()
+ except torch.OutOfMemoryError:
+ cache.close()
+ logger.info("Diffusion tuning prefetch skipped: buffer allocation ran out of memory.")
+ return None
+ logger.info(
+ "Diffusion tuning prefetch enabled with %.2f GiB of GPU buffers; best parameters on %s.",
+ (staging_bytes + (best_bytes if cache.best is not None else 0)) / 2**30,
+ "GPU" if cache.best is not None else "CPU",
+ )
+ return cache
+
+ def _select(self, indices):
+ inputs, others = self.runner.select_batch(self.inputs, self.others, torch.tensor(indices, dtype=torch.long))
+ return inputs, others, self.outputs[indices[0]]
+
+ def _produce(self, first):
+ try:
+ with torch.cuda.device(self.device):
+ indices = first
+ while not self.stop.is_set():
+ try:
+ slot_id = self.free.get(timeout=0.1)
+ except queue.Empty:
+ continue
+ slot = self.slots[slot_id]
+ if slot["finished"] is not None:
+ slot["finished"].synchronize()
+ batch = self._select(indices)
+ if _signature(batch) != self.signature:
+ self.ready.put(None)
+ return
+ with torch.cuda.stream(self.stream):
+ source = tree_flatten(batch)[0]
+ host = tree_flatten(slot["host"])[0]
+ gpu = tree_flatten(slot["gpu"])[0]
+ for src, pinned, dest in zip(source, host, gpu):
+ if isinstance(src, torch.Tensor):
+ pinned.copy_(src)
+ dest.copy_(pinned, non_blocking=True)
+ slot["ready"].record(self.stream)
+ self.ready.put((list(indices), slot_id))
+ indices = next(self.plan, None)
+ if indices is None:
+ return
+ except Exception as error:
+ self.ready.put(error)
+
+ def get(self, indices):
+ if self.stop.is_set():
+ return None
+ if self.current is not None:
+ slot = self.slots[self.current]
+ slot["finished"] = torch.cuda.Event()
+ slot["finished"].record(torch.cuda.current_stream(self.device))
+ self.free.put(self.current)
+ self.current = None
+ try:
+ item = self.ready.get(timeout=60)
+ except queue.Empty:
+ # Prefetch is an optional optimization. If the producer cannot
+ # keep up, release its buffers and let the caller use the legacy
+ # synchronous path for the current batch.
+ self.close()
+ logger.info("Diffusion tuning prefetch timed out; falling back to the legacy path.")
+ return None
+ if isinstance(item, Exception):
+ raise item
+ if item is None or item[0] != list(indices):
+ # Dynamic shapes or a custom forward using Python RNG can invalidate
+ # the forecast. Consume the actual sampler batch on the original path.
+ self.close()
+ logger.info("Diffusion tuning prefetch stopped: batch structure or sample order changed.")
+ return None
+ _, self.current = item
+ slot = self.slots[self.current]
+ torch.cuda.current_stream(self.device).wait_event(slot["ready"])
+ return slot["gpu"]
+
+ def forward(self, block, batch, cache_device):
+ inputs, others, _ = batch
+ shared = self.runner.shared_cache_keys
+ inputs = {
+ key: [value] if isinstance(value, torch.Tensor) or key in shared else value for key, value in inputs.items()
+ }
+ others = {
+ key: (
+ [value]
+ if key != "positional_inputs" and (isinstance(value, (torch.Tensor, list)) or key in shared)
+ else value
+ )
+ for key, value in others.items()
+ }
+ return self.runner.forward(block, inputs, others, [0], cache_device)
+
+ def collect_best_params(self):
+ for dest, source in zip(tree_flatten(self.best)[0], tree_flatten(self.sources)[0]):
+ dest.copy_(source.detach())
+ return self.best
+
+ def close(self):
+ self.stop.set()
+ if self.thread is not None:
+ self.thread.join()
+ # Both streams can still reference buffers on exceptions/early stopping.
+ self.stream.synchronize()
+ torch.cuda.current_stream(self.device).synchronize()
+ self.slots.clear()
+ self.current = None
diff --git a/auto_round/compressors/diffusion_mixin.py b/auto_round/compressors/diffusion_mixin.py
index 5228c80a6..73e5a06e7 100644
--- a/auto_round/compressors/diffusion_mixin.py
+++ b/auto_round/compressors/diffusion_mixin.py
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
+import math
import os
from typing import Any, Optional, Union
@@ -44,6 +45,10 @@ class DiffusionMixin:
num_inference_steps: Number of denoising steps for diffusion generation or evaluation
calib_num_inference_steps: Number of denoising steps used to collect calibration inputs
generator_seed: Seed for initial noise generation
+ diffusion_tuning_cache_size: Extra persistent GPU buffer budget in GiB for
+ single-CUDA SignRound prefetch with low_gpu_mem_usage; 0 disables it.
+ "auto" selects a conservative budget after the first tuning iteration.
+ Training activations/workspace are not included in this budget.
Design note:
``ModelContext._load_model()`` loads the diffusion pipeline and sets
@@ -59,12 +64,22 @@ class DiffusionMixin:
num_inference_steps: int = 50,
calib_num_inference_steps: int = 8,
generator_seed: Optional[int] = None,
+ diffusion_tuning_cache_size: Union[float, str] = 0,
**kwargs,
) -> None:
if num_inference_steps < 1:
raise ValueError("num_inference_steps must be a positive integer.")
if calib_num_inference_steps < 1:
raise ValueError("calib_num_inference_steps must be a positive integer.")
+ if diffusion_tuning_cache_size != "auto":
+ try:
+ diffusion_tuning_cache_size = float(diffusion_tuning_cache_size)
+ if not math.isfinite(diffusion_tuning_cache_size) or diffusion_tuning_cache_size < 0:
+ raise ValueError
+ except (TypeError, ValueError):
+ raise ValueError(
+ "diffusion_tuning_cache_size must be 'auto' or finite and non-negative (GiB)."
+ ) from None
# Store diffusion-specific attributes
self.guidance_scale = guidance_scale
@@ -119,6 +134,7 @@ class DiffusionMixin:
# Call parent class __init__ (will be Compressor, ImatrixCompressor, etc)
super().__init__(*args, **kwargs)
+ self.model_context.diffusion_tuning_cache_size = diffusion_tuning_cache_size
pipe = getattr(self.model_context, "pipe", None)
model = getattr(self.model_context, "model", None)
diff --git a/test/unit/common/compressors/test_diffusion_mixin.py b/test/unit/common/compressors/test_diffusion_mixin.py
index b0edac3e2..c05a51117 100644
--- a/test/unit/common/compressors/test_diffusion_mixin.py
+++ b/test/unit/common/compressors/test_diffusion_mixin.py
@@ -28,8 +28,10 @@ class TestDiffusionMixinProperties:
assert params.get("num_inference_steps") == 50
assert params.get("calib_num_inference_steps") == 8
assert params.get("generator_seed") is None
+ assert params.get("diffusion_tuning_cache_size") == 0
- def test_generation_and_calibration_steps_are_independent(self):
+ @pytest.mark.parametrize("budget", [2, "auto"])
+ def test_generation_and_calibration_steps_are_independent(self, budget):
class Parent:
def __init__(self, *args, **kwargs):
self.model_context = SimpleNamespace(pipe=None, model=None)
@@ -37,16 +39,21 @@ class TestDiffusionMixinProperties:
class MockCompressor(DiffusionMixin, Parent):
pass
- comp = MockCompressor(num_inference_steps=20, calib_num_inference_steps=7)
+ comp = MockCompressor(num_inference_steps=20, calib_num_inference_steps=7, diffusion_tuning_cache_size=budget)
assert comp.num_inference_steps == 20
assert comp.calib_num_inference_steps == 7
+ assert comp.model_context.diffusion_tuning_cache_size == budget
@pytest.mark.parametrize(
("kwargs", "match"),
[
({"calib_num_inference_steps": 0}, "calib_num_inference_steps"),
({"num_inference_steps": 0}, "num_inference_steps"),
+ ({"diffusion_tuning_cache_size": -1}, "diffusion_tuning_cache_size"),
+ ({"diffusion_tuning_cache_size": float("nan")}, "diffusion_tuning_cache_size"),
+ ({"diffusion_tuning_cache_size": float("inf")}, "diffusion_tuning_cache_size"),
+ ({"diffusion_tuning_cache_size": "invalid"}, "diffusion_tuning_cache_size"),
],
)
def test_rejects_non_positive_inference_steps(self, kwargs, match):
diff --git a/test/unit/common/compressors/test_diffusion_tuning_cache.py b/test/unit/common/compressors/test_diffusion_tuning_cache.py
new file mode 100644
index 000000000..4f4fd019b
--- /dev/null
+++ b/test/unit/common/compressors/test_diffusion_tuning_cache.py
@@ -0,0 +1,64 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+
+import pytest
+import torch
+
+from auto_round.algorithms.block_runner import BlockForwardRunner
+from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache, _batch_plan
+from auto_round.compressors.utils import IndexSampler
+
+
[email protected]("global_batch", [1, 2])
+def test_prefetch_plan_preserves_sampler_and_rng(global_batch):
+ state = random.getstate()
+ try:
+ sampler = IndexSampler(5, global_batch)
+ before = random.getstate()
+ predicted = list(_batch_plan(sampler, 8, 1))
+ assert random.getstate() == before
+ assert sampler.index == 0
+ assert predicted == [[i] for _ in range(8) for i in sampler.next_batch()]
+ finally:
+ random.setstate(state)
+
+
[email protected]("shared", [False, True])
+def test_prefetched_batch_preserves_input_structure(shared):
+ class Block(torch.nn.Module):
+ def forward(self, hidden_states, scale, temb, freqs):
+ return hidden_states * scale + temb + freqs[0] + freqs[1]
+
+ cache = DiffusionTuningCache()
+ cache.runner = BlockForwardRunner(
+ batch_size=1,
+ device="cpu",
+ cache_device="cpu",
+ amp=False,
+ is_diffusion=True,
+ shared_cache_keys=("freqs",) if shared else (),
+ )
+ cache.inputs = {"hidden_states": torch.arange(24).reshape(3, 2, 4), "scale": 2.0}
+ cache.others = {
+ "temb": torch.ones(3, 2, 4),
+ "freqs": [[torch.ones(1, 2, 4), torch.ones(1, 2, 4) * 2] for _ in range(3)],
+ "positional_inputs": [],
+ }
+ cache.outputs = [torch.ones(1, 2, 4) * i for i in range(3)]
+ batch = cache._select([1])
+ expected = cache.runner.forward(Block(), cache.inputs, cache.others, [1], "cpu")
+ torch.testing.assert_close(cache.forward(Block(), batch, "cpu"), expected)
+ torch.testing.assert_close(batch[2], cache.outputs[1])
diff --git a/test/unit/common/core/test_entry_normalization.py b/test/unit/common/core/test_entry_normalization.py
index 350260a2e..c8e560f0d 100644
--- a/test/unit/common/core/test_entry_normalization.py
+++ b/test/unit/common/core/test_entry_normalization.py
@@ -14,6 +14,16 @@ def _types(configs):
return [type(config).__name__ for config in configs]
[email protected]("value,expected", [("auto", "auto"), ("2", 2.0), ("0", 0.0)])
+def test_diffusion_cache_cli_and_entry_routing(value, expected):
+ from auto_round.cli.main import _build_entry_model_type_kwargs
+ from auto_round.cli.parser import build_quantize_parser
+
+ args = build_quantize_parser().parse_args(["--diffusion_tuning_cache_size", value])
+ grouped = _split_entry_kwargs(_build_entry_model_type_kwargs(args))
+ assert grouped["diffusion"]["diffusion_tuning_cache_size"] == expected
+
+
def test_split_entry_kwargs_partitions_owned_fields():
processor = object()
@@ -25,6 +35,7 @@ def test_split_entry_kwargs_partitions_owned_fields():
"model_free": False,
"num_inference_steps": 20,
"calib_num_inference_steps": 8,
+ "diffusion_tuning_cache_size": 2,
}
)
@@ -34,6 +45,7 @@ def test_split_entry_kwargs_partitions_owned_fields():
assert grouped["route"]["model_free"] is False
assert grouped["diffusion"]["num_inference_steps"] == 20
assert grouped["diffusion"]["calib_num_inference_steps"] == 8
+ assert grouped["diffusion"]["diffusion_tuning_cache_size"] == 2
def test_split_entry_kwargs_ignores_unknown_fields(monkeypatch):
diff --git a/test/unit/test_cuda/models/test_diffusion.py b/test/unit/test_cuda/models/test_diffusion.py
index 3fa81319b..ba0919ffe 100644
--- a/test/unit/test_cuda/models/test_diffusion.py
+++ b/test/unit/test_cuda/models/test_diffusion.py
@@ -61,9 +61,22 @@ class TestAutoRound:
autoround.quantize_and_save(tmp_path)
@require_optimum
- def test_diffusion_tune(self, tiny_flux_model_path, tmp_path):
+ @pytest.mark.parametrize("low_gpu_mem_usage,cache_size", [(False, 0), (True, "auto"), (True, 1e-9)])
+ def test_diffusion_tune(self, tiny_flux_model_path, tmp_path, monkeypatch, low_gpu_mem_usage, cache_size):
from diffusers import AutoPipelineForText2Image
+ from auto_round.compressors.diffusion.tuning_cache import DiffusionTuningCache
+
+ caches = []
+ create_cache = DiffusionTuningCache.create
+
+ def capture_cache(*args, **kwargs):
+ cache = create_cache(*args, **kwargs)
+ caches.append(cache)
+ return cache
+
+ monkeypatch.setattr(DiffusionTuningCache, "create", capture_cache)
+
## load the model
pipe = AutoPipelineForText2Image.from_pretrained(tiny_flux_model_path)
model = pipe.transformer
@@ -86,15 +99,25 @@ class TestAutoRound:
pipe,
tokenizer=None,
scheme="MXFP4",
- iters=1,
+ iters=2,
nsamples=1,
calib_num_inference_steps=2,
layer_config=layer_config,
dataset="coco2014",
+ low_gpu_mem_usage=low_gpu_mem_usage,
+ diffusion_tuning_cache_size=cache_size,
)
# skip model saving since it takes much time
autoround.quantize_and_save(tmp_path)
+ if cache_size == "auto":
+ assert any(cache is not None for cache in caches)
+ assert all(not cache.thread.is_alive() and not cache.slots for cache in caches if cache is not None)
+ elif cache_size:
+ assert caches and all(cache is None for cache in caches)
+ else:
+ assert not caches
+
@pytest.mark.skip_ci(reason="Architecture: Download large model; Time-consuming")
def test_diffusion_model_checker(self):
from auto_round.utils import is_diffusion_model