Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Qwen-Image-2.1 placement from measured sizes, int8 under offload on torchao 0.17, balanced fit check (#12408)

Daniel Hancf89ed7a4b45
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 3a3e4a3529a..63961bcf64e 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -121,13 +121,18 @@ from .diffusion_memory import (
plan_diffusion_memory,
plan_fits_total_capacity,
plan_keeps_transformer_resident,
+ prequant_seed_device,
raise_on_image_activation_shortfall,
raise_on_unified_memory_shortfall,
reclaimable_snapshot_device_memory,
reclaim_host_memory,
reclaim_offload_host_memory,
release_pinned_host_memory,
+ refine_balanced_plan_for_components,
refine_memory_plan_for_components,
+ refine_plan_from_loaded_weights,
+ release_resident_groups,
+ measured_request_extra_mib,
settled_snapshot_device_memory,
snapshot_device_memory,
_torchao_stream_pinnable,
@@ -6160,6 +6165,10 @@ class DiffusionBackend:
scheme = pipeline_seed_scheme,
dtype = dtype,
device = device,
+ # offloading plans seed it on the host: a GPU seed OOMed 8 GB cards
+ placement_device = prequant_seed_device(
+ plan, device, pipeline_seed_scheme
+ ),
hf_token = hf_token,
target = target,
path_override = transformer_prequant_path,
@@ -6802,7 +6811,9 @@ class DiffusionBackend:
# Whole-module offload still onloads one complete component for its forward. Refine it from the
# loaded, possibly quantized weights so an oversized text encoder uses leaf streaming instead of
# failing during prompt encoding.
- refined_plan = refine_memory_plan_for_components(pipe, plan)
+ refined_plan = refine_memory_plan_for_components(
+ pipe, refine_balanced_plan_for_components(pipe, plan)
+ )
if refined_plan.offload_policy != plan.offload_policy:
logger.info(
"diffusion.memory: refined policy %s -> %s (%s)",
@@ -6811,6 +6822,24 @@ class DiffusionBackend:
"; ".join(refined_plan.reasons),
)
plan = refined_plan
+ # not under the legacy cpu_offload flag alone: that request asked for offload
+ if not cpu_offload or normalize_memory_mode(memory_mode) is not None:
+ plan = refine_plan_from_loaded_weights(
+ pipe,
+ plan,
+ family = fam.name,
+ speed_mode = effective_speed,
+ logger = logger,
+ )
+ headroom = plan.estimates.get("measured_runtime_headroom_mib")
+ if headroom and (
+ plan.resident_transformer_mib or plan.resident_text_encoder_mib
+ ):
+ pipe._unsloth_measured_reserve = (
+ int(headroom),
+ fam.name,
+ effective_speed,
+ )
from .diffusion_qwenimage21_vision import configure_vision_attention
@@ -8573,6 +8602,7 @@ class DiffusionBackend:
# Reset in the finally, so a failed or cancelled generation frees its reused outputs.
static_skip_pipe = None
restore_vae: Optional[Callable[[], None]] = None
+ restore_resident: Optional[Callable[[], None]] = None
try:
self._state_device_target(state)
# The local `state` ref keeps the pipe alive even if unload() nulls _state. Resolve the per-image
@@ -8896,6 +8926,25 @@ class DiffusionBackend:
)
guard_batch = _activation_guard_batch(chunks)
guard_target = self._state_device_target(state)
+ # past what the measured placement reserved: stream resident groups again for this call
+ extra_mib = measured_request_extra_mib(
+ state.pipe,
+ width = guard_width,
+ height = guard_height,
+ batch_size = guard_batch,
+ condition_pixels = (
+ int(
+ (1 + len(ref_extra))
+ * ref_resolution
+ * ref_resolution
+ * getattr(fam, "condition_pixel_weight", 1.0)
+ )
+ if ref_resolution is not None
+ else 0
+ ),
+ )
+ if extra_mib > 0:
+ restore_resident = release_resident_groups(state.pipe, extra_mib, logger)
guard_kwargs = dict(
# NOT the settled snapshot the load uses: that one calls empty_cache(), which is right once
# per load but wrong on a per-generation path, since it releases every cached block and the
@@ -9223,6 +9272,11 @@ class DiffusionBackend:
logger.debug("diffusion.step_skip: reset failed: %s", exc)
if restore_vae is not None:
restore_vae()
+ if restore_resident is not None:
+ try:
+ restore_resident()
+ except Exception as exc: # noqa: BLE001 - the groups keep streaming
+ logger.warning("diffusion.memory: re-residency failed: %s", exc)
with self._generation_cancel_lock:
if self._active_generate_cancel is cancel:
self._active_generate_cancel = None
diff --git a/studio/backend/core/inference/diffusion_denoiser_prequant.py b/studio/backend/core/inference/diffusion_denoiser_prequant.py
index 2855869a85e..a598a6a5b01 100644
--- a/studio/backend/core/inference/diffusion_denoiser_prequant.py
+++ b/studio/backend/core/inference/diffusion_denoiser_prequant.py
@@ -89,9 +89,11 @@ def denoiser_prequant_pipe_kwargs(
local_files_only: bool = False,
cache_dir: Optional[str] = None,
logger: Any = None,
+ placement_device: Optional[str] = None,
) -> dict[str, Any]:
"""``{"transformer": module}`` for pipeline assembly, or ``{}`` when it cannot be seeded, which
- obliges the caller to re-plan at bf16 and restore the dropped shards."""
+ obliges the caller to re-plan at bf16 and restore the dropped shards. ``placement_device``: where the
+ module is materialised (see ``load_prequantized_transformer``)."""
try:
if not pipeline_seed_supported(fam):
return {}
@@ -133,6 +135,7 @@ def denoiser_prequant_pipe_kwargs(
cache_dir = cache_dir,
local_files_only = local_files_only,
logger = logger,
+ placement_device = placement_device,
)
if module is None:
if logger is not None:
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 43bed7f4738..2651f7bfe06 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -564,6 +564,9 @@ class MemoryPlan:
stream_text_encoders: bool = False
# False only on the tier keeping the transformer resident and streaming just the text encoders.
stream_transformer: bool = True
+ # MiB of the streamed denoiser / encoders kept resident (whole groups); None streams every group.
+ resident_transformer_mib: Optional[int] = None
+ resident_text_encoder_mib: Optional[int] = None
@property
def engages_offload(self) -> bool:
@@ -580,6 +583,8 @@ class MemoryPlan:
"reasons": list(self.reasons),
"stream_text_encoders": self.stream_text_encoders,
"stream_transformer": self.stream_transformer,
+ "resident_transformer_mib": self.resident_transformer_mib,
+ "resident_text_encoder_mib": self.resident_text_encoder_mib,
}
@@ -1002,8 +1007,73 @@ def plan_keeps_transformer_resident(plan: Any) -> bool:
return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))
+PREQUANT_SEED_ON_HOST_ENV = "UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST"
+
+
+def prequant_seed_device(
+ plan: Any,
+ device: str,
+ scheme: Optional[str] = None,
+) -> str:
+ """Where a seeded pre-quantized denoiser is materialised: ``device`` when ``plan`` keeps it resident, else "cpu"
+ for the schemes measured under offload. Loading it onto the GPU first adds a whole-denoiser spike that left the
+ streaming hooks no room for their first block (Qwen-Image-2.1 int8 on 8 GB)."""
+ if str(os.environ.get(PREQUANT_SEED_ON_HOST_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return device
+ if plan is None or plan_keeps_transformer_resident(plan):
+ return device
+ if str(scheme) not in _TORCHAO_GROUP_OFFLOAD_MIN:
+ return device
+ return "cpu"
+
+
# Oldest torchao measured bit-exact under streamed group offload; 0.17 int8 (v1, no copy stream) ran 14x slower.
_TORCHAO_GROUP_OFFLOAD_MIN = {"int8": (0, 18), "fp8": (0, 17)}
+# torchao 0.17 already ships the pinnable Int8Tensor (hosted int8 checkpoints) and its v1 int8 weights get the pin
+# ops (install_torchao_v1_int8_pin_ops): without this a fresh install (torchao 0.17) got fp8 whenever it offloaded.
+INT8_STREAM_TORCHAO17_ENV = "UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17"
+_TORCHAO17_INT8_STREAM_MIN = (0, 17)
+
+
+def _int8_tensor_pinnable() -> bool:
+ """Whether the installed torchao's ``Int8Tensor`` implements the pin ops the stream needs (0.17+)."""
+ try:
+ from torchao.quantization.quantize_.workflows.int8 import int8_tensor
+ import torch
+
+ table = getattr(int8_tensor.Int8Tensor, "_ATEN_OP_TABLE", None)
+ if isinstance(table, dict):
+ ops = set()
+ for value in table.values():
+ ops.update(value.keys() if isinstance(value, dict) else ())
+ if ops:
+ return (
+ torch.ops.aten._pin_memory.default in ops
+ and torch.ops.aten.is_pinned.default in ops
+ )
+ return True # no readable op table: the class exists, which is what 0.17 added
+ except Exception: # noqa: BLE001 - no Int8Tensor: keep the 0.18 floor
+ return False
+
+
+def _int8_stream_floor() -> tuple:
+ if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]
+ if not _int8_tensor_pinnable():
+ return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]
+ return _TORCHAO17_INT8_STREAM_MIN
+
+
_TORCHAO_STREAM_SAFE_CLASSES = frozenset(("Int8Tensor", "Float8Tensor"))
# Before 0.38 diffusers moved only the torchao wrapper, leaving quantised data on the host.
_DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN = (0, 38)
@@ -1147,6 +1217,8 @@ def torchao_scheme_streams(scheme: Optional[str], *, torchao_version: Any = _UNS
floor = _TORCHAO_GROUP_OFFLOAD_MIN.get(str(scheme))
if floor is None:
return False
+ if str(scheme) == "int8":
+ floor = _int8_stream_floor()
diffusers_version = _installed_diffusers_version()
if diffusers_version is None or diffusers_version < _DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN:
return False
@@ -1192,7 +1264,9 @@ def _torchao_group_offload_kwargs(
install_group_offload_torchao_swap_retry()
if not kwargs.get("use_stream"):
return kwargs
- if classes <= _TORCHAO_STREAM_SAFE_CLASSES:
+ if classes <= _TORCHAO_STREAM_SAFE_CLASSES or (
+ classes <= _TORCHAO_V1_INT8_CLASSES and install_torchao_v1_int8_pin_ops()
+ ):
if not kwargs.get("low_cpu_mem_usage"):
return kwargs
# Same override the planner's _torchao_stream_pinnable honoured, so the two cannot disagree.
@@ -1219,6 +1293,107 @@ def _torchao_group_offload_kwargs(
return safe
+# torchao <= 0.17's default int8 weight (LinearActivationQuantizedTensor over an AffineQuantizedTensor).
+_TORCHAO_V1_INT8_CLASSES = frozenset(("LinearActivationQuantizedTensor", "AffineQuantizedTensor"))
+_V1_INT8_PIN_OPS_INSTALLED = False
+
+
+def install_torchao_v1_int8_pin_ops() -> bool:
+ """Register ``is_pinned`` / ``pin_memory`` on torchao's v1 int8 classes so group offload streams them (else
+ synchronous copies, 14x slower). Re-wraps the same int8 data, scale and zero point, so outputs stay bit-identical.
+ Only where torchao ships the classes without the ops (<= 0.17)."""
+ global _V1_INT8_PIN_OPS_INSTALLED
+ if _V1_INT8_PIN_OPS_INSTALLED:
+ return True
+ if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return False
+ try:
+ import torch
+ from torchao.dtypes.affine_quantized_tensor import AffineQuantizedTensor as AQT
+ from torchao.quantization.linear_activation_quantized_tensor import (
+ LinearActivationQuantizedTensor as LAQT,
+ )
+
+ aten = torch.ops.aten
+ pin_ops = [aten._pin_memory.default, aten.pin_memory.default]
+ for cls in (LAQT, AQT):
+ table = getattr(cls, "_ATEN_OP_TABLE", {}).get(cls, {})
+ if aten.is_pinned.default in table or aten._pin_memory.default in table:
+ return False # this torchao implements them itself: never override its dispatch
+
+ def _payload(t: Any) -> list:
+ if isinstance(t, LAQT):
+ return _payload(t.original_weight_tensor)
+ if isinstance(t, AQT):
+ impl = t.tensor_impl
+ return [
+ x
+ for x in (getattr(impl, n, None) for n in ("int_data", "scale", "zero_point"))
+ if x is not None
+ ]
+ return [t]
+
+ def _pinned(t: Any) -> Any:
+ if isinstance(t, LAQT):
+ return t._apply_fn_to_data(_pinned)
+ if isinstance(t, AQT):
+ return t._apply_fn_to_data(
+ lambda impl: impl._apply_fn_to_data(lambda x: x.pin_memory())
+ )
+ return t.pin_memory()
+
+ # diffusers restores / record_streams torchao weights via ``tensor_data_names``, which v1 lacks: the payload stayed
+ # on the GPU after offload.
+ from diffusers.hooks import group_offloading as go
+
+ restore = go._restore_torchao_tensor
+ if not getattr(restore, "_unsloth_v1_int8", False):
+
+ def _restore_v1(param: Any, source: Any) -> None:
+ if isinstance(source, LAQT):
+ param.original_weight_tensor = source.original_weight_tensor
+ elif isinstance(source, AQT):
+ param.tensor_impl = source.tensor_impl
+ else:
+ restore(param, source)
+
+ _restore_v1._unsloth_v1_int8 = True
+ go._restore_torchao_tensor = _restore_v1
+
+ record = go._record_stream_torchao_tensor
+ if not getattr(record, "_unsloth_v1_int8", False):
+
+ def _record_v1(param: Any, stream: Any) -> None:
+ if isinstance(param, (LAQT, AQT)):
+ for x in _payload(param):
+ x.record_stream(stream)
+ else:
+ record(param, stream)
+
+ _record_v1._unsloth_v1_int8 = True
+ go._record_stream_torchao_tensor = _record_v1
+
+ for cls in (LAQT, AQT):
+
+ @cls.implements([aten.is_pinned.default])
+ def _is_pinned(func: Any, types: Any, args: Any, kwargs: Any) -> bool:
+ return all(bool(x.is_pinned()) for x in _payload(args[0]))
+
+ @cls.implements(pin_ops)
+ def _pin(func: Any, types: Any, args: Any, kwargs: Any) -> Any:
+ return _pinned(args[0])
+
+ _V1_INT8_PIN_OPS_INSTALLED = True
+ return True
+ except Exception: # noqa: BLE001 - no v1 classes / no dispatch table: keep the synchronous fallback
+ return False
+
+
def _freeze_torchao_weights(module: Any) -> None:
try:
for param in module.parameters():
@@ -1623,6 +1798,96 @@ def largest_streamable_companion_mib(pipe: Any) -> Optional[int]:
return max(sizes) if sizes else None
+BALANCED_FIT_CHECK_ENV = "UNSLOTH_DIFFUSION_BALANCED_FIT_CHECK"
+
+
+def _balanced_fit_check_enabled() -> bool:
+ return str(os.environ.get(BALANCED_FIT_CHECK_ENV, "")).strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def refine_balanced_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:
+ """Fit-check an explicit ``memory_mode=balanced`` plan against the LOADED companions.
+
+ Balanced never checked the budget, so Qwen-Image-2.1 int8 kept its 9 GiB fp8 encoder resident and OOMed at 16 /
+ 12 / 8 GB. Judged after the load because the pre-load estimate can be the dense encoder size (16.7 vs 9.0 GB).
+ Walks only the streamed tiers: encoders streamed too, else whole-module offload (granular streaming for torchao
+ denoisers, which cannot take per-forward ``Module.to()``)."""
+ if not _balanced_fit_check_enabled():
+ return plan
+ if getattr(plan, "requested_mode", None) != MEMORY_MODE_BALANCED:
+ return plan
+ if plan.offload_policy != OFFLOAD_GROUP or not bool(getattr(plan, "stream_transformer", True)):
+ return plan
+ estimates = dict(plan.estimates)
+ budget = estimates.get("safe_device_budget_mib")
+ if budget is None:
+ return plan
+ try:
+ import torch
+
+ components = getattr(pipe, "components", {})
+ if not isinstance(components, dict):
+ return plan
+ streamed = _streamable_components(pipe, torch)
+ mib = 1024 * 1024
+ resident = 0
+ encoders = 0
+ for name, component in components.items():
+ if not isinstance(component, torch.nn.Module):
+ continue
+ if name in streamed and streamed[name][1] == "block_level":
+ continue # the denoisers already stream under group offload
+ size = (_module_storage_bytes(component, set()) + mib - 1) // mib
+ resident += size
+ if name in streamed:
+ encoders += size
+ overhead = int(estimates.get("runtime_headroom_mib") or 0) + int(
+ estimates.get("base_overhead_mib") or 0
+ )
+ except Exception: # noqa: BLE001 - a sizing aid; keep the plan the user asked for
+ return plan
+
+ estimates["balanced_resident_companions_mib"] = resident
+ if resident + overhead <= int(budget):
+ return plan
+ if encoders > 0 and bool(getattr(plan, "stream_text_encoders", False)) is False:
+ if (resident - encoders) + overhead <= int(budget):
+ return replace(
+ plan,
+ stream_text_encoders = True,
+ estimates = estimates,
+ reasons = plan.reasons
+ + (
+ f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget "
+ "beside the runtime headroom; streaming the text encoders too (they run once, before step 0)",
+ ),
+ )
+ torchao = _pipe_denoisers_hold_torchao(pipe)
+ return replace(
+ plan,
+ offload_policy = OFFLOAD_STREAMING if torchao else OFFLOAD_MODEL,
+ stream_text_encoders = False,
+ vae_tiling = True,
+ vae_slicing = True,
+ estimates = estimates,
+ reasons = plan.reasons
+ + (
+ f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget even with "
+ "the text encoders streamed; "
+ + (
+ "streaming transformer blocks and text-encoder layers"
+ if torchao
+ else "whole-module offload of every component"
+ ),
+ ),
+ )
+
+
def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:
"""Replace whole-module offload when a loaded component cannot fit on the device.
@@ -1688,6 +1953,379 @@ def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan
)
+# The flat planner reserves 8192 MiB/MP of activations: Qwen-Image-2.1 (16525 MiB of weights, 1849 MiB measured peak)
+# streamed its encoder on 24 GB.
+MEASURED_ACTIVATION_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION"
+PARTIAL_RESIDENT_ENV = "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT"
+
+# Worst measured CUDA MiB above the resident weights, one 1024x1024 image, encoder + every step + VAE decode, torchao
+# int8 / fp8 denoisers on the compiled tiers (Qwen-Image-2.1: encoder 1849 streamed, denoise 1442, decode 1730).
+_MEASURED_IMAGE_PEAK_MIB: dict[str, int] = {"qwen-image-2.1": 1849}
+_MEASURED_PEAK_SPEED_MODES = ("default", "max")
+_MEASURED_PEAK_MARGIN = 1.15
+_MEASURED_PEAK_ROUND_MIB = 256
+
+
+def _env_off(name: str) -> bool:
+ return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")
+
+
+def measured_image_runtime_mib(
+ family: Optional[str],
+ speed_mode: Optional[str],
+ *,
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ batch_size: int = 1,
+) -> Optional[int]:
+ """Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate)."""
+ if _env_off(MEASURED_ACTIVATION_ENV):
+ return None
+ peak = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())
+ if peak is None or str(speed_mode or "") not in _MEASURED_PEAK_SPEED_MODES:
+ return None
+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
+ scale = max(
+ 1.0,
+ (w * h * max(1, int(batch_size or 1))) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT),
+ )
+ need = peak * scale * _MEASURED_PEAK_MARGIN
+ step = _MEASURED_PEAK_ROUND_MIB
+ return int(-(-need // step) * step)
+
+
+def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:
+ """name -> (MiB, role in 'dit' / 'text_encoder' / 'other') from storage bytes (a torchao int8 weight counts its
+ int8 data + scales); None when unreadable."""
+ try:
+ import torch
+
+ out: dict[str, tuple[int, str]] = {}
+ for name, module in (getattr(pipe, "components", {}) or {}).items():
+ if not isinstance(module, torch.nn.Module):
+ continue
+ seen: set[int] = set()
+ nbytes = 0
+ for tensor in (*module.parameters(), *module.buffers()):
+ if id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ nbytes += sum(_storage_nbytes(tensor))
+ name = str(name)
+ role = (
+ "dit"
+ if name in ("transformer", "transformer_2", "unconditional_transformer")
+ else "text_encoder"
+ if name.startswith("text_encoder")
+ else "other"
+ )
+ out[name] = (-(-nbytes // (1024 * 1024)), role)
+ return out
+ except Exception: # noqa: BLE001 - an optional refinement
+ return None
+
+
+def refine_plan_from_loaded_weights(
+ pipe: Any,
+ plan: MemoryPlan,
+ *,
+ family: Optional[str],
+ speed_mode: Optional[str],
+ logger: Any = None,
+) -> MemoryPlan:
+ """Re-place a streamed ``auto`` load from its LOADED weights and the family's measured activation peak.
+
+ Keeps the flat plan's offload hooks and makes whole groups resident (denoiser first, then encoders) within the
+ safe budget left after the measured peak x margin and the base overhead. No-op for explicit modes, non-CUDA / unified memory, unmeasured families
+ or speed tiers, non-torchao denoisers and ``model`` plans."""
+ try:
+ if getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO:
+ return plan
+ policy = plan.offload_policy
+ if policy not in (OFFLOAD_GROUP, OFFLOAD_STREAMING):
+ return plan
+ memory = plan.device_memory
+ if memory.is_unified or getattr(memory, "device", None) != "cuda":
+ return plan
+ if not _pipe_denoisers_hold_torchao(pipe):
+ return plan
+ headroom = measured_image_runtime_mib(family, speed_mode)
+ if headroom is None:
+ return plan
+ budget = plan.estimates.get("safe_device_budget_mib")
+ if budget is None:
+ return plan
+ budget = int(budget)
+ sizes = _loaded_component_mib(pipe)
+ if not sizes:
+ return plan
+ dit = sum(m for m, r in sizes.values() if r == "dit")
+ encoders = sum(m for m, r in sizes.values() if r == "text_encoder")
+ other = sum(m for m, r in sizes.values() if r == "other")
+ if dit <= 0:
+ return plan
+ overhead = int(plan.estimates.get("base_overhead_mib") or DEFAULT_BASE_OVERHEAD_MIB)
+ floor = headroom + overhead + other
+ estimates = dict(plan.estimates)
+ estimates.update(
+ measured_runtime_headroom_mib = headroom,
+ loaded_transformer_mib = dit,
+ loaded_text_encoder_mib = encoders,
+ loaded_other_mib = other,
+ )
+ if _env_off(PARTIAL_RESIDENT_ENV):
+ return plan
+ # The flat plan's hooks stay installed and whole groups are kept resident within the room, so an oversized
+ # request can stream them again (release_resident_groups) and run exactly as the flat plan would.
+ stream_te = bool(getattr(plan, "stream_text_encoders", False))
+ room = budget - floor
+ if policy == OFFLOAD_GROUP and not stream_te:
+ room -= encoders # resident companions
+ if not bool(getattr(plan, "stream_transformer", True)):
+ room -= dit
+ dit_room = 0
+ else:
+ dit_room = min(max(room, 0), dit)
+ te_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0
+ if dit_room <= 0 and te_room <= 0:
+ return plan
+ new = replace(
+ plan,
+ resident_transformer_mib = int(dit_room) if dit_room > 0 else None,
+ resident_text_encoder_mib = int(te_room) if te_room > 0 else None,
+ estimates = {
+ **estimates,
+ "resident_transformer_mib": int(dit_room),
+ "resident_text_encoder_mib": int(te_room),
+ },
+ reasons = plan.reasons
+ + (
+ f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "
+ f"encoders stay resident (measured peak {headroom} MiB); the rest streams",
+ ),
+ )
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: measured-activation placement %s -> %s (%s)",
+ policy,
+ new.offload_policy,
+ new.reasons[-1],
+ )
+ return new
+ except Exception as exc: # noqa: BLE001 - keep the flat plan
+ if logger is not None:
+ logger.debug("diffusion.memory: measured-activation placement skipped (%s)", exc)
+ return plan
+
+
+def _keep_groups_resident(
+ module: Any,
+ room_mib: int,
+ device: Any,
+ logger: Any = None,
+) -> int:
+ """Make whole offload groups of a streamed ``module`` resident within ``room_mib`` (top-level group first, then
+ blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept."""
+ if room_mib is None or int(room_mib) <= 0 or _env_off(PARTIAL_RESIDENT_ENV):
+ return 0
+ try:
+ import torch
+ from diffusers.hooks import group_offloading as go
+
+ groups = _offload_groups(module)
+ if not groups:
+ return 0
+ top = [g for g in groups if getattr(g, "offload_leader", None) is module]
+ ordered = top + [g for g in groups if getattr(g, "offload_leader", None) is not module]
+ is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)
+ onload = torch.device(device)
+ left = int(room_mib) * 1024 * 1024
+ kept = 0
+
+ def _tensors(group: Any) -> list:
+ out: list = []
+ seen: set[int] = set()
+ for t in (
+ [p for m in group.modules for p in m.parameters()]
+ + [b for m in group.modules for b in m.buffers()]
+ + list(group.parameters or [])
+ + list(group.buffers or [])
+ ):
+ if id(t) not in seen:
+ seen.add(id(t))
+ out.append(t)
+ return out
+
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+
+ def _noop(*args: Any, **kwargs: Any) -> None:
+ return None
+
+ def _resident_onload(stream: Any) -> Callable[[], None]:
+ # a prefetching predecessor skips its own copy-stream wait and relies on this onload_ to do it
+ def onload_(*args: Any, **kwargs: Any) -> None:
+ if stream is not None:
+ stream.synchronize()
+
+ return disable(onload_) if callable(disable) else onload_
+
+ noop = disable(_noop) if callable(disable) else _noop
+
+ try:
+ module._unsloth_resident_room = int(room_mib)
+ module._unsloth_resident_device = onload
+ except AttributeError:
+ pass
+ for group in ordered:
+ if getattr(group, "offload_to_disk_path", None):
+ break
+ tensors = _tensors(group)
+ need = sum(sum(_storage_nbytes(t)) for t in tensors)
+ if getattr(group, "_unsloth_resident", False):
+ left -= need
+ continue
+ if need > left:
+ # too large: it streams; a later, smaller group may still fit
+ continue
+ cpu = getattr(group, "cpu_param_dict", None) or {}
+ for t in tensors:
+ if t.device.type == onload.type:
+ continue
+ moved = cpu.get(t, t).to(onload)
+ if is_torchao(t):
+ go._swap_torchao_tensor(t, moved)
+ else:
+ t.data = moved
+ # host copies stay: release_resident_groups streams the group again for an oversized request
+ group._unsloth_streamed_hooks = (
+ group.__dict__.get("onload_"),
+ group.__dict__.get("offload_"),
+ )
+ group.onload_ = _resident_onload(getattr(group, "stream", None))
+ group.offload_ = noop
+ group._unsloth_resident = True
+ group._unsloth_resident_bytes = need
+ left -= need
+ kept += need
+ if kept and onload.type == "cuda":
+ torch.cuda.synchronize(onload)
+ if logger is not None and kept:
+ logger.info(
+ "diffusion.memory: %s keeps %d MiB resident (%d of %d offload groups); the rest streams",
+ type(module).__name__,
+ kept >> 20,
+ sum(1 for g in ordered if getattr(g, "_unsloth_resident", False)),
+ len(ordered),
+ )
+ return kept >> 20
+ except Exception as exc: # noqa: BLE001 - streaming every group is the safe state
+ if logger is not None:
+ logger.warning("diffusion.memory: partial residency skipped (%s)", exc)
+ return 0
+
+
+def _release_group(group: Any) -> None:
+ onload_, offload_ = getattr(group, "_unsloth_streamed_hooks", (None, None))
+ for name, fn in (("onload_", onload_), ("offload_", offload_)):
+ if fn is not None:
+ setattr(group, name, fn)
+ else:
+ group.__dict__.pop(name, None)
+ group._unsloth_resident = False
+ group.offload_()
+
+
+def release_resident_groups(
+ pipe: Any,
+ need_mib: int,
+ logger: Any = None,
+) -> Optional[Callable[[], None]]:
+ """Stream resident offload groups again until ``need_mib`` is freed (text encoders first, then the denoiser's
+ blocks from the last); returns a callable restoring them, or None when nothing was resident. A request larger
+ than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as the flat plan."""
+ try:
+ import torch
+
+ modules = [
+ m
+ for m in (getattr(pipe, "components", {}) or {}).values()
+ if getattr(m, "_unsloth_resident_room", None)
+ ]
+ if not modules or int(need_mib) <= 0:
+ return None
+ modules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)
+ left = int(need_mib) * 1024 * 1024
+ released: list = []
+ for module in modules:
+ for group in reversed(_offload_groups(module) or []):
+ if left <= 0:
+ break
+ if not getattr(group, "_unsloth_resident", False):
+ continue
+ _release_group(group)
+ left -= int(getattr(group, "_unsloth_resident_bytes", 0))
+ if module not in released:
+ released.append(module)
+ if not released:
+ return None
+ device = getattr(released[0], "_unsloth_resident_device", None)
+ if device is not None and torch.device(device).type == "cuda":
+ torch.cuda.synchronize(device)
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: streaming %d MiB of resident groups for an oversized request",
+ (int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,
+ )
+
+ def restore() -> None:
+ for module in released:
+ _keep_groups_resident(
+ module,
+ module._unsloth_resident_room,
+ module._unsloth_resident_device,
+ logger,
+ )
+
+ return restore
+ except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit
+ if logger is not None:
+ logger.warning("diffusion.memory: releasing resident groups failed (%s)", exc)
+ return None
+
+
+def _is_text_encoder_module(pipe: Any, module: Any) -> bool:
+ for name, component in (getattr(pipe, "components", {}) or {}).items():
+ if component is module:
+ return str(name).startswith("text_encoder")
+ return False
+
+
+def measured_request_extra_mib(
+ pipe: Any,
+ *,
+ width: Optional[int],
+ height: Optional[int],
+ batch_size: int = 1,
+ condition_pixels: int = 0,
+) -> int:
+ """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made)."""
+ reserve = getattr(pipe, "_unsloth_measured_reserve", None)
+ if not reserve:
+ return 0
+ headroom, family, speed_mode = reserve
+ need = measured_image_runtime_mib(
+ family, speed_mode, width = width, height = height, batch_size = batch_size
+ )
+ if need is None:
+ return 0
+ cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))
+ need += int(
+ 8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN
+ )
+ return max(0, int(need) - int(headroom))
+
+
def apply_memory_plan(
pipe: Any,
plan: MemoryPlan,
@@ -1736,6 +2374,12 @@ def apply_memory_plan(
}
if not bool(getattr(plan, "stream_transformer", True)):
group_kwargs["stream_transformer"] = False
+ resident_mib = getattr(plan, "resident_transformer_mib", None)
+ if resident_mib:
+ group_kwargs["resident_transformer_mib"] = int(resident_mib)
+ resident_te_mib = getattr(plan, "resident_text_encoder_mib", None)
+ if resident_te_mib and group_kwargs["stream_text_encoders"]:
+ group_kwargs["resident_text_encoder_mib"] = int(resident_te_mib)
if not _apply_group_offload(pipe, placement, logger, **group_kwargs):
if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):
raise RuntimeError(
@@ -1751,7 +2395,13 @@ def apply_memory_plan(
_fallback_to_model_offload()
policy = OFFLOAD_MODEL
elif policy == OFFLOAD_STREAMING:
- _apply_streaming_offload(pipe, placement, logger)
+ resident_mib = getattr(plan, "resident_transformer_mib", None)
+ if resident_mib:
+ _apply_streaming_offload(
+ pipe, placement, logger, resident_transformer_mib = int(resident_mib)
+ )
+ else:
+ _apply_streaming_offload(pipe, placement, logger)
elif policy == OFFLOAD_SEQUENTIAL:
try:
pipe.enable_sequential_cpu_offload(device = placement)
@@ -2248,6 +2898,118 @@ def install_group_offload_buffer_restore() -> bool:
return True
+PIN_TOP_GROUP_ENV = "UNSLOTH_DIFFUSION_PIN_TOP_GROUP"
+
+
+def _pin_top_level_group(
+ module: Any,
+ logger: Any = None,
+ pinned_mib: Optional[list] = None,
+) -> bool:
+ """Onload a block-streamed DiT's top-level group (embedders, norm_out, proj_out) from one pinned host copy.
+
+ diffusers gives that group no stream: every forward uploaded it from pageable memory and every offload copied it
+ back to a fresh host buffer, which inference never needs. Now the onload is an async H2D on the compute stream and
+ the offload only re-points the tensors; VRAM unchanged. Skipped for torchao weights (not re-pointable via
+ ``.data``) and when the copy does not fit the pinnable host RAM."""
+ if (os.environ.get(PIN_TOP_GROUP_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
+ return False
+ try:
+ import torch
+ from diffusers.hooks import group_offloading as go
+
+ registry = getattr(module, "_diffusers_hook", None)
+ get_hook = getattr(registry, "get_hook", None)
+ hook = (
+ get_hook(getattr(go, "_GROUP_OFFLOADING", "group_offloading"))
+ if callable(get_hook)
+ else None
+ )
+ group = getattr(hook, "group", None)
+ if (
+ group is None
+ or getattr(group, "stream", None) is not None
+ or getattr(group, "offload_to_disk_path", None)
+ or getattr(getattr(group, "onload_device", None), "type", None) != "cuda"
+ or getattr(group, "_unsloth_pinned_top", False)
+ ):
+ return False
+ tensors: list = []
+ seen: set = set()
+ for tensor in (
+ [p for m in group.modules for p in m.parameters()]
+ + [b for m in group.modules for b in m.buffers()]
+ + list(group.parameters or [])
+ + list(group.buffers or [])
+ ):
+ if id(tensor) not in seen:
+ seen.add(id(tensor))
+ tensors.append(tensor)
+ is_torchao = getattr(go, "_is_torchao_tensor", None)
+ if not tensors or (callable(is_torchao) and any(is_torchao(t) for t in tensors)):
+ return False
+ if any(type(t) not in (torch.Tensor, torch.nn.Parameter) for t in tensors):
+ return False
+ # the user's "pin nothing" override wins, as on every other streamed path
+ if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return False
+ # per tensor rounded to a power of two, like torch's pinned allocator (and _module_host_mib)
+ need_mib = sum(
+ 1 << (int(t.numel()) * int(t.element_size()) - 1).bit_length()
+ for t in tensors
+ if int(t.numel()) * int(t.element_size()) > 0
+ ) // (1024 * 1024)
+ budget = None if _pinned_memory_capped() else _pin_budget_mib()
+ # on the running total the encoders and torchao denoisers already pinned (or will, deferred) count against
+ already = pinned_mib[0] if pinned_mib else 0
+ if budget is None or already + need_mib > budget:
+ return False
+ host = {
+ t: (t.data if t.data.device.type == "cpu" else t.data.cpu()).pin_memory()
+ for t in tensors
+ }
+ device = group.onload_device
+
+ def onload_() -> None:
+ for tensor, pinned in list(host.items()):
+ current = tensor.data
+ if current.device.type == "cpu" and current.data_ptr() != pinned.data_ptr():
+ # replaced while offloaded (a .to() conversion, an adapter fused on the host): re-pin what is there now
+ pinned = current if current.is_pinned() else current.pin_memory()
+ host[tensor] = pinned
+ tensor.data = pinned.to(device, non_blocking = True)
+
+ def offload_() -> None:
+ for tensor, pinned in host.items():
+ tensor.data = pinned
+
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ if callable(disable):
+ onload_, offload_ = disable(onload_), disable(offload_)
+ offload_()
+ group.onload_ = onload_
+ group.offload_ = offload_
+ group._unsloth_pinned_top = True
+ if pinned_mib is not None:
+ pinned_mib[0] = already + need_mib
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: %s top-level weights (%d MiB) onload from a pinned copy, no copy back",
+ type(module).__name__,
+ need_mib,
+ )
+ return True
+ except Exception as exc: # noqa: BLE001 - diffusers keeps its own (slower) path
+ if logger is not None:
+ logger.debug("diffusion.memory: top-level group left as diffusers built it (%s)", exc)
+ return False
+
+
def _apply_group_offload(
pipe: Any,
device: str,
@@ -2256,6 +3018,8 @@ def _apply_group_offload(
stream_text_encoders: bool = False,
stream_transformer: bool = True,
background_pin: Optional[bool] = None,
+ resident_transformer_mib: Optional[int] = None,
+ resident_text_encoder_mib: Optional[int] = None,
) -> bool:
"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping the
smaller components resident. Returns False (caller falls back to whole-module) on any failure.
@@ -2369,6 +3133,12 @@ def _apply_group_offload(
module, **_torchao_group_offload_kwargs(module, gkwargs, pinned_mib)
)
installed += 1
+ if use_stream:
+ _pin_top_level_group(module, logger, pinned_mib)
+ if resident_transformer_mib:
+ room = int(resident_transformer_mib)
+ for module in streamed.values():
+ room -= _keep_groups_resident(module, room, onload, logger)
# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far
# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no
# recognisable block list can refuse), and this tier is a rescue: the alternative to streaming an encoder is
@@ -2385,6 +3155,7 @@ def _apply_group_offload(
if "low_cpu_mem_usage" in gkwargs:
ekwargs["low_cpu_mem_usage"] = not pin_streamed[1]
transformer_demoted = False
+ te_room = int(resident_text_encoder_mib or 0)
for name, module in streamed_encoders.items():
try:
if defer and not ekwargs.get("low_cpu_mem_usage", False):
@@ -2394,6 +3165,8 @@ def _apply_group_offload(
apply_group_offloading(module, **ekwargs)
installed += 1
_pin_vision_embedding_device(module)
+ if te_room > 0:
+ te_room -= _keep_groups_resident(module, te_room, onload, logger)
except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load
if not stream_transformer and installed == 0:
# Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.
@@ -2888,7 +3661,26 @@ def raise_on_image_activation_shortfall(
return verdict
-def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
+STREAMING_PREFETCH_ENV = "UNSLOTH_DIFFUSION_STREAMING_PREFETCH"
+
+
+def _streaming_prefetch_enabled() -> bool:
+ """Kill switch for the streaming tier's overlapped onload (pinned host copies + record_stream); default on."""
+ return (os.environ.get(STREAMING_PREFETCH_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _apply_streaming_offload(
+ pipe: Any,
+ device: str,
+ logger: Any,
+ *,
+ resident_transformer_mib: Optional[int] = None,
+) -> None:
"""Stream transformer blocks and text-encoder leaves without whole-component onloads.
This is selected only after measuring a component larger than the safe device budget, so a
@@ -2927,6 +3719,28 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
component.to(onload)
pinned_mib = [0]
+ # Overlap needs a pinned host copy and record_stream (record_stream=False syncs the compute stream per group).
+ prefetch = use_stream and _streaming_prefetch_enabled()
+ pin_dits, pin_encoders = False, False
+ if prefetch:
+ pin_dits, pin_encoders = _streamed_pin_plan(
+ sum(_module_host_mib(m) for m, t in streamed.values() if t == "block_level"),
+ sum(_module_host_mib(m) for m, t in streamed.values() if t != "block_level"),
+ logger,
+ )
+ defer = (
+ prefetch
+ and bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
+ and _background_pin_enabled()
+ )
+ if defer:
+ install_group_pin_wait()
+ # pinned encoders count against the budget a torchao denoiser pins within, as on the group tier
+ if pin_encoders:
+ pinned_mib[0] = sum(
+ _module_host_mib(m) for m, t in streamed.values() if t != "block_level"
+ )
+
for module, offload_type in streamed.values():
kwargs: dict[str, Any] = {
"onload_device": onload,
@@ -2940,15 +3754,24 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
if use_stream and "non_blocking" in params:
kwargs["non_blocking"] = True
if use_stream and "record_stream" in params:
- kwargs["record_stream"] = False
+ kwargs["record_stream"] = prefetch
+ pin = pin_dits if offload_type == "block_level" else pin_encoders
if use_stream and "low_cpu_mem_usage" in params:
- kwargs["low_cpu_mem_usage"] = True
- apply_group_offloading(
- module, **_torchao_group_offload_kwargs(module, kwargs, pinned_mib)
- )
+ kwargs["low_cpu_mem_usage"] = not (pin and not defer)
+ kwargs = _torchao_group_offload_kwargs(module, kwargs, pinned_mib)
+ apply_group_offloading(module, **kwargs)
+ if pin and defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):
+ _defer_pinning(pipe, module, onload, logger)
installed += 1
+ if use_stream and offload_type == "block_level":
+ _pin_top_level_group(module, logger, pinned_mib)
if offload_type == "leaf_level":
_pin_vision_embedding_device(module)
+ if resident_transformer_mib:
+ room = int(resident_transformer_mib)
+ for module, offload_type in streamed.values():
+ if offload_type == "block_level":
+ room -= _keep_groups_resident(module, room, onload, logger)
except Exception as exc:
if logger is not None:
logger.warning(
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index bc18ff1d39f..508b6c8869a 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -1112,9 +1112,12 @@ def load_prequantized_transformer(
component: Optional[str] = None,
local_files_only: bool = False,
logger: Any = None,
+ placement_device: Optional[str] = None,
) -> Optional[Any]:
"""Load the pre-quantized transformer described by ``source`` onto ``device``.
+ ``placement_device`` (default ``device``) is where the module is materialised; ``device`` selects kernels.
+
``cache_dir`` is the live Hub cache root, as every other loader call pins it: unset, a fetch
lands under huggingface_hub's import-time constant, so a mid-session cache change re-downloads
into a root Unsloth no longer reads.
@@ -1228,7 +1231,7 @@ def load_prequantized_transformer(
del state_dict
del ckpt
- transformer = transformer.to(device)
+ transformer = transformer.to(placement_device or device)
if declares_rotation(metadata):
try:
import torch
@@ -1237,7 +1240,12 @@ def load_prequantized_transformer(
dtype = getattr(
torch, str(metadata.get("torch_dtype") or "bfloat16"), torch.bfloat16
)
- warm_rotation_cache(transformer, on.device if on is not None else device, dtype)
+ # a host-placed module still computes on ``device``
+ warm_rotation_cache(
+ transformer,
+ on.device if on is not None and placement_device is None else device,
+ dtype,
+ )
except Exception: # noqa: BLE001
pass
# Same small-M row padding the runtime quantise path applies, and for the same reason: a checkpoint built
@@ -1275,7 +1283,7 @@ def load_prequantized_transformer(
"diffusion.prequant: loaded %s checkpoint (%s) onto %s",
scheme,
source.kind,
- device,
+ placement_device or device,
)
return transformer
except Exception as exc: # noqa: BLE001 - fall back to the dense-quantise path
diff --git a/studio/backend/core/inference/diffusion_qwenimage21.py b/studio/backend/core/inference/diffusion_qwenimage21.py
index 77d300655a5..c9b7b690cc4 100644
--- a/studio/backend/core/inference/diffusion_qwenimage21.py
+++ b/studio/backend/core/inference/diffusion_qwenimage21.py
@@ -21,6 +21,7 @@ import weakref
from typing import Any, Optional
FAST_STEP_ENV = "UNSLOTH_DIFFUSION_Q21_FAST_STEP"
+COMPACT_KV_ENV = "UNSLOTH_DIFFUSION_Q21_COMPACT_KV"
_MODULE = "diffusers.models.transformers.transformer_qwenimage21"
_CLASS = "QwenImage21Transformer2DModel"
@@ -400,6 +401,7 @@ def _make_forward(mod: Any, stock: Any) -> Any:
cache_write_slice = slice(0, prefix_len) if kv_cache_mode == "extract" else None
block_key_valid = joint_key_valid
+ compact_kv = compact_kv_enabled()
for index_block, block in enumerate(self.transformer_blocks):
layer_cache = kv_cache.get_layer(index_block) if kv_cache is not None else None
joint_hidden_states = block(
@@ -414,6 +416,8 @@ def _make_forward(mod: Any, stock: Any) -> Any:
segments = block_segments,
key_valid = block_key_valid,
)
+ if layer_cache is not None and cache_write_slice is not None and compact_kv:
+ _compact_layer_cache(layer_cache)
joint_hidden_states = self.norm_out(joint_hidden_states, temb, modulation_mask)
output = self.proj_out(joint_hidden_states)
@@ -429,6 +433,27 @@ def _make_forward(mod: Any, stock: Any) -> Any:
return wrapped
+def compact_kv_enabled() -> bool:
+ return (os.environ.get(COMPACT_KV_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
+
+
+def _compact_layer_cache(layer_cache: Any) -> None:
+ """Copy a prefix K/V entry that is a view of a larger buffer. Under compile Inductor turns the extract step's
+ ``key[:, :prefix].clone()`` into a view of the full text + image K/V, pinning ~2 GiB across 32 blocks at 1 MP for
+ step 0. Bit-identical."""
+ for name in ("k", "v"):
+ tensor = getattr(layer_cache, name, None)
+ try:
+ if (
+ tensor is None
+ or tensor.untyped_storage().nbytes() <= tensor.numel() * tensor.element_size()
+ ):
+ continue
+ setattr(layer_cache, name, tensor.clone())
+ except Exception: # noqa: BLE001 - a memory saving only; keep the entry as stored
+ continue
+
+
def install(logger: Any = None) -> bool:
if fast_step_disabled():
return False
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index a4a86f1ced4..e9fe920365d 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -55,6 +55,20 @@ def _params_text(meta: dict[str, Any]) -> str:
return "\n".join(lines)
+# Pillow's default zlib level 6 costs ~0.4 s per 1024x1024 PNG on the request path; level 1 ~0.1 s, ~10% larger, lossless.
+PNG_COMPRESS_LEVEL_ENV = "UNSLOTH_IMAGE_PNG_COMPRESS_LEVEL"
+_DEFAULT_PNG_COMPRESS_LEVEL = 1
+
+
+def png_compress_level() -> int:
+ raw = os.environ.get(PNG_COMPRESS_LEVEL_ENV, "").strip()
+ try:
+ level = int(raw)
+ except ValueError:
+ return _DEFAULT_PNG_COMPRESS_LEVEL
+ return level if 0 <= level <= 9 else _DEFAULT_PNG_COMPRESS_LEVEL
+
+
def _png_bytes(image: Any, meta: dict[str, Any]) -> bytes:
import io
@@ -64,7 +78,7 @@ def _png_bytes(image: Any, meta: dict[str, Any]) -> bytes:
info.add_text(_META_KEY, json.dumps(meta))
info.add_text("parameters", _params_text(meta))
buf = io.BytesIO()
- image.save(buf, format = "PNG", pnginfo = info)
+ image.save(buf, format = "PNG", pnginfo = info, compress_level = png_compress_level())
return buf.getvalue()
diff --git a/studio/backend/tests/test_diffusion_int8_stream_torchao17.py b/studio/backend/tests/test_diffusion_int8_stream_torchao17.py
new file mode 100644
index 00000000000..ddae12adf7e
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_int8_stream_torchao17.py
@@ -0,0 +1,267 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""INT8 under streamed offload on torchao 0.17 (what a fresh install resolves: torch < 2.12 -> torchao 0.17).
+
+The planner used to require torchao 0.18 for int8 under any tier that streams the denoiser, so every offloading
+install got the fp8 transformer. 0.17 already ships the pinnable ``Int8Tensor``; only the v1 int8 class it still
+defaults to is unpinnable, and that is rebuilt before streaming. Version-simulated, CPU only."""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+from core.inference import diffusion_memory as mem
+from core.inference.diffusion_memory import (
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
+ OFFLOAD_STREAMING,
+ MemoryPlan,
+ torchao_offload_plan,
+ torchao_scheme_streams,
+ torchao_survives_plan,
+)
+
+ENV = "UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17"
+AO17, AO18, AO16 = (0, 17), (0, 18), (0, 16)
+
+
[email protected](autouse = True)
+def _host(monkeypatch):
+ # hermetic: the installed torchao / diffusers of the test host never decide these tests
+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 41))
+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)
+ monkeypatch.delenv(ENV, raising = False)
+
+
+def _plan(policy, stream_transformer = True):
+ # Qwen-Image-2.1 int8 at a 16 GB budget: denoiser 6922 MiB, companions 9603 MiB
+ return MemoryPlan(
+ requested_mode = "auto",
+ offload_policy = policy,
+ vae_tiling = False,
+ vae_slicing = True,
+ device_memory = None,
+ estimates = {
+ "safe_device_budget_mib": 13320,
+ "model_dense_mib": 16525,
+ "companion_dense_mib": 9603,
+ "text_encoder_dense_mib": 8959,
+ "runtime_headroom_mib": 8192,
+ "base_overhead_mib": 2048,
+ },
+ reasons = (),
+ stream_transformer = stream_transformer,
+ )
+
+
+def test_int8_streams_on_torchao_017():
+ assert torchao_scheme_streams("int8", torchao_version = AO17) is True
+
+
[email protected]("policy", [OFFLOAD_GROUP, OFFLOAD_STREAMING])
+def test_int8_survives_a_streamed_plan_on_torchao_017(monkeypatch, policy):
+ monkeypatch.setattr(mem, "_torchao_stream_pinnable", lambda *a, **k: True)
+ plan = _plan(policy)
+ assert torchao_survives_plan(plan, "int8", torchao_version = AO17) is True
+ assert torchao_offload_plan(plan, "int8", torchao_version = AO17).offload_policy == policy
+
+
+def test_int8_whole_module_too_big_streams_on_torchao_017(monkeypatch):
+ monkeypatch.setattr(mem, "_torchao_stream_pinnable", lambda *a, **k: True)
+ plan = _plan(OFFLOAD_MODEL)
+ placed = torchao_offload_plan(plan, "int8", torchao_version = AO17)
+ assert placed is not None and placed.offload_policy == OFFLOAD_STREAMING
+
+
+def test_kill_switch_restores_the_018_floor(monkeypatch):
+ monkeypatch.setenv(ENV, "0")
+ assert torchao_scheme_streams("int8", torchao_version = AO17) is False
+ assert torchao_scheme_streams("int8", torchao_version = AO18) is True
+
+
+def test_torchao_without_a_pinnable_int8_tensor_keeps_the_018_floor(monkeypatch):
+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: False)
+ assert torchao_scheme_streams("int8", torchao_version = AO17) is False
+
+
[email protected]("version", [AO16, (0, 14)])
+def test_older_torchao_still_declines_int8_streaming(version):
+ assert torchao_scheme_streams("int8", torchao_version = version) is False
+
+
+def test_fp8_and_other_schemes_unchanged():
+ assert torchao_scheme_streams("fp8", torchao_version = AO17) is True
+ assert torchao_scheme_streams("fp8", torchao_version = AO16) is False
+ assert torchao_scheme_streams("nvfp4", torchao_version = AO18) is False
+
+
+# ---- apply time: a v1 int8 module (0.17's default int8 class) keeps the copy stream through the pin-op shim
+
+
+class _V1Weight:
+ pass
+
+
+_V1Weight.__module__ = "torchao.quantization.linear_activation_quantized_tensor"
+_V1Weight.__name__ = "LinearActivationQuantizedTensor"
+
+
+class _Int8Weight:
+ pass
+
+
+_Int8Weight.__module__ = "torchao.quantization"
+_Int8Weight.__name__ = "Int8Tensor"
+
+
+class _Module:
+ def __init__(self, cls):
+ self.weights = [cls()]
+
+ def parameters(self):
+ return iter(self.weights)
+
+
+STREAM_KWARGS = {
+ "use_stream": True,
+ "non_blocking": True,
+ "record_stream": True,
+ "low_cpu_mem_usage": False,
+}
+
+
+def _apply_env(monkeypatch):
+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: True)
+ monkeypatch.setattr(mem, "_freeze_torchao_weights", lambda module: None)
+
+
+def test_v1_int8_keeps_the_copy_stream_once_its_pin_ops_exist(monkeypatch, _apply_env):
+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: True, raising = False)
+ out = mem._torchao_group_offload_kwargs(_Module(_V1Weight), dict(STREAM_KWARGS), [0])
+ assert (
+ out["use_stream"] is True
+ and out.get("record_stream") is True
+ and out.get("non_blocking") is True
+ )
+
+
+def test_v1_int8_without_its_pin_ops_falls_back_to_synchronous(monkeypatch, _apply_env):
+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: False, raising = False)
+ out = mem._torchao_group_offload_kwargs(_Module(_V1Weight), dict(STREAM_KWARGS), [0])
+ assert out["use_stream"] is False
+
+
+def test_pin_op_shim_honours_the_kill_switch(monkeypatch):
+ monkeypatch.setenv(ENV, "0")
+ monkeypatch.setattr(mem, "_V1_INT8_PIN_OPS_INSTALLED", False, raising = False)
+ assert mem.install_torchao_v1_int8_pin_ops() is False
+
+
+def test_int8_tensor_module_keeps_the_stream(monkeypatch, _apply_env):
+ out = mem._torchao_group_offload_kwargs(_Module(_Int8Weight), dict(STREAM_KWARGS), [0])
+ assert out["use_stream"] is True
+
+
+def _v1_int8_stack():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA (pinned host memory)")
+ try:
+ from torchao.quantization.linear_activation_quantized_tensor import ( # noqa: F401
+ LinearActivationQuantizedTensor,
+ )
+ except Exception: # noqa: BLE001
+ pytest.skip("this torchao has no v1 int8 classes (0.18+)")
+ return torch
+
+
+def test_v1_int8_streams_bit_identically_with_the_shim():
+ """Real torchao <= 0.17 + CUDA: the v1 weights pin, stream on the copy stream, and match resident exactly."""
+ import copy
+
+ torch = _v1_int8_stack()
+ from diffusers.hooks import apply_group_offloading
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ assert mem.install_torchao_v1_int8_pin_ops() is True
+ torch.manual_seed(0)
+ blocks = torch.nn.Sequential(
+ *[
+ torch.nn.Sequential(
+ torch.nn.Linear(256, 512), torch.nn.GELU(), torch.nn.Linear(512, 256)
+ )
+ for _ in range(3)
+ ]
+ ).to(torch.bfloat16)
+ offloaded = copy.deepcopy(blocks)
+ # set_inductor_config = False as Studio builds it: the bare config sets float32 matmul precision process-wide
+ quantize_(offloaded, Int8DynamicActivationInt8WeightConfig(set_inductor_config = False))
+ offloaded.requires_grad_(False)
+ assert type(next(offloaded.parameters())).__name__ == "LinearActivationQuantizedTensor"
+ resident = copy.deepcopy(offloaded).cuda()
+ kwargs = mem._torchao_group_offload_kwargs(
+ offloaded,
+ {
+ "onload_device": torch.device("cuda"),
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": 1,
+ **STREAM_KWARGS,
+ },
+ [0],
+ )
+ assert kwargs["use_stream"] is True
+ apply_group_offloading(offloaded, **kwargs)
+ x = torch.randn(2, 16, 256, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ want = resident(x)
+ for _ in range(3):
+ assert torch.equal(offloaded(x), want)
+
+
+def test_v1_int8_payload_returns_to_the_host_after_each_forward():
+ """The streamed v1 payload (int8 data + scales) must leave the GPU on offload, else the whole denoiser accumulates
+ there: diffusers restores torchao weights through ``tensor_data_names``, which the v1 classes do not declare."""
+ torch = _v1_int8_stack()
+ from diffusers.hooks import apply_group_offloading
+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_
+
+ assert mem.install_torchao_v1_int8_pin_ops() is True
+ torch.manual_seed(0)
+ blocks = torch.nn.Sequential(
+ *[
+ torch.nn.Sequential(
+ torch.nn.Linear(1024, 1024), torch.nn.GELU(), torch.nn.Linear(1024, 1024)
+ )
+ for _ in range(4)
+ ]
+ ).to(torch.bfloat16)
+ quantize_(blocks, Int8DynamicActivationInt8WeightConfig(set_inductor_config = False))
+ blocks.requires_grad_(False)
+ kwargs = mem._torchao_group_offload_kwargs(
+ blocks,
+ {
+ "onload_device": torch.device("cuda"),
+ "offload_device": torch.device("cpu"),
+ "offload_type": "block_level",
+ "num_blocks_per_group": 1,
+ **STREAM_KWARGS,
+ },
+ [0],
+ )
+ assert kwargs["use_stream"] is True
+ apply_group_offloading(blocks, **kwargs)
+ x = torch.randn(2, 16, 1024, dtype = torch.bfloat16, device = "cuda")
+ with torch.no_grad():
+ for _ in range(2):
+ blocks(x)
+ torch.cuda.synchronize()
+ for block in blocks:
+ for linear in (block[0], block[2]):
+ impl = linear.weight.original_weight_tensor.tensor_impl
+ assert impl.int_data.device.type == "cpu"
+ assert impl.scale.device.type == "cpu"
diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.py
new file mode 100644
index 00000000000..fc368480711
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_measured_placement.py
@@ -0,0 +1,489 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Measured-activation placement and partial denoiser residency (``diffusion_memory.py``), plus the Qwen-Image-2.1
+prefix K/V compaction (``diffusion_qwenimage21.py``).
+
+The planner cases use the estimates a real Qwen-Image-2.1 auto load logged (int8 DiT + fp8 text encoder): on a 24 GB
+card the flat plan read ``safe_device_budget_mib 21432, resident_required_mib 29870`` and streamed the text encoder
+on every prompt, although the loaded weights are 6922 + 8959 + 644 MiB. CPU-only; the loaded sizes and the torchao
+check are stubbed. The partial-residency mechanics need a CUDA device and real diffusers group offloading.
+"""
+
+from __future__ import annotations
+
+import types
+
+import pytest
+
+import core.inference.diffusion_memory as dm
+
+# Loaded storage of Qwen-Image-2.1 auto (int8 transformer, fp8 text encoder, bf16 VAE), MiB.
+Q21_LOADED = {
+ "transformer": (6922, "dit"),
+ "text_encoder": (8959, "text_encoder"),
+ "vae": (644, "other"),
+}
+# The flat plan's inputs for that load (family table sizes, flat 8192 headroom).
+Q21_FLAT = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)
+
+
+def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":
+ """A discrete CUDA snapshot whose safe budget is exactly ``budget_mib``."""
+ reserve = dm._reserve_mib("discrete_vram", total_mib)
+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)
+
+
+def _flat_plan(
+ budget_mib: int,
+ total_mib: int,
+ mode = None,
+):
+ plan = dm.plan_diffusion_memory(
+ target = types.SimpleNamespace(supports_model_cpu_offload = True),
+ device_memory = _memory(budget_mib, total_mib),
+ runtime_headroom_mib = dm.estimate_image_runtime_mib(
+ width = None, height = None, family = "qwen-image-2.1"
+ ),
+ requested_mode = mode,
+ **Q21_FLAT,
+ )
+ if plan.offload_policy == dm.OFFLOAD_MODEL:
+ # what torchao_offload_plan makes of whole-module offload for a streamable int8 denoiser
+ plan = dm.torchao_streaming_plan(plan)
+ return plan
+
+
+def q21_pipe(monkeypatch):
+ monkeypatch.delenv(
+ getattr(dm, "MEASURED_ACTIVATION_ENV", "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION"),
+ raising = False,
+ )
+ monkeypatch.delenv(
+ getattr(dm, "PARTIAL_RESIDENT_ENV", "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT"), raising = False
+ )
+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(Q21_LOADED), raising = False)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ return object()
+
+
+def _refine(
+ pipe,
+ plan,
+ family = "qwen-image-2.1",
+ speed = "default",
+):
+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed)
+
+
+def test_measured_headroom_values(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)
+ # 1849 MiB measured worst phase x 1.15, rounded up to 256 MiB; linear in pixels
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") == 2304
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "max") == 2304
+ assert (
+ dm.measured_image_runtime_mib("qwen-image-2.1", "default", width = 2048, height = 2048) == 8704
+ )
+ # never below the 1 MP figure for a smaller canvas
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default", width = 512, height = 512) == 2304
+ # unmeasured family / eager tiers keep the flat estimate
+ assert dm.measured_image_runtime_mib("flux", "default") is None
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "off") is None
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "eager") is None
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", "0")
+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") is None
+
+
+def test_24gb_keeps_transformer_and_text_encoder_resident(q21_pipe):
+ plan = _flat_plan(21432, 24576)
+ # the reported flat decision: DiT resident, text encoder streamed on every prompt
+ assert plan.estimates["resident_required_mib"] == 29870
+ assert (
+ plan.offload_policy == dm.OFFLOAD_GROUP
+ and plan.stream_text_encoders
+ and not plan.stream_transformer
+ )
+ new = _refine(q21_pipe, plan)
+ # the flat hooks stay (so an oversized request can stream again); the whole encoder is kept resident in them
+ assert new.offload_policy == dm.OFFLOAD_GROUP
+ assert new.stream_text_encoders and not new.stream_transformer
+ assert new.resident_transformer_mib is None
+ assert (
+ new.resident_text_encoder_mib
+ == 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922
+ >= 8959
+ )
+ assert new.estimates["measured_runtime_headroom_mib"] == 2304
+ # never under-reserved: loaded weights + measured peak x margin + base overhead within the safe budget
+ assert 6922 + 8959 + 644 + 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB <= 21432
+
+
+def test_16gb_keeps_transformer_resident_streams_encoder(q21_pipe):
+ plan = _flat_plan(13638, 16376)
+ assert (
+ plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformer
+ ) # flat: every DiT block streamed
+ new = _refine(q21_pipe, plan)
+ assert new.offload_policy == dm.OFFLOAD_GROUP
+ assert new.stream_text_encoders and new.stream_transformer
+ # every DiT group kept resident through its hooks, the rest of the room holds encoder layers
+ assert new.resident_transformer_mib == 6922
+
+
+def test_l4_class_card_keeps_most_of_the_encoder_resident(q21_pipe):
+ """An L4 reports 23034 MiB: its 10% reserve leaves the whole encoder ~900 MiB short, so the DiT stays resident
+ and only the encoder layers past the budget stream."""
+ budget = 20000
+ new = _refine(q21_pipe, _flat_plan(budget, 23034))
+ assert new.offload_policy == dm.OFFLOAD_GROUP and new.stream_text_encoders
+ room = budget - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922
+ assert new.resident_text_encoder_mib == room and 0 < room < 8959
+ assert new.as_public_dict()["resident_text_encoder_mib"] == room
+
+
+def test_16gb_encoder_room_and_kill_switch(q21_pipe, monkeypatch):
+ new = _refine(q21_pipe, _flat_plan(13638, 16376))
+ assert new.resident_text_encoder_mib == 13638 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")
+ plan = _flat_plan(13638, 16376)
+ assert _refine(q21_pipe, plan) is plan
+
+
+def test_12gb_partial_residency_sized_from_budget(q21_pipe):
+ plan = _flat_plan(9550, 12288)
+ assert plan.offload_policy == dm.OFFLOAD_STREAMING
+ new = _refine(q21_pipe, plan)
+ assert new.offload_policy == dm.OFFLOAD_STREAMING
+ room = 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644
+ assert new.resident_transformer_mib == room
+ assert 0 < room < 6922
+ assert new.as_public_dict()["resident_transformer_mib"] == room
+
+
+def test_no_room_keeps_flat_plan(q21_pipe):
+ plan = _flat_plan(4000, 8188)
+ assert _refine(q21_pipe, plan) is plan
+
+
+ "total,budget", [(24576, 21432), (16376, 13638), (12288, 9550), (8188, 5450)]
+)
+def test_kill_switch_restores_flat_plan(q21_pipe, monkeypatch, total, budget):
+ plan = _flat_plan(budget, total)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", "0")
+ assert _refine(q21_pipe, plan) is plan
+
+
+def test_partial_kill_switch_only_drops_partial_tier(q21_pipe, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")
+ for total, budget in ((12288, 9550), (16376, 13638), (24576, 21432)):
+ plan = _flat_plan(budget, total)
+ assert _refine(q21_pipe, plan) is plan
+
+
+ "family,speed,mode",
+ [
+ ("flux", "default", None),
+ ("qwen-image-2.1", "off", None),
+ ("qwen-image-2.1", "default", "balanced"),
+ ],
+)
+def test_unmeasured_or_explicit_untouched(q21_pipe, family, speed, mode):
+ plan = _flat_plan(21432, 24576, mode = mode)
+ assert _refine(q21_pipe, plan, family = family, speed = speed) is plan
+
+
+def test_non_torchao_and_unified_untouched(q21_pipe, monkeypatch):
+ plan = _flat_plan(21432, 24576)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)
+ assert _refine(q21_pipe, plan) is plan
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ unified = dm.replace(
+ plan, device_memory = dm.DeviceMemory("cuda", "cuda", "unified_memory", 23000, 24576)
+ )
+ assert _refine(q21_pipe, unified) is unified
+
+
+def test_never_moves_away_from_residency(q21_pipe):
+ resident = _flat_plan(200000, 200000)
+ assert resident.offload_policy == dm.OFFLOAD_NONE
+ assert _refine(q21_pipe, resident) is resident
+ model = dm.replace(_flat_plan(9550, 12288), offload_policy = dm.OFFLOAD_MODEL)
+ assert _refine(q21_pipe, model) is model
+
+
[email protected]("budget", list(range(3000, 30001, 250)))
+def test_every_tier_fits_measured_need(q21_pipe, budget):
+ """Whatever the refinement picks, the bytes it keeps on the device plus the measured peak x margin plus the base
+ overhead fit the safe budget (the reserve below it is untouched), and it is never slower than the flat pick."""
+ total = max(8188, budget + 3000)
+ plan = _flat_plan(budget, total)
+ new = _refine(q21_pipe, plan)
+ head = 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB
+ if new is plan:
+ return
+ # the policy and stream flags are the flat plan's: only resident rooms are added
+ assert (new.offload_policy, new.stream_transformer, new.stream_text_encoders) == (
+ plan.offload_policy,
+ plan.stream_transformer,
+ plan.stream_text_encoders,
+ )
+ kept = (
+ 644
+ + (6922 if not new.stream_transformer else int(new.resident_transformer_mib or 0))
+ + int(new.resident_text_encoder_mib or 0)
+ + (0 if new.stream_text_encoders or new.offload_policy == dm.OFFLOAD_STREAMING else 8959)
+ )
+ assert int(new.resident_transformer_mib or 0) <= 6922
+ assert kept + head <= budget
+
+
+def test_compact_layer_cache_frees_full_storage(monkeypatch):
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_qwenimage21 as q21
+
+ full_k = torch.randn(1, 40, 4, 8)
+ full_v = torch.randn(1, 40, 4, 8)
+ cache = types.SimpleNamespace(k = full_k[:, :6], v = full_v[:, :6])
+ assert cache.k.untyped_storage().nbytes() > cache.k.numel() * cache.k.element_size()
+ q21._compact_layer_cache(cache)
+ for got, ref in ((cache.k, full_k[:, :6]), (cache.v, full_v[:, :6])):
+ assert got.untyped_storage().nbytes() == got.numel() * got.element_size()
+ assert torch.equal(got, ref)
+ owned = cache.k
+ q21._compact_layer_cache(cache)
+ assert cache.k is owned # already compact: untouched
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_Q21_COMPACT_KV", "0")
+ assert not q21.compact_kv_enabled()
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_Q21_COMPACT_KV")
+ assert q21.compact_kv_enabled()
+
+
+def _cuda_offload_model():
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("CUDA-only: diffusers group offloading with a copy stream")
+ pytest.importorskip("diffusers.hooks")
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, 1024)
+ self.blocks = torch.nn.ModuleList(
+ torch.nn.Linear(1024, 1024) for _ in range(6)
+ ) # ~4 MiB each
+ self.proj_out = torch.nn.Linear(1024, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = torch.nn.functional.gelu(block(x))
+ return self.proj_out(x)
+
+ torch.manual_seed(0)
+ return torch, Net()
+
+
[email protected]("keep_blocks", [0, 2, 6])
+def test_partial_residency_bit_identical_and_placed(keep_blocks, monkeypatch):
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ x = torch.randn(3, 64)
+ ref = net.to("cuda")(x.cuda()).cpu()
+ net.to("cpu")
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ mib = 1024 * 1024
+ top = sum(
+ p.numel() * p.element_size() for m in (net.proj_in, net.proj_out) for p in m.parameters()
+ )
+ block = sum(p.numel() * p.element_size() for p in net.blocks[0].parameters())
+ # whole MiB, rounded up: room for the top-level group and keep_blocks blocks, short of one more block
+ room = -(-(top + keep_blocks * block) // mib) if keep_blocks else 0
+ kept = dm._keep_groups_resident(net, room, "cuda")
+ for _ in range(3): # first forward traces the prefetch order, later ones prefetch
+ out = net(x.cuda()).cpu()
+ assert torch.equal(out, ref)
+ placed = [next(b.parameters()).device.type for b in net.blocks]
+ want = ["cuda"] * min(keep_blocks, 6) + ["cpu"] * (6 - min(keep_blocks, 6))
+ if keep_blocks:
+ assert kept > 0
+ assert next(net.proj_in.parameters()).device.type == "cuda"
+ assert placed[:keep_blocks] == want[:keep_blocks]
+ assert placed[keep_blocks:] == want[keep_blocks:]
+
+
+def test_partial_residency_kill_switch(monkeypatch):
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ )
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")
+ assert dm._keep_groups_resident(net, 10_000, "cuda") == 0
+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)
+
+
+def _family_names():
+ from core.inference.diffusion_families import supported_family_names
+ return supported_family_names()
+
+
[email protected]("speed", ["off", "eager", "default", "max"])
+ "budget,total", [(21432, 24576), (13638, 16376), (9550, 12288), (5450, 8188)]
+)
+def test_only_measured_family_moves(q21_pipe, speed, budget, total):
+ """Every other supported family keeps its flat plan byte for byte, on every speed tier and budget."""
+ names = _family_names()
+ assert "qwen-image-2.1" in names and len(names) > 5
+ plan = _flat_plan(budget, total)
+ for family in names:
+ new = _refine(q21_pipe, plan, family = family, speed = speed)
+ if family == "qwen-image-2.1" and speed in ("default", "max"):
+ continue
+ assert new is plan, family
+
+
+def test_resident_group_keeps_copy_stream_wait(monkeypatch):
+ """A streamed group that prefetched its successor skips its own wait and relies on the successor's onload_ to
+ synchronize the copy stream. A resident successor must still do that wait (else the streamed group computes on
+ weights in flight), and its offload must free nothing."""
+ pytest.importorskip("torch")
+ pytest.importorskip("diffusers.hooks")
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+
+ class Stream:
+ waits = 0
+
+ def synchronize(self):
+ Stream.waits += 1
+
+ module = object()
+ stream = Stream()
+ groups = [
+ types.SimpleNamespace(
+ modules = [],
+ parameters = [],
+ buffers = [],
+ offload_leader = object(),
+ stream = stream,
+ cpu_param_dict = {},
+ offload_to_disk_path = None,
+ )
+ for _ in range(3)
+ ]
+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)
+ dm._keep_groups_resident(module, 1, "cpu")
+ assert all(getattr(g, "_unsloth_resident", False) for g in groups)
+ for g in groups:
+ g.onload_()
+ g.offload_()
+ assert Stream.waits == 3
+
+
+def test_oversized_request_streams_resident_groups_and_restores_them(monkeypatch):
+ """A request past the measured reserve streams the kept groups again (the flat plan's placement), stays
+ bit-identical, and the groups are resident again afterwards."""
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ x = torch.randn(3, 64)
+ ref = net.to("cuda")(x.cuda()).cpu()
+ net.to("cpu")
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ record_stream = True,
+ non_blocking = True,
+ )
+ pipe = types.SimpleNamespace(components = {"transformer": net})
+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0
+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+
+ restore = dm.release_resident_groups(pipe, 1024)
+ assert restore is not None
+ # blocks first, from the last; the top-level group goes last
+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)
+
+ restore()
+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)
+ for _ in range(2):
+ assert torch.equal(net(x.cuda()).cpu(), ref)
+ assert dm.release_resident_groups(types.SimpleNamespace(components = {}), 1024) is None
+
+
+def test_partial_release_frees_only_what_the_request_needs(monkeypatch):
+ torch, net = _cuda_offload_model()
+ from diffusers.hooks import apply_group_offloading
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ )
+ dm._keep_groups_resident(net, 1024, "cuda")
+ pipe = types.SimpleNamespace(components = {"transformer": net})
+ restore = dm.release_resident_groups(pipe, 1) # one ~4 MiB block covers 1 MiB
+ placed = [next(b.parameters()).device.type for b in net.blocks]
+ assert placed == ["cuda"] * 5 + ["cpu"]
+ restore()
+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)
+
+
+def test_measured_request_extra(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)
+ pipe = types.SimpleNamespace(_unsloth_measured_reserve = (2304, "qwen-image-2.1", "default"))
+ # the default request is what the reserve was sized for
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0
+ assert dm.measured_request_extra_mib(pipe, width = 512, height = 512) == 0
+ # a bigger canvas, a batch and reference images each need more
+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 2048) == 8704 - 2304
+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, batch_size = 2) > 0
+ one_ref = int(1024 * 1024 * 0.32)
+ assert dm.measured_request_extra_mib(
+ pipe, width = 1024, height = 1024, condition_pixels = one_ref
+ ) == int(8192 * 0.32 * 1.15)
+ # no measured placement: never releases
+ assert dm.measured_request_extra_mib(object(), width = 2048, height = 2048) == 0
+
+
+def test_generate_releases_and_restores_resident_groups():
+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(
+ encoding = "utf-8"
+ )
+ assert "restore_resident = release_resident_groups(state.pipe, extra_mib, logger)" in src
+ finally_at = src.index("if restore_resident is not None:")
+ assert "restore_resident()" in src[finally_at : finally_at + 200]
+ assert "pipe._unsloth_measured_reserve = (" in src
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 03d8eb01808..721141fb3ab 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -935,6 +935,14 @@ def test_apply_streaming_uses_block_and_leaf_hooks_with_bounded_cpu_memory(monke
"vae": vae,
},
)
+ import core.inference.diffusion_memory as mem
+
+ # no pinnable host RAM: every streamed module stays unpinned (bounded CPU memory), and record_stream keeps the
+ # offload from draining the compute stream
+ monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
effective, _ = apply_memory_plan(
pipe, _manual_plan(OFFLOAD_STREAMING, tiling = True), device = "cuda"
)
@@ -945,7 +953,104 @@ def test_apply_streaming_uses_block_and_leaf_hooks_with_bounded_cpu_memory(monke
("block_level", DEFAULT_GROUP_BLOCKS),
("leaf_level", None),
]
- assert all(call[3:] == (True, True, False, True) for call in calls)
+ assert all(call[3:] == (True, True, True, True) for call in calls)
+
+
+def _streaming_apply_kwargs(
+ monkeypatch,
+ budget,
+ *,
+ env = None,
+ request_background = False,
+):
+ """{component: apply kwargs} and the modules handed to a deferred pinner, for one streaming apply."""
+ import sys
+
+ import core.inference.diffusion_memory as mem
+
+ Module = _install_sized_torch(monkeypatch)
+ seen: dict = {}
+ deferred: list = []
+
+ def _apply(
+ module,
+ *,
+ onload_device,
+ offload_device,
+ offload_type,
+ num_blocks_per_group = None,
+ use_stream = False,
+ non_blocking = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ ):
+ seen[module.name] = {
+ "use_stream": use_stream,
+ "record_stream": record_stream,
+ "low_cpu_mem_usage": low_cpu_mem_usage,
+ "offload_type": offload_type,
+ }
+
+ if "diffusers" not in sys.modules:
+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
+ hooks = types.ModuleType("diffusers.hooks")
+ hooks.apply_group_offloading = _apply
+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
+ if env is None:
+ monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)
+ else:
+ monkeypatch.setenv(mem.STREAMING_PREFETCH_ENV, env)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.delenv(mem.BACKGROUND_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setattr(mem, "install_group_pin_wait", lambda: True)
+ monkeypatch.setattr(
+ mem,
+ "_defer_pinning",
+ lambda pipe, module, device, logger: deferred.append(module.name) or True,
+ )
+ parts = {"transformer": Module(4000), "text_encoder": Module(7500), "vae": Module(200)}
+ for name, module in parts.items():
+ module.name = name
+ pipe = types.SimpleNamespace(transformer = parts["transformer"], components = parts)
+ if request_background:
+ mem.request_background_pins(pipe)
+ mem._apply_streaming_offload(pipe, "cuda", None)
+ return seen, deferred
+
+
+def test_streaming_pins_the_transformer_within_the_host_budget(monkeypatch):
+ # An unpinned host copy is re-pinned on the CPU at every onload, and record_stream=False synchronises the compute
+ # stream after every group, so that pinning ran with the GPU idle (Wan2.2-5B streamed on an L4: see the PR).
+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000)
+ assert seen["transformer"]["low_cpu_mem_usage"] is False
+ assert seen["transformer"]["record_stream"] is True
+ # the encoder does not fit beside the transformer: it stays unpinned, still overlapped
+ assert seen["text_encoder"]["low_cpu_mem_usage"] is True
+ assert seen["text_encoder"]["record_stream"] is True
+ assert deferred == []
+
+
+def test_streaming_pins_everything_on_a_ram_rich_host(monkeypatch):
+ seen, _ = _streaming_apply_kwargs(monkeypatch, 40_000)
+ assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen
+
+
+def test_streaming_pins_off_the_load_path_when_asked(monkeypatch):
+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000, request_background = True)
+ # applied unpinned, then handed to the background pinner; the encoder is outside the budget so never pinned
+ assert seen["transformer"]["low_cpu_mem_usage"] is True
+ assert seen["transformer"]["record_stream"] is True
+ assert deferred == ["transformer"]
+
+
+def test_streaming_prefetch_kill_switch_restores_the_old_kwargs(monkeypatch):
+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 40_000, env = "0", request_background = True)
+ assert all(
+ kw["low_cpu_mem_usage"] is True and kw["record_stream"] is False for kw in seen.values()
+ ), seen
+ assert deferred == []
def test_apply_streaming_does_not_fall_back_to_known_oom_path(monkeypatch):
@@ -2741,3 +2846,265 @@ def test_a_late_encoder_refusal_never_streams_a_torchao_transformer(monkeypatch)
)
assert order == ["hook:text_encoder"]
assert te2.placed is None
+
+
+def test_streamed_transformer_runs_pinned_and_overlapped_on_a_real_gpu(monkeypatch):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ pytest.importorskip("diffusers.hooks")
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, 64)
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(6))
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = x + torch.tanh(block(x))
+ return x
+
+ torch.manual_seed(0)
+ net = Net().eval()
+ x = torch.randn(8, 64)
+ with torch.no_grad():
+ want = net(x)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ mem._apply_streaming_offload(pipe, "cuda", None)
+ groups = [g for g in mem._offload_groups(net) if g.stream is not None]
+ assert groups and all(g.record_stream for g in groups)
+ assert all(t.is_pinned() for g in groups for t in g.cpu_param_dict.values())
+ with torch.no_grad():
+ for _ in range(3): # the first forward traces the order, the later ones prefetch
+ got = net(x.cuda())
+ torch.cuda.synchronize()
+ assert torch.allclose(got.cpu(), want, atol = 1e-5)
+
+
+def _top_group_module(
+ monkeypatch,
+ *,
+ stream = None,
+ torchao = False,
+):
+ """A module whose diffusers registry holds a top-level group, on a fake diffusers.hooks.group_offloading."""
+ import sys
+
+ torch = pytest.importorskip("torch")
+ go = types.ModuleType("diffusers.hooks.group_offloading")
+ go._GROUP_OFFLOADING = "group_offloading"
+ go._is_torchao_tensor = lambda t: torchao
+ hooks = types.ModuleType("diffusers.hooks")
+ hooks.group_offloading = go
+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
+ monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)
+ if "diffusers" not in sys.modules:
+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
+ leaf = torch.nn.Linear(4, 4)
+ group = types.SimpleNamespace(
+ stream = stream,
+ offload_to_disk_path = None,
+ onload_device = types.SimpleNamespace(type = "cuda"),
+ modules = [leaf],
+ parameters = [],
+ buffers = [],
+ onload_ = "diffusers",
+ offload_ = "diffusers",
+ )
+ hook = types.SimpleNamespace(group = group)
+ module = types.SimpleNamespace(
+ _diffusers_hook = types.SimpleNamespace(
+ get_hook = lambda name: hook if name == "group_offloading" else None
+ )
+ )
+ return module, group
+
+
+ "env", ["UNSLOTH_DIFFUSION_PIN_TOP_GROUP", "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"]
+)
+def test_top_level_group_kill_switch(monkeypatch, env):
+ import core.inference.diffusion_memory as mem
+
+ module, group = _top_group_module(monkeypatch)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ monkeypatch.setenv(env, "0")
+ assert mem._pin_top_level_group(module) is False
+ assert group.onload_ == "diffusers" and group.offload_ == "diffusers"
+
+
+def test_top_level_group_respects_the_pinned_allocator_rounding(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ module, group = _top_group_module(monkeypatch)
+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ # a 3 MiB weight pins as a 4 MiB block: a 3 MiB budget must refuse it
+ group.modules = [
+ types.SimpleNamespace(
+ parameters = lambda: [__import__("torch").empty(3 << 18)], buffers = lambda: []
+ )
+ ]
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 3)
+ assert mem._pin_top_level_group(module) is False
+ assert group.onload_ == "diffusers"
+
+
+def test_top_level_group_counts_against_the_running_pin_total(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ module, group = _top_group_module(monkeypatch)
+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ group.modules = [
+ types.SimpleNamespace(
+ parameters = lambda: [__import__("torch").empty(1 << 18)], buffers = lambda: []
+ )
+ ]
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 8)
+ # 8 MiB budget, 8 MiB of encoders already pinned: the 1 MiB top group no longer fits
+ assert mem._pin_top_level_group(module, None, [8]) is False
+ assert group.onload_ == "diffusers"
+ total = [4]
+ assert mem._pin_top_level_group(module, None, total) is True
+ assert total == [5]
+
+
[email protected]("kind", ["streamed_group", "torchao"])
+def test_top_level_group_left_alone_when_not_the_streamless_top_group(monkeypatch, kind):
+ import core.inference.diffusion_memory as mem
+
+ module, group = _top_group_module(
+ monkeypatch,
+ stream = object() if kind == "streamed_group" else None,
+ torchao = kind == "torchao",
+ )
+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+ assert mem._pin_top_level_group(module) is False
+ assert group.onload_ == "diffusers"
+
+
[email protected]("tier", ["group", "streaming"])
+def test_top_level_weights_onload_from_one_pinned_copy_on_a_real_gpu(monkeypatch, tier):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ pytest.importorskip("diffusers.hooks")
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj_in = torch.nn.Linear(64, 64)
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(4))
+ self.proj_out = torch.nn.Linear(64, 64)
+
+ def forward(self, x):
+ x = self.proj_in(x)
+ for block in self.blocks:
+ x = x + torch.tanh(block(x))
+ return self.proj_out(x)
+
+ torch.manual_seed(0)
+ net = Net().eval()
+ x = torch.randn(8, 64)
+ with torch.no_grad():
+ want = net(x)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ if tier == "group":
+ assert mem._apply_group_offload(pipe, "cuda", logger = None) is True
+ else:
+ mem._apply_streaming_offload(pipe, "cuda", None)
+ with torch.no_grad():
+ got = [net(x.cuda()).cpu() for _ in range(3)]
+ torch.cuda.synchronize()
+ # offloaded between forwards onto the same pinned host copy: no device-to-host copy into a fresh host buffer
+ ptr = net.proj_out.weight.data_ptr()
+ assert net.proj_out.weight.device.type == "cpu" and net.proj_out.weight.is_pinned()
+ with torch.no_grad():
+ got.append(net(x.cuda()).cpu())
+ torch.cuda.synchronize()
+ assert net.proj_out.weight.data_ptr() == ptr
+ assert all(torch.allclose(g, want, atol = 1e-5) for g in got)
+
+
+def test_top_level_weights_replaced_while_offloaded_are_picked_up(monkeypatch):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ pytest.importorskip("diffusers.hooks")
+ import copy
+
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(16, 16) for _ in range(3))
+ self.proj_out = torch.nn.Linear(16, 16)
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = x + torch.tanh(block(x))
+ return self.proj_out(x)
+
+ torch.manual_seed(0)
+ net = Net().eval()
+ ref = copy.deepcopy(net)
+ x = torch.randn(4, 16)
+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})
+ mem._apply_streaming_offload(pipe, "cuda", None)
+ with torch.no_grad():
+ for _ in range(2):
+ net(x.cuda())
+ torch.cuda.synchronize()
+ # e.g. a .to() conversion or an adapter fused on the host while the weights sit offloaded
+ net.proj_out.weight.data = net.proj_out.weight.data * 2
+ ref.proj_out.weight.data = ref.proj_out.weight.data * 2
+ got = net(x.cuda()).cpu()
+ want = ref(x)
+ assert torch.allclose(got, want, atol = 1e-5)
+
+
+def test_streaming_counts_pinned_encoders_before_a_torchao_denoiser_pins(monkeypatch):
+ # The torchao denoiser pins within a running total; encoders the plan pins must already be on it (as on the group
+ # tier), or the two together exceed the pinnable host budget.
+ import core.inference.diffusion_memory as mem
+
+ seen_totals: dict = {}
+ real = mem._torchao_group_offload_kwargs
+
+ def _spy(
+ module,
+ kwargs,
+ pinned_mib = None,
+ ):
+ seen_totals[module.name] = pinned_mib[0] if pinned_mib else None
+ return real(module, kwargs, pinned_mib)
+
+ monkeypatch.setattr(mem, "_torchao_group_offload_kwargs", _spy)
+ _streaming_apply_kwargs(monkeypatch, 40_000)
+ # the 7500 MiB encoder, rounded up to a power of two like the pinned allocator
+ assert seen_totals["transformer"] == 8192
diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.py
index 3abd8e186ca..e6fcf4e7143 100644
--- a/studio/backend/tests/test_diffusion_offload_precision.py
+++ b/studio/backend/tests/test_diffusion_offload_precision.py
@@ -82,7 +82,8 @@ SURVIVAL = {
s: {v: (True, OFFLOAD_GROUP) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
"group_dit_streamed": {
- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_GROUP)},
+ # 0.17 streams int8 through its pinnable Int8Tensor (UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17)
+ "int8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},
"fp8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},
"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
},
@@ -90,7 +91,11 @@ SURVIVAL = {
s: {v: (True, OFFLOAD_MODEL) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")
},
"model_too_big": {
- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},
+ "int8": {
+ T016: (False, None),
+ T017: (True, OFFLOAD_STREAMING),
+ T018: (True, OFFLOAD_STREAMING),
+ },
"fp8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
@@ -99,7 +104,11 @@ SURVIVAL = {
"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},
},
"streaming": {
- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},
+ "int8": {
+ T016: (False, None),
+ T017: (True, OFFLOAD_STREAMING),
+ T018: (True, OFFLOAD_STREAMING),
+ },
"fp8": {
T016: (False, None),
T017: (True, OFFLOAD_STREAMING),
@@ -120,7 +129,10 @@ CASES = [
@pytest.mark.parametrize("placement, scheme, version, survives, policy", CASES)
-def test_survival_table(placement, scheme, version, survives, policy):
+def test_survival_table(monkeypatch, placement, scheme, version, survives, policy):
+ # the table is about versions, not about which torchao this test host happens to have
+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)
plan = PLACEMENTS[placement]
assert torchao_survives_plan(plan, scheme, torchao_version = version) is survives
placed = torchao_offload_plan(plan, scheme, torchao_version = version)
@@ -246,14 +258,18 @@ NO_STREAM = {"onload_device": "cuda", "use_stream": False}
(("Int8Tensor",), True, 10**7, "pinned"),
(("Float8Tensor",), True, 10**7, "pinned"),
(("Int8Tensor",), True, None, "no stream"),
- # torchao 0.17 default int8: no aten.is_pinned, so no copy stream
- (("LinearActivationQuantizedTensor",), False, 10**7, "no stream"),
+ # torchao 0.17 default int8: no aten.is_pinned of its own; streams once Studio registers the pin ops
+ (("LinearActivationQuantizedTensor",), False, 10**7, "as is"),
+ (("LinearActivationQuantizedTensor",), False, 10**7, "no stream, no pin ops"),
],
)
def test_group_offload_kwargs_per_weight_class(
monkeypatch, classes, low_cpu_mem_usage, pin_budget, expected
):
monkeypatch.setattr(mem, "_pin_budget_mib", lambda: pin_budget)
+ # whether this host's torchao ships the v1 classes must not decide the table
+ pin_ops = expected != "no stream, no pin ops"
+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: pin_ops, raising = False)
kwargs = {**STREAM_KW, "low_cpu_mem_usage": low_cpu_mem_usage}
out = mem._torchao_group_offload_kwargs(_Module(*classes, plain = True), kwargs)
if expected == "as is":
@@ -509,9 +525,12 @@ def test_auto_keeps_int8_on_the_offload_tier(planner, family, base, gib, expecte
assert planner(family, base, gib) == expected
[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted fp8")])
+# 0.17 now streams int8 (pinnable Int8Tensor), so it takes the same hosted int8 as 0.18 instead of falling to fp8
[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted int8")])
def test_older_torchao_streamed_seeds(planner, monkeypatch, version, expected):
monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)
+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)
precision, _policy = planner(
"hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers", 16
)
@@ -587,6 +606,8 @@ def test_quantised_blocks_survive_group_offload_under_no_grad(
want = resident(x)
for _ in range(3):
assert torch.equal(offloaded(x), want)
+ if type(next(offloaded.parameters())).__name__ == "LinearActivationQuantizedTensor":
+ return # torchao <= 0.17 v1 int8 (pin-op shim): its onload is a plain re-wrap, which inference_mode allows
with pytest.raises(Exception), torch.inference_mode():
offloaded(x)
diff --git a/studio/backend/tests/test_diffusion_offload_seed_balanced.py b/studio/backend/tests/test_diffusion_offload_seed_balanced.py
new file mode 100644
index 00000000000..0b657a1a1f7
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_seed_balanced.py
@@ -0,0 +1,276 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Two small-card load failures, Qwen-Image-2.1 sized, CPU only.
+
+1. An offloading plan must seed a pre-quantized (int8 / fp8) denoiser on the host: seeding it onto the GPU whole
+ (~7 GiB int8) left the 8 GB streaming hooks no room for their first block.
+2. An explicit ``memory_mode=balanced`` is fit-checked against the loaded companions, walking down the streamed
+ tiers, instead of loading every companion resident and OOMing at the first generate.
+"""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+import pytest
+import torch
+
+import core.inference.diffusion_memory as dm
+from core.inference.diffusion_memory import (
+ BALANCED_FIT_CHECK_ENV,
+ DeviceMemory,
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
+ OFFLOAD_NONE,
+ OFFLOAD_STREAMING,
+ PREQUANT_SEED_ON_HOST_ENV,
+ plan_diffusion_memory,
+ prequant_seed_device,
+ refine_balanced_plan_for_components,
+ refine_memory_plan_for_components,
+)
+
+MIB = 1024 * 1024
+BACKEND = Path(__file__).resolve().parents[1]
+
+# Qwen-Image-2.1 auto (int8 DiT + fp8 TE) as the planner prices it, and as it loads.
+DENSE = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)
+# The explicit transformer_quant=int8 path prices the encoder dense (16.7 GB) while it loads fp8 (9.0 GB).
+DENSE_EXPLICIT = dict(
+ model_dense_mib = 31566, companion_dense_mib = 18024, text_encoder_dense_mib = 16689
+)
+TE_LOADED_MIB = 8960
+VAE_LOADED_MIB = 645
+DIT_LOADED_MIB = 7448
+
+
+class _Target:
+ supports_model_cpu_offload = True
+
+
+def _card(free_mib, total_mib):
+ return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, total_mib)
+
+
+# (ballast free, simulated total) of the user's budget cells
+CARD_24 = _card(24176, 24576)
+CARD_16 = _card(15976, 16376)
+CARD_12 = _card(11888, 12288)
+CARD_8 = _card(7788, 8188)
+
+
+def _plan(
+ card,
+ mode = None,
+ sizes = DENSE,
+):
+ return plan_diffusion_memory(
+ target = _Target(),
+ device_memory = card,
+ runtime_headroom_mib = 8192,
+ requested_mode = mode,
+ **sizes,
+ )
+
+
+def _module(mib):
+ m = torch.nn.Module()
+ # meta: sized like the loaded component, allocates nothing
+ m.w = torch.nn.Parameter(
+ torch.empty(mib * MIB, dtype = torch.uint8, device = "meta"), requires_grad = False
+ )
+ return m
+
+
+class _Pipe:
+ def __init__(self):
+ self.transformer = _module(DIT_LOADED_MIB)
+ self.text_encoder = _module(TE_LOADED_MIB)
+ self.vae = _module(VAE_LOADED_MIB)
+ self.components = {
+ "transformer": self.transformer,
+ "text_encoder": self.text_encoder,
+ "vae": self.vae,
+ "tokenizer": object(),
+ "scheduler": object(),
+ }
+
+
[email protected](autouse = True)
+def _clean_env(monkeypatch):
+ monkeypatch.delenv(PREQUANT_SEED_ON_HOST_ENV, raising = False)
+ monkeypatch.delenv(BALANCED_FIT_CHECK_ENV, raising = False)
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)
+
+
+# ---------------------------------------------------------------- 1. host seed for offloaded prequant denoisers
+
+
[email protected]("card", [CARD_8, CARD_12, CARD_16])
+def test_offloading_auto_plan_seeds_int8_on_the_host(card):
+ plan = (
+ dm.torchao_streaming_plan(_plan(card))
+ if _plan(card).offload_policy == OFFLOAD_MODEL
+ else _plan(card)
+ )
+ assert plan.offload_policy != OFFLOAD_NONE
+ assert not dm.plan_keeps_transformer_resident(plan)
+ assert prequant_seed_device(plan, "cuda", "int8") == "cpu"
+ assert prequant_seed_device(plan, "cuda:1", "fp8") == "cpu"
+
+
+def test_resident_plans_keep_the_gpu_seed():
+ # 24 GB: transformer resident, encoders streamed -> unchanged (seed lands where it runs)
+ plan24 = _plan(CARD_24)
+ assert dm.plan_keeps_transformer_resident(plan24)
+ assert prequant_seed_device(plan24, "cuda", "int8") == "cuda"
+ full = _plan(_card(47000, 49140))
+ assert full.offload_policy == OFFLOAD_NONE
+ assert prequant_seed_device(full, "cuda:0", "int8") == "cuda:0"
+
+
+def test_seed_device_scheme_and_kill_switch(monkeypatch):
+ plan = dm.torchao_streaming_plan(_plan(CARD_8))
+ # schemes never measured under offload keep today's placement
+ assert prequant_seed_device(plan, "cuda", "nvfp4") == "cuda"
+ assert prequant_seed_device(plan, "cuda", None) == "cuda"
+ monkeypatch.setenv(PREQUANT_SEED_ON_HOST_ENV, "0")
+ assert prequant_seed_device(plan, "cuda", "int8") == "cuda"
+
+
+def test_denoiser_seed_forwards_the_placement(monkeypatch):
+ import core.inference.diffusion_denoiser_prequant as dp
+ import core.inference.diffusion_prequant as prequant
+ import diffusers
+
+ seen = {}
+
+ def fake_load(cls, base, source, **kwargs):
+ seen.update(kwargs)
+ return torch.nn.Linear(2, 2)
+
+ class _Fam:
+ name = "qwen-image-2.1"
+ transformer_class = "QwenImage21Transformer2DModel"
+
+ monkeypatch.setattr(prequant, "load_prequantized_transformer", fake_load)
+ monkeypatch.setattr(dp, "pipeline_seed_supported", lambda fam: True)
+ monkeypatch.setattr(dp, "denoiser_prequant_source", lambda *a, **k: object())
+ monkeypatch.setattr(diffusers, "QwenImage21Transformer2DModel", object, raising = False)
+ out = dp.denoiser_prequant_pipe_kwargs(
+ _Fam(),
+ "Qwen/Qwen-Image-2.1",
+ scheme = "int8",
+ dtype = torch.bfloat16,
+ device = "cuda",
+ placement_device = "cpu",
+ )
+ assert out
+ assert seen["device"] == "cuda" # kernels still selected for the compute device
+ assert seen["placement_device"] == "cpu"
+
+
+def test_loader_materialises_on_the_placement_device():
+ src = (BACKEND / "core/inference/diffusion_prequant.py").read_text()
+ assert "transformer = transformer.to(placement_device or device)" in src
+ assert "transformer = transformer.to(device)\n" not in src
+
+
+def test_pipeline_seed_call_passes_the_plan_placement():
+ tree = ast.parse((BACKEND / "core/inference/diffusion.py").read_text())
+ calls = [
+ n
+ for n in ast.walk(tree)
+ if isinstance(n, ast.Call)
+ and getattr(n.func, "id", None) == "denoiser_prequant_pipe_kwargs"
+ ]
+ assert calls, "pipeline seed call not found"
+ for call in calls:
+ kw = {k.arg: k.value for k in call.keywords}
+ assert "placement_device" in kw
+ value = kw["placement_device"]
+ assert (
+ isinstance(value, ast.Call)
+ and getattr(value.func, "id", None) == "prequant_seed_device"
+ )
+ assert [getattr(a, "id", None) for a in value.args] == [
+ "plan",
+ "device",
+ "pipeline_seed_scheme",
+ ]
+
+
+# ---------------------------------------------------------------- 2. balanced fit check
+
+
+def test_balanced_that_fits_is_unchanged():
+ # 24 GB, explicit int8: the estimate (dense TE) says companions do not fit, the loaded ones do -> keep it
+ plan = _plan(CARD_24, "balanced", DENSE_EXPLICIT)
+ assert plan.offload_policy == OFFLOAD_GROUP and not plan.stream_text_encoders
+ out = refine_balanced_plan_for_components(_Pipe(), plan)
+ assert out.offload_policy == OFFLOAD_GROUP
+ assert not out.stream_text_encoders
+ assert out.reasons == plan.reasons
+
+
[email protected]("sizes", [DENSE, DENSE_EXPLICIT])
+def test_balanced_16gb_streams_the_text_encoders(sizes):
+ plan = _plan(CARD_16, "balanced", sizes)
+ out = refine_balanced_plan_for_components(_Pipe(), plan)
+ assert out.offload_policy == OFFLOAD_GROUP
+ assert out.stream_text_encoders
+ assert out.stream_transformer
+
+
+def test_balanced_12gb_drops_to_whole_module_offload():
+ plan = _plan(CARD_12, "balanced", DENSE_EXPLICIT)
+ out = refine_memory_plan_for_components(
+ _Pipe(), refine_balanced_plan_for_components(_Pipe(), plan)
+ )
+ # the 9.0 GB encoder fits the 9.8 GB budget whole, so whole-module offload stays
+ assert out.offload_policy == OFFLOAD_MODEL
+ assert out.vae_tiling
+
+
+def test_balanced_8gb_streams_granularly():
+ plan = _plan(CARD_8, "balanced", DENSE_EXPLICIT)
+ out = refine_memory_plan_for_components(
+ _Pipe(), refine_balanced_plan_for_components(_Pipe(), plan)
+ )
+ assert out.offload_policy == OFFLOAD_STREAMING
+
+
[email protected]("card", [CARD_12, CARD_8])
+def test_balanced_torchao_denoiser_streams_instead_of_model_offload(monkeypatch, card):
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)
+ out = refine_balanced_plan_for_components(_Pipe(), _plan(card, "balanced"))
+ assert out.offload_policy == OFFLOAD_STREAMING
+
+
+def test_balanced_kill_switch_and_scope(monkeypatch):
+ plan16 = _plan(CARD_16, "balanced")
+ auto16 = _plan(CARD_16)
+ # auto is not touched
+ assert refine_balanced_plan_for_components(_Pipe(), auto16) is auto16
+ monkeypatch.setenv(BALANCED_FIT_CHECK_ENV, "0")
+ assert refine_balanced_plan_for_components(_Pipe(), plan16) is plan16
+
+
+def test_balanced_refinement_is_wired_before_component_refinement():
+ src = (BACKEND / "core/inference/diffusion.py").read_text()
+ assert (
+ "refine_memory_plan_for_components(\n pipe, refine_balanced_plan_for_components(pipe, plan)\n"
+ in src
+ )
+
+
+def test_measured_placement_honours_the_legacy_cpu_offload_flag():
+ # cpu_offload=True without a memory_mode asks for offload; the measured refinement must not make that load resident
+ src = (BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8")
+ call = src.index("plan = refine_plan_from_loaded_weights(")
+ guard = src.rindex("\n", 0, src.rindex("\n", 0, call))
+ assert (
+ "if not cpu_offload or normalize_memory_mode(memory_mode) is not None:" in src[guard:call]
+ )
diff --git a/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.py b/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.py
new file mode 100644
index 00000000000..b4cea88761c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Behaviour of the two small-card fixes through entry points that exist before and after them.
+
+Imports only names present on main, so on main these fail by ASSERTION, not ImportError:
+
+1. The pipeline seed call in diffusion.py, evaluated as written, drives ``load_prequantized_transformer`` (checkpoint
+ I/O faked) under an offloading Qwen-Image-2.1 plan: the module must be materialised on the host, not the GPU.
+2. The post-load refinement expression in diffusion.py, evaluated as written, applied to a balanced 16 GB plan and a
+ pipe whose loaded fp8 text encoder is 8960 MiB: the encoder must not stay resident.
+"""
+
+from __future__ import annotations
+
+import ast
+import inspect
+from pathlib import Path
+
+import pytest
+import torch
+
+import core.inference.diffusion as diffusion
+import core.inference.diffusion_memory as dm
+import core.inference.diffusion_prequant as prequant
+from core.inference.diffusion_memory import (
+ DeviceMemory,
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
+ plan_diffusion_memory,
+ torchao_streaming_plan,
+)
+
+BACKEND = Path(__file__).resolve().parents[1]
+DIFFUSION_SRC = (BACKEND / "core/inference/diffusion.py").read_text()
+MIB = 1024 * 1024
+SIZES_AUTO = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)
+SIZES_EXPLICIT_INT8 = dict(
+ model_dense_mib = 31566, companion_dense_mib = 18024, text_encoder_dense_mib = 16689
+)
+
+
+class _Target:
+ supports_model_cpu_offload = True
+
+
+def _plan(
+ free,
+ total,
+ mode = None,
+ sizes = SIZES_AUTO,
+):
+ return plan_diffusion_memory(
+ target = _Target(),
+ device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", free, total),
+ runtime_headroom_mib = 8192,
+ requested_mode = mode,
+ **sizes,
+ )
+
+
+def _call(name):
+ calls = [
+ n
+ for n in ast.walk(ast.parse(DIFFUSION_SRC))
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == name
+ ]
+ assert len(calls) == 1, f"expected one {name}() call in diffusion.py"
+ return calls[0]
+
+
+# ---------------------------------------------------------------- 1. seed placement
+
+
+class _FakeDiT(torch.nn.Module):
+ placed: list = []
+
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(4, 4)
+
+ @classmethod
+ def from_config(cls, config):
+ return cls()
+
+ def to(self, *args, **kwargs): # record, never move (CPU-only test host)
+ _FakeDiT.placed.append(str(args[0] if args else kwargs.get("device")))
+ return self
+
+
+def fake_checkpoint(monkeypatch):
+ _FakeDiT.placed = []
+ state = {k: v.clone() for k, v in torch.nn.Linear(4, 4).state_dict().items()}
+ state = {f"proj.{k}": v for k, v in state.items()}
+ monkeypatch.setattr(prequant, "_resolve_checkpoint_path", lambda *a, **k: "fake.safetensors")
+ monkeypatch.setattr(
+ prequant, "_load_prequant_checkpoint", lambda *a, **k: {"state_dict": state, "metadata": {}}
+ )
+ monkeypatch.setattr(prequant, "_validate_checkpoint", lambda *a, **k: True)
+ monkeypatch.setattr(prequant, "_verify_packed_fingerprint", lambda *a, **k: True)
+ monkeypatch.setattr(prequant, "_pin_kernel_preference", lambda *a, **k: 0)
+ monkeypatch.setattr(prequant, "_load_transformer_config", lambda *a, **k: {})
+
+
+class _Source:
+ kind = "repo"
+ location = "unsloth/Qwen-Image-2.1-FP8"
+ filename = "Qwen-Image-2.1-INT8.safetensors"
+
+
+def _seed_through_call_site(
+ plan,
+ device = "cuda",
+ scheme = "int8",
+):
+ """Evaluate every keyword of diffusion.py's seed call, then run the loader with the ones it accepts."""
+ call = _call("denoiser_prequant_pipe_kwargs")
+ scope = dict(vars(diffusion))
+ scope.update(plan = plan, device = device, pipeline_seed_scheme = scheme)
+ kwargs = {}
+ for kw in call.keywords:
+ if kw.arg in ("device", "placement_device"):
+ kwargs[kw.arg] = eval(compile(ast.Expression(kw.value), "<seed-call>", "eval"), scope)
+ accepted = inspect.signature(prequant.load_prequantized_transformer).parameters
+ kwargs = {k: v for k, v in kwargs.items() if k in accepted}
+ module = prequant.load_prequantized_transformer(
+ _FakeDiT, "Qwen/Qwen-Image-2.1", _Source(), dtype = torch.bfloat16, scheme = scheme, **kwargs
+ )
+ assert module is not None, "fake checkpoint load failed"
+ return _FakeDiT.placed[-1]
+
+
+ "free,total", [(7788, 8188), (11888, 12288), (15976, 16376)], ids = ["8GB", "12GB", "16GB"]
+)
+def test_offloading_plan_materialises_the_seed_on_the_host(fake_checkpoint, free, total):
+ plan = _plan(free, total)
+ if plan.offload_policy == OFFLOAD_MODEL:
+ plan = torchao_streaming_plan(plan) # what torchao_offload_plan makes of it for int8
+ assert not dm.plan_keeps_transformer_resident(plan)
+ assert _seed_through_call_site(plan) == "cpu"
+
+
+def test_resident_plan_keeps_the_seed_on_the_device(fake_checkpoint):
+ plan = _plan(24176, 24576) # transformer resident, encoders streamed
+ assert dm.plan_keeps_transformer_resident(plan)
+ assert _seed_through_call_site(plan) == "cuda"
+
+
+# ---------------------------------------------------------------- 2. balanced refinement
+
+
+def _meta(mib):
+ m = torch.nn.Module()
+ m.w = torch.nn.Parameter(
+ torch.empty(mib * MIB, dtype = torch.uint8, device = "meta"), requires_grad = False
+ )
+ return m
+
+
+class _Pipe:
+ def __init__(self):
+ self.transformer = _meta(7448)
+ self.text_encoder = _meta(8960) # fp8 Qwen3-VL as loaded
+ self.vae = _meta(645)
+ self.components = {
+ "transformer": self.transformer,
+ "text_encoder": self.text_encoder,
+ "vae": self.vae,
+ }
+
+
+def _refine_as_loaded(pipe, plan):
+ """The expression diffusion.py assigns to ``refined_plan`` after the load, evaluated as written."""
+ for node in ast.walk(ast.parse(DIFFUSION_SRC)):
+ if (
+ isinstance(node, ast.Assign)
+ and len(node.targets) == 1
+ and getattr(node.targets[0], "id", None) == "refined_plan"
+ ):
+ scope = dict(vars(diffusion))
+ scope.update(pipe = pipe, plan = plan)
+ return eval(compile(ast.Expression(node.value), "<refine>", "eval"), scope)
+ raise AssertionError("refined_plan assignment not found in diffusion.py")
+
+
+def no_torchao(monkeypatch):
+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)
+
+
[email protected]("sizes", [SIZES_AUTO, SIZES_EXPLICIT_INT8], ids = ["auto_quant", "int8"])
+def test_balanced_16gb_does_not_keep_the_9gib_encoder_resident(no_torchao, sizes):
+ plan = _plan(15976, 16376, "balanced", sizes)
+ assert (
+ plan.offload_policy == OFFLOAD_GROUP and not plan.stream_text_encoders
+ ) # the plan main ships
+ final = _refine_as_loaded(_Pipe(), plan)
+ encoder_resident = final.offload_policy == OFFLOAD_GROUP and not final.stream_text_encoders
+ assert not encoder_resident
+
+
[email protected]("free,total", [(11888, 12288), (7788, 8188)], ids = ["12GB", "8GB"])
+def test_balanced_small_cards_leave_group_offload(no_torchao, free, total):
+ final = _refine_as_loaded(_Pipe(), _plan(free, total, "balanced", SIZES_EXPLICIT_INT8))
+ assert final.offload_policy != OFFLOAD_GROUP
+
+
+def test_balanced_24gb_keeps_its_placement(no_torchao):
+ plan = _plan(24176, 24576, "balanced", SIZES_EXPLICIT_INT8)
+ final = _refine_as_loaded(_Pipe(), plan)
+ assert final.offload_policy == OFFLOAD_GROUP and not final.stream_text_encoders
diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.py
index 75ef97e1032..eacd9ef23f0 100644
--- a/studio/backend/tests/test_image_gallery.py
+++ b/studio/backend/tests/test_image_gallery.py
@@ -540,3 +540,55 @@ def test_thumbnail_is_a_downscaled_webp_of_the_png():
with Image.open(io.BytesIO(data)) as im:
assert im.format == "WEBP"
assert im.size == (16, 8)
+
+
+def _noisy(size = 128):
+ # Incompressible-ish content so the deflate level changes the encoded bytes.
+ import random
+ rng = random.Random(1234)
+ return Image.frombytes(
+ "RGB", (size, size), bytes(rng.randrange(256) for _ in range(size * size * 3))
+ )
+
+
+def _encode(image, level):
+ from PIL.PngImagePlugin import PngInfo
+
+ info = PngInfo()
+ info.add_text("unsloth", _json.dumps(_meta()))
+ info.add_text("parameters", gallery._params_text(_meta()))
+ buf = io.BytesIO()
+ image.save(buf, format = "PNG", pnginfo = info, compress_level = level)
+ return buf.getvalue()
+
+
+def test_gallery_png_uses_fast_deflate_by_default(monkeypatch):
+ # The gallery encode is on the request path after the last denoise step; the default level must be the fast one.
+ monkeypatch.delenv(gallery.PNG_COMPRESS_LEVEL_ENV, raising = False)
+ image = _noisy()
+ data = gallery._png_bytes(image, _meta())
+ assert data == _encode(image, 1)
+ assert data != _encode(image, 6)
+
+
+def test_gallery_png_level_kill_switch_restores_pillow_default(monkeypatch):
+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, "6")
+ image = _noisy()
+ assert gallery._png_bytes(image, _meta()) == _encode(image, 6)
+
+
[email protected]("raw", ["", "fast", "-1", "10"])
+def test_gallery_png_level_rejects_invalid_values(monkeypatch, raw):
+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, raw)
+ assert gallery.png_compress_level() == 1
+
+
[email protected]("level", ["0", "1", "6", "9"])
+def test_gallery_png_is_lossless_at_any_level(monkeypatch, level):
+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, level)
+ image = _noisy(64)
+ record = gallery.save(image, _meta())
+ raw = base64.b64decode(gallery.image_b64(record["id"]))
+ with Image.open(io.BytesIO(raw)) as im:
+ assert im.convert("RGB").tobytes() == image.tobytes()
+ assert _json.loads(im.text["unsloth"])["seed"] == 7