Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Qwen-Image-2.1 placement from measured sizes, int8 under offload on torchao 0.17, balanced fit check (#12408)
Daniel Hancf89ed7a4b45
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 3a3e4a3529a..63961bcf64e 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -121,13 +121,18 @@ from .diffusion_memory import (plan_diffusion_memory,plan_fits_total_capacity,plan_keeps_transformer_resident,+ prequant_seed_device,raise_on_image_activation_shortfall,raise_on_unified_memory_shortfall,reclaimable_snapshot_device_memory,reclaim_host_memory,reclaim_offload_host_memory,release_pinned_host_memory,+ refine_balanced_plan_for_components,refine_memory_plan_for_components,+ refine_plan_from_loaded_weights,+ release_resident_groups,+ measured_request_extra_mib,settled_snapshot_device_memory,snapshot_device_memory,_torchao_stream_pinnable,@@ -6160,6 +6165,10 @@ class DiffusionBackend:scheme = pipeline_seed_scheme,dtype = dtype,device = device,+ # offloading plans seed it on the host: a GPU seed OOMed 8 GB cards+ placement_device = prequant_seed_device(+ plan, device, pipeline_seed_scheme+ ),hf_token = hf_token,target = target,path_override = transformer_prequant_path,@@ -6802,7 +6811,9 @@ class DiffusionBackend:# Whole-module offload still onloads one complete component for its forward. Refine it from the# loaded, possibly quantized weights so an oversized text encoder uses leaf streaming instead of# failing during prompt encoding.- refined_plan = refine_memory_plan_for_components(pipe, plan)+ refined_plan = refine_memory_plan_for_components(+ pipe, refine_balanced_plan_for_components(pipe, plan)+ )if refined_plan.offload_policy != plan.offload_policy:logger.info("diffusion.memory: refined policy %s -> %s (%s)",@@ -6811,6 +6822,24 @@ class DiffusionBackend:"; ".join(refined_plan.reasons),)plan = refined_plan+ # not under the legacy cpu_offload flag alone: that request asked for offload+ if not cpu_offload or normalize_memory_mode(memory_mode) is not None:+ plan = refine_plan_from_loaded_weights(+ pipe,+ plan,+ family = fam.name,+ speed_mode = effective_speed,+ logger = logger,+ )+ headroom = plan.estimates.get("measured_runtime_headroom_mib")+ if headroom and (+ plan.resident_transformer_mib or plan.resident_text_encoder_mib+ ):+ pipe._unsloth_measured_reserve = (+ int(headroom),+ fam.name,+ effective_speed,+ )from .diffusion_qwenimage21_vision import configure_vision_attention@@ -8573,6 +8602,7 @@ class DiffusionBackend:# Reset in the finally, so a failed or cancelled generation frees its reused outputs.static_skip_pipe = Nonerestore_vae: Optional[Callable[[], None]] = None+ restore_resident: Optional[Callable[[], None]] = Nonetry:self._state_device_target(state)# The local `state` ref keeps the pipe alive even if unload() nulls _state. Resolve the per-image@@ -8896,6 +8926,25 @@ class DiffusionBackend:)guard_batch = _activation_guard_batch(chunks)guard_target = self._state_device_target(state)+ # past what the measured placement reserved: stream resident groups again for this call+ extra_mib = measured_request_extra_mib(+ state.pipe,+ width = guard_width,+ height = guard_height,+ batch_size = guard_batch,+ condition_pixels = (+ int(+ (1 + len(ref_extra))+ * ref_resolution+ * ref_resolution+ * getattr(fam, "condition_pixel_weight", 1.0)+ )+ if ref_resolution is not None+ else 0+ ),+ )+ if extra_mib > 0:+ restore_resident = release_resident_groups(state.pipe, extra_mib, logger)guard_kwargs = dict(# NOT the settled snapshot the load uses: that one calls empty_cache(), which is right once# per load but wrong on a per-generation path, since it releases every cached block and the@@ -9223,6 +9272,11 @@ class DiffusionBackend:logger.debug("diffusion.step_skip: reset failed: %s", exc)if restore_vae is not None:restore_vae()+ if restore_resident is not None:+ try:+ restore_resident()+ except Exception as exc: # noqa: BLE001 - the groups keep streaming+ logger.warning("diffusion.memory: re-residency failed: %s", exc)with self._generation_cancel_lock:if self._active_generate_cancel is cancel:self._active_generate_cancel = Nonediff --git a/studio/backend/core/inference/diffusion_denoiser_prequant.py b/studio/backend/core/inference/diffusion_denoiser_prequant.pyindex 2855869a85e..a598a6a5b01 100644--- a/studio/backend/core/inference/diffusion_denoiser_prequant.py+++ b/studio/backend/core/inference/diffusion_denoiser_prequant.py@@ -89,9 +89,11 @@ def denoiser_prequant_pipe_kwargs(local_files_only: bool = False,cache_dir: Optional[str] = None,logger: Any = None,+ placement_device: Optional[str] = None,) -> dict[str, Any]:"""``{"transformer": module}`` for pipeline assembly, or ``{}`` when it cannot be seeded, which- obliges the caller to re-plan at bf16 and restore the dropped shards."""+ obliges the caller to re-plan at bf16 and restore the dropped shards. ``placement_device``: where the+ module is materialised (see ``load_prequantized_transformer``)."""try:if not pipeline_seed_supported(fam):return {}@@ -133,6 +135,7 @@ def denoiser_prequant_pipe_kwargs(cache_dir = cache_dir,local_files_only = local_files_only,logger = logger,+ placement_device = placement_device,)if module is None:if logger is not None:diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex 43bed7f4738..2651f7bfe06 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -564,6 +564,9 @@ class MemoryPlan:stream_text_encoders: bool = False# False only on the tier keeping the transformer resident and streaming just the text encoders.stream_transformer: bool = True+ # MiB of the streamed denoiser / encoders kept resident (whole groups); None streams every group.+ resident_transformer_mib: Optional[int] = None+ resident_text_encoder_mib: Optional[int] = None@propertydef engages_offload(self) -> bool:@@ -580,6 +583,8 @@ class MemoryPlan:"reasons": list(self.reasons),"stream_text_encoders": self.stream_text_encoders,"stream_transformer": self.stream_transformer,+ "resident_transformer_mib": self.resident_transformer_mib,+ "resident_text_encoder_mib": self.resident_text_encoder_mib,}@@ -1002,8 +1007,73 @@ def plan_keeps_transformer_resident(plan: Any) -> bool:return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_transformer", True))+PREQUANT_SEED_ON_HOST_ENV = "UNSLOTH_DIFFUSION_PREQUANT_SEED_ON_HOST"+++def prequant_seed_device(+ plan: Any,+ device: str,+ scheme: Optional[str] = None,+) -> str:+ """Where a seeded pre-quantized denoiser is materialised: ``device`` when ``plan`` keeps it resident, else "cpu"+ for the schemes measured under offload. Loading it onto the GPU first adds a whole-denoiser spike that left the+ streaming hooks no room for their first block (Qwen-Image-2.1 int8 on 8 GB)."""+ if str(os.environ.get(PREQUANT_SEED_ON_HOST_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return device+ if plan is None or plan_keeps_transformer_resident(plan):+ return device+ if str(scheme) not in _TORCHAO_GROUP_OFFLOAD_MIN:+ return device+ return "cpu"++# Oldest torchao measured bit-exact under streamed group offload; 0.17 int8 (v1, no copy stream) ran 14x slower._TORCHAO_GROUP_OFFLOAD_MIN = {"int8": (0, 18), "fp8": (0, 17)}+# torchao 0.17 already ships the pinnable Int8Tensor (hosted int8 checkpoints) and its v1 int8 weights get the pin+# ops (install_torchao_v1_int8_pin_ops): without this a fresh install (torchao 0.17) got fp8 whenever it offloaded.+INT8_STREAM_TORCHAO17_ENV = "UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17"+_TORCHAO17_INT8_STREAM_MIN = (0, 17)+++def _int8_tensor_pinnable() -> bool:+ """Whether the installed torchao's ``Int8Tensor`` implements the pin ops the stream needs (0.17+)."""+ try:+ from torchao.quantization.quantize_.workflows.int8 import int8_tensor+ import torch++ table = getattr(int8_tensor.Int8Tensor, "_ATEN_OP_TABLE", None)+ if isinstance(table, dict):+ ops = set()+ for value in table.values():+ ops.update(value.keys() if isinstance(value, dict) else ())+ if ops:+ return (+ torch.ops.aten._pin_memory.default in ops+ and torch.ops.aten.is_pinned.default in ops+ )+ return True # no readable op table: the class exists, which is what 0.17 added+ except Exception: # noqa: BLE001 - no Int8Tensor: keep the 0.18 floor+ return False+++def _int8_stream_floor() -> tuple:+ if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]+ if not _int8_tensor_pinnable():+ return _TORCHAO_GROUP_OFFLOAD_MIN["int8"]+ return _TORCHAO17_INT8_STREAM_MIN++_TORCHAO_STREAM_SAFE_CLASSES = frozenset(("Int8Tensor", "Float8Tensor"))# Before 0.38 diffusers moved only the torchao wrapper, leaving quantised data on the host._DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN = (0, 38)@@ -1147,6 +1217,8 @@ def torchao_scheme_streams(scheme: Optional[str], *, torchao_version: Any = _UNSfloor = _TORCHAO_GROUP_OFFLOAD_MIN.get(str(scheme))if floor is None:return False+ if str(scheme) == "int8":+ floor = _int8_stream_floor()diffusers_version = _installed_diffusers_version()if diffusers_version is None or diffusers_version < _DIFFUSERS_TORCHAO_GROUP_OFFLOAD_MIN:return False@@ -1192,7 +1264,9 @@ def _torchao_group_offload_kwargs(install_group_offload_torchao_swap_retry()if not kwargs.get("use_stream"):return kwargs- if classes <= _TORCHAO_STREAM_SAFE_CLASSES:+ if classes <= _TORCHAO_STREAM_SAFE_CLASSES or (+ classes <= _TORCHAO_V1_INT8_CLASSES and install_torchao_v1_int8_pin_ops()+ ):if not kwargs.get("low_cpu_mem_usage"):return kwargs# Same override the planner's _torchao_stream_pinnable honoured, so the two cannot disagree.@@ -1219,6 +1293,107 @@ def _torchao_group_offload_kwargs(return safe+# torchao <= 0.17's default int8 weight (LinearActivationQuantizedTensor over an AffineQuantizedTensor).+_TORCHAO_V1_INT8_CLASSES = frozenset(("LinearActivationQuantizedTensor", "AffineQuantizedTensor"))+_V1_INT8_PIN_OPS_INSTALLED = False+++def install_torchao_v1_int8_pin_ops() -> bool:+ """Register ``is_pinned`` / ``pin_memory`` on torchao's v1 int8 classes so group offload streams them (else+ synchronous copies, 14x slower). Re-wraps the same int8 data, scale and zero point, so outputs stay bit-identical.+ Only where torchao ships the classes without the ops (<= 0.17)."""+ global _V1_INT8_PIN_OPS_INSTALLED+ if _V1_INT8_PIN_OPS_INSTALLED:+ return True+ if str(os.environ.get(INT8_STREAM_TORCHAO17_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return False+ try:+ import torch+ from torchao.dtypes.affine_quantized_tensor import AffineQuantizedTensor as AQT+ from torchao.quantization.linear_activation_quantized_tensor import (+ LinearActivationQuantizedTensor as LAQT,+ )++ aten = torch.ops.aten+ pin_ops = [aten._pin_memory.default, aten.pin_memory.default]+ for cls in (LAQT, AQT):+ table = getattr(cls, "_ATEN_OP_TABLE", {}).get(cls, {})+ if aten.is_pinned.default in table or aten._pin_memory.default in table:+ return False # this torchao implements them itself: never override its dispatch++ def _payload(t: Any) -> list:+ if isinstance(t, LAQT):+ return _payload(t.original_weight_tensor)+ if isinstance(t, AQT):+ impl = t.tensor_impl+ return [+ x+ for x in (getattr(impl, n, None) for n in ("int_data", "scale", "zero_point"))+ if x is not None+ ]+ return [t]++ def _pinned(t: Any) -> Any:+ if isinstance(t, LAQT):+ return t._apply_fn_to_data(_pinned)+ if isinstance(t, AQT):+ return t._apply_fn_to_data(+ lambda impl: impl._apply_fn_to_data(lambda x: x.pin_memory())+ )+ return t.pin_memory()++ # diffusers restores / record_streams torchao weights via ``tensor_data_names``, which v1 lacks: the payload stayed+ # on the GPU after offload.+ from diffusers.hooks import group_offloading as go++ restore = go._restore_torchao_tensor+ if not getattr(restore, "_unsloth_v1_int8", False):++ def _restore_v1(param: Any, source: Any) -> None:+ if isinstance(source, LAQT):+ param.original_weight_tensor = source.original_weight_tensor+ elif isinstance(source, AQT):+ param.tensor_impl = source.tensor_impl+ else:+ restore(param, source)++ _restore_v1._unsloth_v1_int8 = True+ go._restore_torchao_tensor = _restore_v1++ record = go._record_stream_torchao_tensor+ if not getattr(record, "_unsloth_v1_int8", False):++ def _record_v1(param: Any, stream: Any) -> None:+ if isinstance(param, (LAQT, AQT)):+ for x in _payload(param):+ x.record_stream(stream)+ else:+ record(param, stream)++ _record_v1._unsloth_v1_int8 = True+ go._record_stream_torchao_tensor = _record_v1++ for cls in (LAQT, AQT):++ @cls.implements([aten.is_pinned.default])+ def _is_pinned(func: Any, types: Any, args: Any, kwargs: Any) -> bool:+ return all(bool(x.is_pinned()) for x in _payload(args[0]))++ @cls.implements(pin_ops)+ def _pin(func: Any, types: Any, args: Any, kwargs: Any) -> Any:+ return _pinned(args[0])++ _V1_INT8_PIN_OPS_INSTALLED = True+ return True+ except Exception: # noqa: BLE001 - no v1 classes / no dispatch table: keep the synchronous fallback+ return False++def _freeze_torchao_weights(module: Any) -> None:try:for param in module.parameters():@@ -1623,6 +1798,96 @@ def largest_streamable_companion_mib(pipe: Any) -> Optional[int]:return max(sizes) if sizes else None+BALANCED_FIT_CHECK_ENV = "UNSLOTH_DIFFUSION_BALANCED_FIT_CHECK"+++def _balanced_fit_check_enabled() -> bool:+ return str(os.environ.get(BALANCED_FIT_CHECK_ENV, "")).strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def refine_balanced_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:+ """Fit-check an explicit ``memory_mode=balanced`` plan against the LOADED companions.++ Balanced never checked the budget, so Qwen-Image-2.1 int8 kept its 9 GiB fp8 encoder resident and OOMed at 16 /+ 12 / 8 GB. Judged after the load because the pre-load estimate can be the dense encoder size (16.7 vs 9.0 GB).+ Walks only the streamed tiers: encoders streamed too, else whole-module offload (granular streaming for torchao+ denoisers, which cannot take per-forward ``Module.to()``)."""+ if not _balanced_fit_check_enabled():+ return plan+ if getattr(plan, "requested_mode", None) != MEMORY_MODE_BALANCED:+ return plan+ if plan.offload_policy != OFFLOAD_GROUP or not bool(getattr(plan, "stream_transformer", True)):+ return plan+ estimates = dict(plan.estimates)+ budget = estimates.get("safe_device_budget_mib")+ if budget is None:+ return plan+ try:+ import torch++ components = getattr(pipe, "components", {})+ if not isinstance(components, dict):+ return plan+ streamed = _streamable_components(pipe, torch)+ mib = 1024 * 1024+ resident = 0+ encoders = 0+ for name, component in components.items():+ if not isinstance(component, torch.nn.Module):+ continue+ if name in streamed and streamed[name][1] == "block_level":+ continue # the denoisers already stream under group offload+ size = (_module_storage_bytes(component, set()) + mib - 1) // mib+ resident += size+ if name in streamed:+ encoders += size+ overhead = int(estimates.get("runtime_headroom_mib") or 0) + int(+ estimates.get("base_overhead_mib") or 0+ )+ except Exception: # noqa: BLE001 - a sizing aid; keep the plan the user asked for+ return plan++ estimates["balanced_resident_companions_mib"] = resident+ if resident + overhead <= int(budget):+ return plan+ if encoders > 0 and bool(getattr(plan, "stream_text_encoders", False)) is False:+ if (resident - encoders) + overhead <= int(budget):+ return replace(+ plan,+ stream_text_encoders = True,+ estimates = estimates,+ reasons = plan.reasons+ + (+ f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget "+ "beside the runtime headroom; streaming the text encoders too (they run once, before step 0)",+ ),+ )+ torchao = _pipe_denoisers_hold_torchao(pipe)+ return replace(+ plan,+ offload_policy = OFFLOAD_STREAMING if torchao else OFFLOAD_MODEL,+ stream_text_encoders = False,+ vae_tiling = True,+ vae_slicing = True,+ estimates = estimates,+ reasons = plan.reasons+ + (+ f"balanced: the loaded companions ({resident} MiB) do not fit the {int(budget)} MiB budget even with "+ "the text encoders streamed; "+ + (+ "streaming transformer blocks and text-encoder layers"+ if torchao+ else "whole-module offload of every component"+ ),+ ),+ )++def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan:"""Replace whole-module offload when a loaded component cannot fit on the device.@@ -1688,6 +1953,379 @@ def refine_memory_plan_for_components(pipe: Any, plan: MemoryPlan) -> MemoryPlan)+# The flat planner reserves 8192 MiB/MP of activations: Qwen-Image-2.1 (16525 MiB of weights, 1849 MiB measured peak)+# streamed its encoder on 24 GB.+MEASURED_ACTIVATION_ENV = "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION"+PARTIAL_RESIDENT_ENV = "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT"++# Worst measured CUDA MiB above the resident weights, one 1024x1024 image, encoder + every step + VAE decode, torchao+# int8 / fp8 denoisers on the compiled tiers (Qwen-Image-2.1: encoder 1849 streamed, denoise 1442, decode 1730).+_MEASURED_IMAGE_PEAK_MIB: dict[str, int] = {"qwen-image-2.1": 1849}+_MEASURED_PEAK_SPEED_MODES = ("default", "max")+_MEASURED_PEAK_MARGIN = 1.15+_MEASURED_PEAK_ROUND_MIB = 256+++def _env_off(name: str) -> bool:+ return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")+++def measured_image_runtime_mib(+ family: Optional[str],+ speed_mode: Optional[str],+ *,+ width: Optional[int] = None,+ height: Optional[int] = None,+ batch_size: int = 1,+) -> Optional[int]:+ """Measured runtime headroom for ``family`` at this size, or None (unmeasured: use the flat estimate)."""+ if _env_off(MEASURED_ACTIVATION_ENV):+ return None+ peak = _MEASURED_IMAGE_PEAK_MIB.get(str(family or "").lower())+ if peak is None or str(speed_mode or "") not in _MEASURED_PEAK_SPEED_MODES:+ return None+ w = max(64, int(width or DEFAULT_IMAGE_WIDTH))+ h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))+ scale = max(+ 1.0,+ (w * h * max(1, int(batch_size or 1))) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT),+ )+ need = peak * scale * _MEASURED_PEAK_MARGIN+ step = _MEASURED_PEAK_ROUND_MIB+ return int(-(-need // step) * step)+++def _loaded_component_mib(pipe: Any) -> Optional[dict[str, tuple[int, str]]]:+ """name -> (MiB, role in 'dit' / 'text_encoder' / 'other') from storage bytes (a torchao int8 weight counts its+ int8 data + scales); None when unreadable."""+ try:+ import torch++ out: dict[str, tuple[int, str]] = {}+ for name, module in (getattr(pipe, "components", {}) or {}).items():+ if not isinstance(module, torch.nn.Module):+ continue+ seen: set[int] = set()+ nbytes = 0+ for tensor in (*module.parameters(), *module.buffers()):+ if id(tensor) in seen:+ continue+ seen.add(id(tensor))+ nbytes += sum(_storage_nbytes(tensor))+ name = str(name)+ role = (+ "dit"+ if name in ("transformer", "transformer_2", "unconditional_transformer")+ else "text_encoder"+ if name.startswith("text_encoder")+ else "other"+ )+ out[name] = (-(-nbytes // (1024 * 1024)), role)+ return out+ except Exception: # noqa: BLE001 - an optional refinement+ return None+++def refine_plan_from_loaded_weights(+ pipe: Any,+ plan: MemoryPlan,+ *,+ family: Optional[str],+ speed_mode: Optional[str],+ logger: Any = None,+) -> MemoryPlan:+ """Re-place a streamed ``auto`` load from its LOADED weights and the family's measured activation peak.++ Keeps the flat plan's offload hooks and makes whole groups resident (denoiser first, then encoders) within the+ safe budget left after the measured peak x margin and the base overhead. No-op for explicit modes, non-CUDA / unified memory, unmeasured families+ or speed tiers, non-torchao denoisers and ``model`` plans."""+ try:+ if getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO:+ return plan+ policy = plan.offload_policy+ if policy not in (OFFLOAD_GROUP, OFFLOAD_STREAMING):+ return plan+ memory = plan.device_memory+ if memory.is_unified or getattr(memory, "device", None) != "cuda":+ return plan+ if not _pipe_denoisers_hold_torchao(pipe):+ return plan+ headroom = measured_image_runtime_mib(family, speed_mode)+ if headroom is None:+ return plan+ budget = plan.estimates.get("safe_device_budget_mib")+ if budget is None:+ return plan+ budget = int(budget)+ sizes = _loaded_component_mib(pipe)+ if not sizes:+ return plan+ dit = sum(m for m, r in sizes.values() if r == "dit")+ encoders = sum(m for m, r in sizes.values() if r == "text_encoder")+ other = sum(m for m, r in sizes.values() if r == "other")+ if dit <= 0:+ return plan+ overhead = int(plan.estimates.get("base_overhead_mib") or DEFAULT_BASE_OVERHEAD_MIB)+ floor = headroom + overhead + other+ estimates = dict(plan.estimates)+ estimates.update(+ measured_runtime_headroom_mib = headroom,+ loaded_transformer_mib = dit,+ loaded_text_encoder_mib = encoders,+ loaded_other_mib = other,+ )+ if _env_off(PARTIAL_RESIDENT_ENV):+ return plan+ # The flat plan's hooks stay installed and whole groups are kept resident within the room, so an oversized+ # request can stream them again (release_resident_groups) and run exactly as the flat plan would.+ stream_te = bool(getattr(plan, "stream_text_encoders", False))+ room = budget - floor+ if policy == OFFLOAD_GROUP and not stream_te:+ room -= encoders # resident companions+ if not bool(getattr(plan, "stream_transformer", True)):+ room -= dit+ dit_room = 0+ else:+ dit_room = min(max(room, 0), dit)+ te_room = max(0, room - dit_room) if stream_te and policy == OFFLOAD_GROUP else 0+ if dit_room <= 0 and te_room <= 0:+ return plan+ new = replace(+ plan,+ resident_transformer_mib = int(dit_room) if dit_room > 0 else None,+ resident_text_encoder_mib = int(te_room) if te_room > 0 else None,+ estimates = {+ **estimates,+ "resident_transformer_mib": int(dit_room),+ "resident_text_encoder_mib": int(te_room),+ },+ reasons = plan.reasons+ + (+ f"{int(dit_room)} MiB of the {dit} MiB transformer and {int(te_room)} MiB of the {encoders} MiB "+ f"encoders stay resident (measured peak {headroom} MiB); the rest streams",+ ),+ )+ if logger is not None:+ logger.info(+ "diffusion.memory: measured-activation placement %s -> %s (%s)",+ policy,+ new.offload_policy,+ new.reasons[-1],+ )+ return new+ except Exception as exc: # noqa: BLE001 - keep the flat plan+ if logger is not None:+ logger.debug("diffusion.memory: measured-activation placement skipped (%s)", exc)+ return plan+++def _keep_groups_resident(+ module: Any,+ room_mib: int,+ device: Any,+ logger: Any = None,+) -> int:+ """Make whole offload groups of a streamed ``module`` resident within ``room_mib`` (top-level group first, then+ blocks in order); their onload / offload become no-ops and the rest keeps streaming. Returns the MiB kept."""+ if room_mib is None or int(room_mib) <= 0 or _env_off(PARTIAL_RESIDENT_ENV):+ return 0+ try:+ import torch+ from diffusers.hooks import group_offloading as go++ groups = _offload_groups(module)+ if not groups:+ return 0+ top = [g for g in groups if getattr(g, "offload_leader", None) is module]+ ordered = top + [g for g in groups if getattr(g, "offload_leader", None) is not module]+ is_torchao = getattr(go, "_is_torchao_tensor", lambda t: False)+ onload = torch.device(device)+ left = int(room_mib) * 1024 * 1024+ kept = 0++ def _tensors(group: Any) -> list:+ out: list = []+ seen: set[int] = set()+ for t in (+ [p for m in group.modules for p in m.parameters()]+ + [b for m in group.modules for b in m.buffers()]+ + list(group.parameters or [])+ + list(group.buffers or [])+ ):+ if id(t) not in seen:+ seen.add(id(t))+ out.append(t)+ return out++ disable = getattr(getattr(torch, "compiler", None), "disable", None)++ def _noop(*args: Any, **kwargs: Any) -> None:+ return None++ def _resident_onload(stream: Any) -> Callable[[], None]:+ # a prefetching predecessor skips its own copy-stream wait and relies on this onload_ to do it+ def onload_(*args: Any, **kwargs: Any) -> None:+ if stream is not None:+ stream.synchronize()++ return disable(onload_) if callable(disable) else onload_++ noop = disable(_noop) if callable(disable) else _noop++ try:+ module._unsloth_resident_room = int(room_mib)+ module._unsloth_resident_device = onload+ except AttributeError:+ pass+ for group in ordered:+ if getattr(group, "offload_to_disk_path", None):+ break+ tensors = _tensors(group)+ need = sum(sum(_storage_nbytes(t)) for t in tensors)+ if getattr(group, "_unsloth_resident", False):+ left -= need+ continue+ if need > left:+ # too large: it streams; a later, smaller group may still fit+ continue+ cpu = getattr(group, "cpu_param_dict", None) or {}+ for t in tensors:+ if t.device.type == onload.type:+ continue+ moved = cpu.get(t, t).to(onload)+ if is_torchao(t):+ go._swap_torchao_tensor(t, moved)+ else:+ t.data = moved+ # host copies stay: release_resident_groups streams the group again for an oversized request+ group._unsloth_streamed_hooks = (+ group.__dict__.get("onload_"),+ group.__dict__.get("offload_"),+ )+ group.onload_ = _resident_onload(getattr(group, "stream", None))+ group.offload_ = noop+ group._unsloth_resident = True+ group._unsloth_resident_bytes = need+ left -= need+ kept += need+ if kept and onload.type == "cuda":+ torch.cuda.synchronize(onload)+ if logger is not None and kept:+ logger.info(+ "diffusion.memory: %s keeps %d MiB resident (%d of %d offload groups); the rest streams",+ type(module).__name__,+ kept >> 20,+ sum(1 for g in ordered if getattr(g, "_unsloth_resident", False)),+ len(ordered),+ )+ return kept >> 20+ except Exception as exc: # noqa: BLE001 - streaming every group is the safe state+ if logger is not None:+ logger.warning("diffusion.memory: partial residency skipped (%s)", exc)+ return 0+++def _release_group(group: Any) -> None:+ onload_, offload_ = getattr(group, "_unsloth_streamed_hooks", (None, None))+ for name, fn in (("onload_", onload_), ("offload_", offload_)):+ if fn is not None:+ setattr(group, name, fn)+ else:+ group.__dict__.pop(name, None)+ group._unsloth_resident = False+ group.offload_()+++def release_resident_groups(+ pipe: Any,+ need_mib: int,+ logger: Any = None,+) -> Optional[Callable[[], None]]:+ """Stream resident offload groups again until ``need_mib`` is freed (text encoders first, then the denoiser's+ blocks from the last); returns a callable restoring them, or None when nothing was resident. A request larger+ than the measured placement reserved (reference images, a bigger canvas, a batch) then runs as the flat plan."""+ try:+ import torch++ modules = [+ m+ for m in (getattr(pipe, "components", {}) or {}).values()+ if getattr(m, "_unsloth_resident_room", None)+ ]+ if not modules or int(need_mib) <= 0:+ return None+ modules.sort(key = lambda m: 0 if _is_text_encoder_module(pipe, m) else 1)+ left = int(need_mib) * 1024 * 1024+ released: list = []+ for module in modules:+ for group in reversed(_offload_groups(module) or []):+ if left <= 0:+ break+ if not getattr(group, "_unsloth_resident", False):+ continue+ _release_group(group)+ left -= int(getattr(group, "_unsloth_resident_bytes", 0))+ if module not in released:+ released.append(module)+ if not released:+ return None+ device = getattr(released[0], "_unsloth_resident_device", None)+ if device is not None and torch.device(device).type == "cuda":+ torch.cuda.synchronize(device)+ if logger is not None:+ logger.info(+ "diffusion.memory: streaming %d MiB of resident groups for an oversized request",+ (int(need_mib) * 1024 * 1024 - max(left, 0)) >> 20,+ )++ def restore() -> None:+ for module in released:+ _keep_groups_resident(+ module,+ module._unsloth_resident_room,+ module._unsloth_resident_device,+ logger,+ )++ return restore+ except Exception as exc: # noqa: BLE001 - the guard still refuses what cannot fit+ if logger is not None:+ logger.warning("diffusion.memory: releasing resident groups failed (%s)", exc)+ return None+++def _is_text_encoder_module(pipe: Any, module: Any) -> bool:+ for name, component in (getattr(pipe, "components", {}) or {}).items():+ if component is module:+ return str(name).startswith("text_encoder")+ return False+++def measured_request_extra_mib(+ pipe: Any,+ *,+ width: Optional[int],+ height: Optional[int],+ batch_size: int = 1,+ condition_pixels: int = 0,+) -> int:+ """MiB a request needs beyond what the measured placement reserved (0 when it fits or none was made)."""+ reserve = getattr(pipe, "_unsloth_measured_reserve", None)+ if not reserve:+ return 0+ headroom, family, speed_mode = reserve+ need = measured_image_runtime_mib(+ family, speed_mode, width = width, height = height, batch_size = batch_size+ )+ if need is None:+ return 0+ cond = max(0, int(condition_pixels or 0)) * max(1, int(batch_size or 1))+ need += int(+ 8192 * cond / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT) * _MEASURED_PEAK_MARGIN+ )+ return max(0, int(need) - int(headroom))++def apply_memory_plan(pipe: Any,plan: MemoryPlan,@@ -1736,6 +2374,12 @@ def apply_memory_plan(}if not bool(getattr(plan, "stream_transformer", True)):group_kwargs["stream_transformer"] = False+ resident_mib = getattr(plan, "resident_transformer_mib", None)+ if resident_mib:+ group_kwargs["resident_transformer_mib"] = int(resident_mib)+ resident_te_mib = getattr(plan, "resident_text_encoder_mib", None)+ if resident_te_mib and group_kwargs["stream_text_encoders"]:+ group_kwargs["resident_text_encoder_mib"] = int(resident_te_mib)if not _apply_group_offload(pipe, placement, logger, **group_kwargs):if "stream_transformer" in group_kwargs and _pipe_denoisers_hold_torchao(pipe):raise RuntimeError(@@ -1751,7 +2395,13 @@ def apply_memory_plan(_fallback_to_model_offload()policy = OFFLOAD_MODELelif policy == OFFLOAD_STREAMING:- _apply_streaming_offload(pipe, placement, logger)+ resident_mib = getattr(plan, "resident_transformer_mib", None)+ if resident_mib:+ _apply_streaming_offload(+ pipe, placement, logger, resident_transformer_mib = int(resident_mib)+ )+ else:+ _apply_streaming_offload(pipe, placement, logger)elif policy == OFFLOAD_SEQUENTIAL:try:pipe.enable_sequential_cpu_offload(device = placement)@@ -2248,6 +2898,118 @@ def install_group_offload_buffer_restore() -> bool:return True+PIN_TOP_GROUP_ENV = "UNSLOTH_DIFFUSION_PIN_TOP_GROUP"+++def _pin_top_level_group(+ module: Any,+ logger: Any = None,+ pinned_mib: Optional[list] = None,+) -> bool:+ """Onload a block-streamed DiT's top-level group (embedders, norm_out, proj_out) from one pinned host copy.++ diffusers gives that group no stream: every forward uploaded it from pageable memory and every offload copied it+ back to a fresh host buffer, which inference never needs. Now the onload is an async H2D on the compute stream and+ the offload only re-points the tensors; VRAM unchanged. Skipped for torchao weights (not re-pointable via+ ``.data``) and when the copy does not fit the pinnable host RAM."""+ if (os.environ.get(PIN_TOP_GROUP_ENV) or "").strip().lower() in ("0", "off", "false", "no"):+ return False+ try:+ import torch+ from diffusers.hooks import group_offloading as go++ registry = getattr(module, "_diffusers_hook", None)+ get_hook = getattr(registry, "get_hook", None)+ hook = (+ get_hook(getattr(go, "_GROUP_OFFLOADING", "group_offloading"))+ if callable(get_hook)+ else None+ )+ group = getattr(hook, "group", None)+ if (+ group is None+ or getattr(group, "stream", None) is not None+ or getattr(group, "offload_to_disk_path", None)+ or getattr(getattr(group, "onload_device", None), "type", None) != "cuda"+ or getattr(group, "_unsloth_pinned_top", False)+ ):+ return False+ tensors: list = []+ seen: set = set()+ for tensor in (+ [p for m in group.modules for p in m.parameters()]+ + [b for m in group.modules for b in m.buffers()]+ + list(group.parameters or [])+ + list(group.buffers or [])+ ):+ if id(tensor) not in seen:+ seen.add(id(tensor))+ tensors.append(tensor)+ is_torchao = getattr(go, "_is_torchao_tensor", None)+ if not tensors or (callable(is_torchao) and any(is_torchao(t) for t in tensors)):+ return False+ if any(type(t) not in (torch.Tensor, torch.nn.Parameter) for t in tensors):+ return False+ # the user's "pin nothing" override wins, as on every other streamed path+ if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return False+ # per tensor rounded to a power of two, like torch's pinned allocator (and _module_host_mib)+ need_mib = sum(+ 1 << (int(t.numel()) * int(t.element_size()) - 1).bit_length()+ for t in tensors+ if int(t.numel()) * int(t.element_size()) > 0+ ) // (1024 * 1024)+ budget = None if _pinned_memory_capped() else _pin_budget_mib()+ # on the running total the encoders and torchao denoisers already pinned (or will, deferred) count against+ already = pinned_mib[0] if pinned_mib else 0+ if budget is None or already + need_mib > budget:+ return False+ host = {+ t: (t.data if t.data.device.type == "cpu" else t.data.cpu()).pin_memory()+ for t in tensors+ }+ device = group.onload_device++ def onload_() -> None:+ for tensor, pinned in list(host.items()):+ current = tensor.data+ if current.device.type == "cpu" and current.data_ptr() != pinned.data_ptr():+ # replaced while offloaded (a .to() conversion, an adapter fused on the host): re-pin what is there now+ pinned = current if current.is_pinned() else current.pin_memory()+ host[tensor] = pinned+ tensor.data = pinned.to(device, non_blocking = True)++ def offload_() -> None:+ for tensor, pinned in host.items():+ tensor.data = pinned++ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ if callable(disable):+ onload_, offload_ = disable(onload_), disable(offload_)+ offload_()+ group.onload_ = onload_+ group.offload_ = offload_+ group._unsloth_pinned_top = True+ if pinned_mib is not None:+ pinned_mib[0] = already + need_mib+ if logger is not None:+ logger.info(+ "diffusion.memory: %s top-level weights (%d MiB) onload from a pinned copy, no copy back",+ type(module).__name__,+ need_mib,+ )+ return True+ except Exception as exc: # noqa: BLE001 - diffusers keeps its own (slower) path+ if logger is not None:+ logger.debug("diffusion.memory: top-level group left as diffusers built it (%s)", exc)+ return False++def _apply_group_offload(pipe: Any,device: str,@@ -2256,6 +3018,8 @@ def _apply_group_offload(stream_text_encoders: bool = False,stream_transformer: bool = True,background_pin: Optional[bool] = None,+ resident_transformer_mib: Optional[int] = None,+ resident_text_encoder_mib: Optional[int] = None,) -> bool:"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping thesmaller components resident. Returns False (caller falls back to whole-module) on any failure.@@ -2369,6 +3133,12 @@ def _apply_group_offload(module, **_torchao_group_offload_kwargs(module, gkwargs, pinned_mib))installed += 1+ if use_stream:+ _pin_top_level_group(module, logger, pinned_mib)+ if resident_transformer_mib:+ room = int(resident_transformer_mib)+ for module in streamed.values():+ room -= _keep_groups_resident(module, room, onload, logger)# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no# recognisable block list can refuse), and this tier is a rescue: the alternative to streaming an encoder is@@ -2385,6 +3155,7 @@ def _apply_group_offload(if "low_cpu_mem_usage" in gkwargs:ekwargs["low_cpu_mem_usage"] = not pin_streamed[1]transformer_demoted = False+ te_room = int(resident_text_encoder_mib or 0)for name, module in streamed_encoders.items():try:if defer and not ekwargs.get("low_cpu_mem_usage", False):@@ -2394,6 +3165,8 @@ def _apply_group_offload(apply_group_offloading(module, **ekwargs)installed += 1_pin_vision_embedding_device(module)+ if te_room > 0:+ te_room -= _keep_groups_resident(module, te_room, onload, logger)except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the loadif not stream_transformer and installed == 0:# Resident encoder here would OOM; fall back to model offload, which rejects partial hooks.@@ -2888,7 +3661,26 @@ def raise_on_image_activation_shortfall(return verdict-def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:+STREAMING_PREFETCH_ENV = "UNSLOTH_DIFFUSION_STREAMING_PREFETCH"+++def _streaming_prefetch_enabled() -> bool:+ """Kill switch for the streaming tier's overlapped onload (pinned host copies + record_stream); default on."""+ return (os.environ.get(STREAMING_PREFETCH_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _apply_streaming_offload(+ pipe: Any,+ device: str,+ logger: Any,+ *,+ resident_transformer_mib: Optional[int] = None,+) -> None:"""Stream transformer blocks and text-encoder leaves without whole-component onloads.This is selected only after measuring a component larger than the safe device budget, so a@@ -2927,6 +3719,28 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:component.to(onload)pinned_mib = [0]+ # Overlap needs a pinned host copy and record_stream (record_stream=False syncs the compute stream per group).+ prefetch = use_stream and _streaming_prefetch_enabled()+ pin_dits, pin_encoders = False, False+ if prefetch:+ pin_dits, pin_encoders = _streamed_pin_plan(+ sum(_module_host_mib(m) for m, t in streamed.values() if t == "block_level"),+ sum(_module_host_mib(m) for m, t in streamed.values() if t != "block_level"),+ logger,+ )+ defer = (+ prefetch+ and bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))+ and _background_pin_enabled()+ )+ if defer:+ install_group_pin_wait()+ # pinned encoders count against the budget a torchao denoiser pins within, as on the group tier+ if pin_encoders:+ pinned_mib[0] = sum(+ _module_host_mib(m) for m, t in streamed.values() if t != "block_level"+ )+for module, offload_type in streamed.values():kwargs: dict[str, Any] = {"onload_device": onload,@@ -2940,15 +3754,24 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:if use_stream and "non_blocking" in params:kwargs["non_blocking"] = Trueif use_stream and "record_stream" in params:- kwargs["record_stream"] = False+ kwargs["record_stream"] = prefetch+ pin = pin_dits if offload_type == "block_level" else pin_encodersif use_stream and "low_cpu_mem_usage" in params:- kwargs["low_cpu_mem_usage"] = True- apply_group_offloading(- module, **_torchao_group_offload_kwargs(module, kwargs, pinned_mib)- )+ kwargs["low_cpu_mem_usage"] = not (pin and not defer)+ kwargs = _torchao_group_offload_kwargs(module, kwargs, pinned_mib)+ apply_group_offloading(module, **kwargs)+ if pin and defer and kwargs.get("use_stream") and kwargs.get("low_cpu_mem_usage"):+ _defer_pinning(pipe, module, onload, logger)installed += 1+ if use_stream and offload_type == "block_level":+ _pin_top_level_group(module, logger, pinned_mib)if offload_type == "leaf_level":_pin_vision_embedding_device(module)+ if resident_transformer_mib:+ room = int(resident_transformer_mib)+ for module, offload_type in streamed.values():+ if offload_type == "block_level":+ room -= _keep_groups_resident(module, room, onload, logger)except Exception as exc:if logger is not None:logger.warning(diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex bc18ff1d39f..508b6c8869a 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -1112,9 +1112,12 @@ def load_prequantized_transformer(component: Optional[str] = None,local_files_only: bool = False,logger: Any = None,+ placement_device: Optional[str] = None,) -> Optional[Any]:"""Load the pre-quantized transformer described by ``source`` onto ``device``.+ ``placement_device`` (default ``device``) is where the module is materialised; ``device`` selects kernels.+``cache_dir`` is the live Hub cache root, as every other loader call pins it: unset, a fetchlands under huggingface_hub's import-time constant, so a mid-session cache change re-downloadsinto a root Unsloth no longer reads.@@ -1228,7 +1231,7 @@ def load_prequantized_transformer(del state_dictdel ckpt- transformer = transformer.to(device)+ transformer = transformer.to(placement_device or device)if declares_rotation(metadata):try:import torch@@ -1237,7 +1240,12 @@ def load_prequantized_transformer(dtype = getattr(torch, str(metadata.get("torch_dtype") or "bfloat16"), torch.bfloat16)- warm_rotation_cache(transformer, on.device if on is not None else device, dtype)+ # a host-placed module still computes on ``device``+ warm_rotation_cache(+ transformer,+ on.device if on is not None and placement_device is None else device,+ dtype,+ )except Exception: # noqa: BLE001pass# Same small-M row padding the runtime quantise path applies, and for the same reason: a checkpoint built@@ -1275,7 +1283,7 @@ def load_prequantized_transformer("diffusion.prequant: loaded %s checkpoint (%s) onto %s",scheme,source.kind,- device,+ placement_device or device,)return transformerexcept Exception as exc: # noqa: BLE001 - fall back to the dense-quantise pathdiff --git a/studio/backend/core/inference/diffusion_qwenimage21.py b/studio/backend/core/inference/diffusion_qwenimage21.pyindex 77d300655a5..c9b7b690cc4 100644--- a/studio/backend/core/inference/diffusion_qwenimage21.py+++ b/studio/backend/core/inference/diffusion_qwenimage21.py@@ -21,6 +21,7 @@ import weakreffrom typing import Any, OptionalFAST_STEP_ENV = "UNSLOTH_DIFFUSION_Q21_FAST_STEP"+COMPACT_KV_ENV = "UNSLOTH_DIFFUSION_Q21_COMPACT_KV"_MODULE = "diffusers.models.transformers.transformer_qwenimage21"_CLASS = "QwenImage21Transformer2DModel"@@ -400,6 +401,7 @@ def _make_forward(mod: Any, stock: Any) -> Any:cache_write_slice = slice(0, prefix_len) if kv_cache_mode == "extract" else Noneblock_key_valid = joint_key_valid+ compact_kv = compact_kv_enabled()for index_block, block in enumerate(self.transformer_blocks):layer_cache = kv_cache.get_layer(index_block) if kv_cache is not None else Nonejoint_hidden_states = block(@@ -414,6 +416,8 @@ def _make_forward(mod: Any, stock: Any) -> Any:segments = block_segments,key_valid = block_key_valid,)+ if layer_cache is not None and cache_write_slice is not None and compact_kv:+ _compact_layer_cache(layer_cache)joint_hidden_states = self.norm_out(joint_hidden_states, temb, modulation_mask)output = self.proj_out(joint_hidden_states)@@ -429,6 +433,27 @@ def _make_forward(mod: Any, stock: Any) -> Any:return wrapped+def compact_kv_enabled() -> bool:+ return (os.environ.get(COMPACT_KV_ENV) or "").strip().lower() not in ("0", "off", "false", "no")+++def _compact_layer_cache(layer_cache: Any) -> None:+ """Copy a prefix K/V entry that is a view of a larger buffer. Under compile Inductor turns the extract step's+ ``key[:, :prefix].clone()`` into a view of the full text + image K/V, pinning ~2 GiB across 32 blocks at 1 MP for+ step 0. Bit-identical."""+ for name in ("k", "v"):+ tensor = getattr(layer_cache, name, None)+ try:+ if (+ tensor is None+ or tensor.untyped_storage().nbytes() <= tensor.numel() * tensor.element_size()+ ):+ continue+ setattr(layer_cache, name, tensor.clone())+ except Exception: # noqa: BLE001 - a memory saving only; keep the entry as stored+ continue++def install(logger: Any = None) -> bool:if fast_step_disabled():return Falsediff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.pyindex a4a86f1ced4..e9fe920365d 100644--- a/studio/backend/core/inference/image_gallery.py+++ b/studio/backend/core/inference/image_gallery.py@@ -55,6 +55,20 @@ def _params_text(meta: dict[str, Any]) -> str:return "\n".join(lines)+# Pillow's default zlib level 6 costs ~0.4 s per 1024x1024 PNG on the request path; level 1 ~0.1 s, ~10% larger, lossless.+PNG_COMPRESS_LEVEL_ENV = "UNSLOTH_IMAGE_PNG_COMPRESS_LEVEL"+_DEFAULT_PNG_COMPRESS_LEVEL = 1+++def png_compress_level() -> int:+ raw = os.environ.get(PNG_COMPRESS_LEVEL_ENV, "").strip()+ try:+ level = int(raw)+ except ValueError:+ return _DEFAULT_PNG_COMPRESS_LEVEL+ return level if 0 <= level <= 9 else _DEFAULT_PNG_COMPRESS_LEVEL++def _png_bytes(image: Any, meta: dict[str, Any]) -> bytes:import io@@ -64,7 +78,7 @@ def _png_bytes(image: Any, meta: dict[str, Any]) -> bytes:info.add_text(_META_KEY, json.dumps(meta))info.add_text("parameters", _params_text(meta))buf = io.BytesIO()- image.save(buf, format = "PNG", pnginfo = info)+ image.save(buf, format = "PNG", pnginfo = info, compress_level = png_compress_level())return buf.getvalue()diff --git a/studio/backend/tests/test_diffusion_int8_stream_torchao17.py b/studio/backend/tests/test_diffusion_int8_stream_torchao17.pynew file mode 100644index 00000000000..ddae12adf7e--- /dev/null+++ b/studio/backend/tests/test_diffusion_int8_stream_torchao17.py@@ -0,0 +1,267 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""INT8 under streamed offload on torchao 0.17 (what a fresh install resolves: torch < 2.12 -> torchao 0.17).++The planner used to require torchao 0.18 for int8 under any tier that streams the denoiser, so every offloading+install got the fp8 transformer. 0.17 already ships the pinnable ``Int8Tensor``; only the v1 int8 class it still+defaults to is unpinnable, and that is rebuilt before streaming. Version-simulated, CPU only."""++from __future__ import annotations++import types++import pytest++from core.inference import diffusion_memory as mem+from core.inference.diffusion_memory import (+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,+ OFFLOAD_STREAMING,+ MemoryPlan,+ torchao_offload_plan,+ torchao_scheme_streams,+ torchao_survives_plan,+)++ENV = "UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17"+AO17, AO18, AO16 = (0, 17), (0, 18), (0, 16)++[email protected](autouse = True)+def _host(monkeypatch):+ # hermetic: the installed torchao / diffusers of the test host never decide these tests+ monkeypatch.setattr(mem, "_installed_diffusers_version", lambda: (0, 41))+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)+ monkeypatch.delenv(ENV, raising = False)+++def _plan(policy, stream_transformer = True):+ # Qwen-Image-2.1 int8 at a 16 GB budget: denoiser 6922 MiB, companions 9603 MiB+ return MemoryPlan(+ requested_mode = "auto",+ offload_policy = policy,+ vae_tiling = False,+ vae_slicing = True,+ device_memory = None,+ estimates = {+ "safe_device_budget_mib": 13320,+ "model_dense_mib": 16525,+ "companion_dense_mib": 9603,+ "text_encoder_dense_mib": 8959,+ "runtime_headroom_mib": 8192,+ "base_overhead_mib": 2048,+ },+ reasons = (),+ stream_transformer = stream_transformer,+ )+++def test_int8_streams_on_torchao_017():+ assert torchao_scheme_streams("int8", torchao_version = AO17) is True++[email protected]("policy", [OFFLOAD_GROUP, OFFLOAD_STREAMING])+def test_int8_survives_a_streamed_plan_on_torchao_017(monkeypatch, policy):+ monkeypatch.setattr(mem, "_torchao_stream_pinnable", lambda *a, **k: True)+ plan = _plan(policy)+ assert torchao_survives_plan(plan, "int8", torchao_version = AO17) is True+ assert torchao_offload_plan(plan, "int8", torchao_version = AO17).offload_policy == policy+++def test_int8_whole_module_too_big_streams_on_torchao_017(monkeypatch):+ monkeypatch.setattr(mem, "_torchao_stream_pinnable", lambda *a, **k: True)+ plan = _plan(OFFLOAD_MODEL)+ placed = torchao_offload_plan(plan, "int8", torchao_version = AO17)+ assert placed is not None and placed.offload_policy == OFFLOAD_STREAMING+++def test_kill_switch_restores_the_018_floor(monkeypatch):+ monkeypatch.setenv(ENV, "0")+ assert torchao_scheme_streams("int8", torchao_version = AO17) is False+ assert torchao_scheme_streams("int8", torchao_version = AO18) is True+++def test_torchao_without_a_pinnable_int8_tensor_keeps_the_018_floor(monkeypatch):+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: False)+ assert torchao_scheme_streams("int8", torchao_version = AO17) is False++[email protected]("version", [AO16, (0, 14)])+def test_older_torchao_still_declines_int8_streaming(version):+ assert torchao_scheme_streams("int8", torchao_version = version) is False+++def test_fp8_and_other_schemes_unchanged():+ assert torchao_scheme_streams("fp8", torchao_version = AO17) is True+ assert torchao_scheme_streams("fp8", torchao_version = AO16) is False+ assert torchao_scheme_streams("nvfp4", torchao_version = AO18) is False+++# ---- apply time: a v1 int8 module (0.17's default int8 class) keeps the copy stream through the pin-op shim+++class _V1Weight:+ pass+++_V1Weight.__module__ = "torchao.quantization.linear_activation_quantized_tensor"+_V1Weight.__name__ = "LinearActivationQuantizedTensor"+++class _Int8Weight:+ pass+++_Int8Weight.__module__ = "torchao.quantization"+_Int8Weight.__name__ = "Int8Tensor"+++class _Module:+ def __init__(self, cls):+ self.weights = [cls()]++ def parameters(self):+ return iter(self.weights)+++STREAM_KWARGS = {+ "use_stream": True,+ "non_blocking": True,+ "record_stream": True,+ "low_cpu_mem_usage": False,+}+++def _apply_env(monkeypatch):+ monkeypatch.setattr(mem, "install_group_offload_torchao_swap_retry", lambda: True)+ monkeypatch.setattr(mem, "_freeze_torchao_weights", lambda module: None)+++def test_v1_int8_keeps_the_copy_stream_once_its_pin_ops_exist(monkeypatch, _apply_env):+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: True, raising = False)+ out = mem._torchao_group_offload_kwargs(_Module(_V1Weight), dict(STREAM_KWARGS), [0])+ assert (+ out["use_stream"] is True+ and out.get("record_stream") is True+ and out.get("non_blocking") is True+ )+++def test_v1_int8_without_its_pin_ops_falls_back_to_synchronous(monkeypatch, _apply_env):+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: False, raising = False)+ out = mem._torchao_group_offload_kwargs(_Module(_V1Weight), dict(STREAM_KWARGS), [0])+ assert out["use_stream"] is False+++def test_pin_op_shim_honours_the_kill_switch(monkeypatch):+ monkeypatch.setenv(ENV, "0")+ monkeypatch.setattr(mem, "_V1_INT8_PIN_OPS_INSTALLED", False, raising = False)+ assert mem.install_torchao_v1_int8_pin_ops() is False+++def test_int8_tensor_module_keeps_the_stream(monkeypatch, _apply_env):+ out = mem._torchao_group_offload_kwargs(_Module(_Int8Weight), dict(STREAM_KWARGS), [0])+ assert out["use_stream"] is True+++def _v1_int8_stack():+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA (pinned host memory)")+ try:+ from torchao.quantization.linear_activation_quantized_tensor import ( # noqa: F401+ LinearActivationQuantizedTensor,+ )+ except Exception: # noqa: BLE001+ pytest.skip("this torchao has no v1 int8 classes (0.18+)")+ return torch+++def test_v1_int8_streams_bit_identically_with_the_shim():+ """Real torchao <= 0.17 + CUDA: the v1 weights pin, stream on the copy stream, and match resident exactly."""+ import copy++ torch = _v1_int8_stack()+ from diffusers.hooks import apply_group_offloading+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ assert mem.install_torchao_v1_int8_pin_ops() is True+ torch.manual_seed(0)+ blocks = torch.nn.Sequential(+ *[+ torch.nn.Sequential(+ torch.nn.Linear(256, 512), torch.nn.GELU(), torch.nn.Linear(512, 256)+ )+ for _ in range(3)+ ]+ ).to(torch.bfloat16)+ offloaded = copy.deepcopy(blocks)+ # set_inductor_config = False as Studio builds it: the bare config sets float32 matmul precision process-wide+ quantize_(offloaded, Int8DynamicActivationInt8WeightConfig(set_inductor_config = False))+ offloaded.requires_grad_(False)+ assert type(next(offloaded.parameters())).__name__ == "LinearActivationQuantizedTensor"+ resident = copy.deepcopy(offloaded).cuda()+ kwargs = mem._torchao_group_offload_kwargs(+ offloaded,+ {+ "onload_device": torch.device("cuda"),+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": 1,+ **STREAM_KWARGS,+ },+ [0],+ )+ assert kwargs["use_stream"] is True+ apply_group_offloading(offloaded, **kwargs)+ x = torch.randn(2, 16, 256, dtype = torch.bfloat16, device = "cuda")+ with torch.no_grad():+ want = resident(x)+ for _ in range(3):+ assert torch.equal(offloaded(x), want)+++def test_v1_int8_payload_returns_to_the_host_after_each_forward():+ """The streamed v1 payload (int8 data + scales) must leave the GPU on offload, else the whole denoiser accumulates+ there: diffusers restores torchao weights through ``tensor_data_names``, which the v1 classes do not declare."""+ torch = _v1_int8_stack()+ from diffusers.hooks import apply_group_offloading+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ assert mem.install_torchao_v1_int8_pin_ops() is True+ torch.manual_seed(0)+ blocks = torch.nn.Sequential(+ *[+ torch.nn.Sequential(+ torch.nn.Linear(1024, 1024), torch.nn.GELU(), torch.nn.Linear(1024, 1024)+ )+ for _ in range(4)+ ]+ ).to(torch.bfloat16)+ quantize_(blocks, Int8DynamicActivationInt8WeightConfig(set_inductor_config = False))+ blocks.requires_grad_(False)+ kwargs = mem._torchao_group_offload_kwargs(+ blocks,+ {+ "onload_device": torch.device("cuda"),+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": 1,+ **STREAM_KWARGS,+ },+ [0],+ )+ assert kwargs["use_stream"] is True+ apply_group_offloading(blocks, **kwargs)+ x = torch.randn(2, 16, 1024, dtype = torch.bfloat16, device = "cuda")+ with torch.no_grad():+ for _ in range(2):+ blocks(x)+ torch.cuda.synchronize()+ for block in blocks:+ for linear in (block[0], block[2]):+ impl = linear.weight.original_weight_tensor.tensor_impl+ assert impl.int_data.device.type == "cpu"+ assert impl.scale.device.type == "cpu"diff --git a/studio/backend/tests/test_diffusion_measured_placement.py b/studio/backend/tests/test_diffusion_measured_placement.pynew file mode 100644index 00000000000..fc368480711--- /dev/null+++ b/studio/backend/tests/test_diffusion_measured_placement.py@@ -0,0 +1,489 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Measured-activation placement and partial denoiser residency (``diffusion_memory.py``), plus the Qwen-Image-2.1+prefix K/V compaction (``diffusion_qwenimage21.py``).++The planner cases use the estimates a real Qwen-Image-2.1 auto load logged (int8 DiT + fp8 text encoder): on a 24 GB+card the flat plan read ``safe_device_budget_mib 21432, resident_required_mib 29870`` and streamed the text encoder+on every prompt, although the loaded weights are 6922 + 8959 + 644 MiB. CPU-only; the loaded sizes and the torchao+check are stubbed. The partial-residency mechanics need a CUDA device and real diffusers group offloading.+"""++from __future__ import annotations++import types++import pytest++import core.inference.diffusion_memory as dm++# Loaded storage of Qwen-Image-2.1 auto (int8 transformer, fp8 text encoder, bf16 VAE), MiB.+Q21_LOADED = {+ "transformer": (6922, "dit"),+ "text_encoder": (8959, "text_encoder"),+ "vae": (644, "other"),+}+# The flat plan's inputs for that load (family table sizes, flat 8192 headroom).+Q21_FLAT = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)+++def _memory(budget_mib: int, total_mib: int) -> "dm.DeviceMemory":+ """A discrete CUDA snapshot whose safe budget is exactly ``budget_mib``."""+ reserve = dm._reserve_mib("discrete_vram", total_mib)+ return dm.DeviceMemory("cuda", "cuda", "discrete_vram", budget_mib + reserve, total_mib)+++def _flat_plan(+ budget_mib: int,+ total_mib: int,+ mode = None,+):+ plan = dm.plan_diffusion_memory(+ target = types.SimpleNamespace(supports_model_cpu_offload = True),+ device_memory = _memory(budget_mib, total_mib),+ runtime_headroom_mib = dm.estimate_image_runtime_mib(+ width = None, height = None, family = "qwen-image-2.1"+ ),+ requested_mode = mode,+ **Q21_FLAT,+ )+ if plan.offload_policy == dm.OFFLOAD_MODEL:+ # what torchao_offload_plan makes of whole-module offload for a streamable int8 denoiser+ plan = dm.torchao_streaming_plan(plan)+ return plan+++def q21_pipe(monkeypatch):+ monkeypatch.delenv(+ getattr(dm, "MEASURED_ACTIVATION_ENV", "UNSLOTH_DIFFUSION_MEASURED_ACTIVATION"),+ raising = False,+ )+ monkeypatch.delenv(+ getattr(dm, "PARTIAL_RESIDENT_ENV", "UNSLOTH_DIFFUSION_PARTIAL_RESIDENT"), raising = False+ )+ monkeypatch.setattr(dm, "_loaded_component_mib", lambda pipe: dict(Q21_LOADED), raising = False)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ return object()+++def _refine(+ pipe,+ plan,+ family = "qwen-image-2.1",+ speed = "default",+):+ return dm.refine_plan_from_loaded_weights(pipe, plan, family = family, speed_mode = speed)+++def test_measured_headroom_values(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)+ # 1849 MiB measured worst phase x 1.15, rounded up to 256 MiB; linear in pixels+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") == 2304+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "max") == 2304+ assert (+ dm.measured_image_runtime_mib("qwen-image-2.1", "default", width = 2048, height = 2048) == 8704+ )+ # never below the 1 MP figure for a smaller canvas+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default", width = 512, height = 512) == 2304+ # unmeasured family / eager tiers keep the flat estimate+ assert dm.measured_image_runtime_mib("flux", "default") is None+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "off") is None+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "eager") is None+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", "0")+ assert dm.measured_image_runtime_mib("qwen-image-2.1", "default") is None+++def test_24gb_keeps_transformer_and_text_encoder_resident(q21_pipe):+ plan = _flat_plan(21432, 24576)+ # the reported flat decision: DiT resident, text encoder streamed on every prompt+ assert plan.estimates["resident_required_mib"] == 29870+ assert (+ plan.offload_policy == dm.OFFLOAD_GROUP+ and plan.stream_text_encoders+ and not plan.stream_transformer+ )+ new = _refine(q21_pipe, plan)+ # the flat hooks stay (so an oversized request can stream again); the whole encoder is kept resident in them+ assert new.offload_policy == dm.OFFLOAD_GROUP+ assert new.stream_text_encoders and not new.stream_transformer+ assert new.resident_transformer_mib is None+ assert (+ new.resident_text_encoder_mib+ == 21432 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922+ >= 8959+ )+ assert new.estimates["measured_runtime_headroom_mib"] == 2304+ # never under-reserved: loaded weights + measured peak x margin + base overhead within the safe budget+ assert 6922 + 8959 + 644 + 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB <= 21432+++def test_16gb_keeps_transformer_resident_streams_encoder(q21_pipe):+ plan = _flat_plan(13638, 16376)+ assert (+ plan.offload_policy == dm.OFFLOAD_GROUP and plan.stream_transformer+ ) # flat: every DiT block streamed+ new = _refine(q21_pipe, plan)+ assert new.offload_policy == dm.OFFLOAD_GROUP+ assert new.stream_text_encoders and new.stream_transformer+ # every DiT group kept resident through its hooks, the rest of the room holds encoder layers+ assert new.resident_transformer_mib == 6922+++def test_l4_class_card_keeps_most_of_the_encoder_resident(q21_pipe):+ """An L4 reports 23034 MiB: its 10% reserve leaves the whole encoder ~900 MiB short, so the DiT stays resident+ and only the encoder layers past the budget stream."""+ budget = 20000+ new = _refine(q21_pipe, _flat_plan(budget, 23034))+ assert new.offload_policy == dm.OFFLOAD_GROUP and new.stream_text_encoders+ room = budget - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922+ assert new.resident_text_encoder_mib == room and 0 < room < 8959+ assert new.as_public_dict()["resident_text_encoder_mib"] == room+++def test_16gb_encoder_room_and_kill_switch(q21_pipe, monkeypatch):+ new = _refine(q21_pipe, _flat_plan(13638, 16376))+ assert new.resident_text_encoder_mib == 13638 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644 - 6922+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")+ plan = _flat_plan(13638, 16376)+ assert _refine(q21_pipe, plan) is plan+++def test_12gb_partial_residency_sized_from_budget(q21_pipe):+ plan = _flat_plan(9550, 12288)+ assert plan.offload_policy == dm.OFFLOAD_STREAMING+ new = _refine(q21_pipe, plan)+ assert new.offload_policy == dm.OFFLOAD_STREAMING+ room = 9550 - 2304 - dm.DEFAULT_BASE_OVERHEAD_MIB - 644+ assert new.resident_transformer_mib == room+ assert 0 < room < 6922+ assert new.as_public_dict()["resident_transformer_mib"] == room+++def test_no_room_keeps_flat_plan(q21_pipe):+ plan = _flat_plan(4000, 8188)+ assert _refine(q21_pipe, plan) is plan+++ "total,budget", [(24576, 21432), (16376, 13638), (12288, 9550), (8188, 5450)]+)+def test_kill_switch_restores_flat_plan(q21_pipe, monkeypatch, total, budget):+ plan = _flat_plan(budget, total)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", "0")+ assert _refine(q21_pipe, plan) is plan+++def test_partial_kill_switch_only_drops_partial_tier(q21_pipe, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")+ for total, budget in ((12288, 9550), (16376, 13638), (24576, 21432)):+ plan = _flat_plan(budget, total)+ assert _refine(q21_pipe, plan) is plan+++ "family,speed,mode",+ [+ ("flux", "default", None),+ ("qwen-image-2.1", "off", None),+ ("qwen-image-2.1", "default", "balanced"),+ ],+)+def test_unmeasured_or_explicit_untouched(q21_pipe, family, speed, mode):+ plan = _flat_plan(21432, 24576, mode = mode)+ assert _refine(q21_pipe, plan, family = family, speed = speed) is plan+++def test_non_torchao_and_unified_untouched(q21_pipe, monkeypatch):+ plan = _flat_plan(21432, 24576)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)+ assert _refine(q21_pipe, plan) is plan+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ unified = dm.replace(+ plan, device_memory = dm.DeviceMemory("cuda", "cuda", "unified_memory", 23000, 24576)+ )+ assert _refine(q21_pipe, unified) is unified+++def test_never_moves_away_from_residency(q21_pipe):+ resident = _flat_plan(200000, 200000)+ assert resident.offload_policy == dm.OFFLOAD_NONE+ assert _refine(q21_pipe, resident) is resident+ model = dm.replace(_flat_plan(9550, 12288), offload_policy = dm.OFFLOAD_MODEL)+ assert _refine(q21_pipe, model) is model++[email protected]("budget", list(range(3000, 30001, 250)))+def test_every_tier_fits_measured_need(q21_pipe, budget):+ """Whatever the refinement picks, the bytes it keeps on the device plus the measured peak x margin plus the base+ overhead fit the safe budget (the reserve below it is untouched), and it is never slower than the flat pick."""+ total = max(8188, budget + 3000)+ plan = _flat_plan(budget, total)+ new = _refine(q21_pipe, plan)+ head = 2304 + dm.DEFAULT_BASE_OVERHEAD_MIB+ if new is plan:+ return+ # the policy and stream flags are the flat plan's: only resident rooms are added+ assert (new.offload_policy, new.stream_transformer, new.stream_text_encoders) == (+ plan.offload_policy,+ plan.stream_transformer,+ plan.stream_text_encoders,+ )+ kept = (+ 644+ + (6922 if not new.stream_transformer else int(new.resident_transformer_mib or 0))+ + int(new.resident_text_encoder_mib or 0)+ + (0 if new.stream_text_encoders or new.offload_policy == dm.OFFLOAD_STREAMING else 8959)+ )+ assert int(new.resident_transformer_mib or 0) <= 6922+ assert kept + head <= budget+++def test_compact_layer_cache_frees_full_storage(monkeypatch):+ torch = pytest.importorskip("torch")+ from core.inference import diffusion_qwenimage21 as q21++ full_k = torch.randn(1, 40, 4, 8)+ full_v = torch.randn(1, 40, 4, 8)+ cache = types.SimpleNamespace(k = full_k[:, :6], v = full_v[:, :6])+ assert cache.k.untyped_storage().nbytes() > cache.k.numel() * cache.k.element_size()+ q21._compact_layer_cache(cache)+ for got, ref in ((cache.k, full_k[:, :6]), (cache.v, full_v[:, :6])):+ assert got.untyped_storage().nbytes() == got.numel() * got.element_size()+ assert torch.equal(got, ref)+ owned = cache.k+ q21._compact_layer_cache(cache)+ assert cache.k is owned # already compact: untouched+ monkeypatch.setenv("UNSLOTH_DIFFUSION_Q21_COMPACT_KV", "0")+ assert not q21.compact_kv_enabled()+ monkeypatch.delenv("UNSLOTH_DIFFUSION_Q21_COMPACT_KV")+ assert q21.compact_kv_enabled()+++def _cuda_offload_model():+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("CUDA-only: diffusers group offloading with a copy stream")+ pytest.importorskip("diffusers.hooks")++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, 1024)+ self.blocks = torch.nn.ModuleList(+ torch.nn.Linear(1024, 1024) for _ in range(6)+ ) # ~4 MiB each+ self.proj_out = torch.nn.Linear(1024, 64)++ def forward(self, x):+ x = self.proj_in(x)+ for block in self.blocks:+ x = torch.nn.functional.gelu(block(x))+ return self.proj_out(x)++ torch.manual_seed(0)+ return torch, Net()++[email protected]("keep_blocks", [0, 2, 6])+def test_partial_residency_bit_identical_and_placed(keep_blocks, monkeypatch):+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ x = torch.randn(3, 64)+ ref = net.to("cuda")(x.cuda()).cpu()+ net.to("cpu")+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ mib = 1024 * 1024+ top = sum(+ p.numel() * p.element_size() for m in (net.proj_in, net.proj_out) for p in m.parameters()+ )+ block = sum(p.numel() * p.element_size() for p in net.blocks[0].parameters())+ # whole MiB, rounded up: room for the top-level group and keep_blocks blocks, short of one more block+ room = -(-(top + keep_blocks * block) // mib) if keep_blocks else 0+ kept = dm._keep_groups_resident(net, room, "cuda")+ for _ in range(3): # first forward traces the prefetch order, later ones prefetch+ out = net(x.cuda()).cpu()+ assert torch.equal(out, ref)+ placed = [next(b.parameters()).device.type for b in net.blocks]+ want = ["cuda"] * min(keep_blocks, 6) + ["cpu"] * (6 - min(keep_blocks, 6))+ if keep_blocks:+ assert kept > 0+ assert next(net.proj_in.parameters()).device.type == "cuda"+ assert placed[:keep_blocks] == want[:keep_blocks]+ assert placed[keep_blocks:] == want[keep_blocks:]+++def test_partial_residency_kill_switch(monkeypatch):+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ )+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", "0")+ assert dm._keep_groups_resident(net, 10_000, "cuda") == 0+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)+++def _family_names():+ from core.inference.diffusion_families import supported_family_names+ return supported_family_names()++[email protected]("speed", ["off", "eager", "default", "max"])+ "budget,total", [(21432, 24576), (13638, 16376), (9550, 12288), (5450, 8188)]+)+def test_only_measured_family_moves(q21_pipe, speed, budget, total):+ """Every other supported family keeps its flat plan byte for byte, on every speed tier and budget."""+ names = _family_names()+ assert "qwen-image-2.1" in names and len(names) > 5+ plan = _flat_plan(budget, total)+ for family in names:+ new = _refine(q21_pipe, plan, family = family, speed = speed)+ if family == "qwen-image-2.1" and speed in ("default", "max"):+ continue+ assert new is plan, family+++def test_resident_group_keeps_copy_stream_wait(monkeypatch):+ """A streamed group that prefetched its successor skips its own wait and relies on the successor's onload_ to+ synchronize the copy stream. A resident successor must still do that wait (else the streamed group computes on+ weights in flight), and its offload must free nothing."""+ pytest.importorskip("torch")+ pytest.importorskip("diffusers.hooks")+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)++ class Stream:+ waits = 0++ def synchronize(self):+ Stream.waits += 1++ module = object()+ stream = Stream()+ groups = [+ types.SimpleNamespace(+ modules = [],+ parameters = [],+ buffers = [],+ offload_leader = object(),+ stream = stream,+ cpu_param_dict = {},+ offload_to_disk_path = None,+ )+ for _ in range(3)+ ]+ monkeypatch.setattr(dm, "_offload_groups", lambda m: groups)+ dm._keep_groups_resident(module, 1, "cpu")+ assert all(getattr(g, "_unsloth_resident", False) for g in groups)+ for g in groups:+ g.onload_()+ g.offload_()+ assert Stream.waits == 3+++def test_oversized_request_streams_resident_groups_and_restores_them(monkeypatch):+ """A request past the measured reserve streams the kept groups again (the flat plan's placement), stays+ bit-identical, and the groups are resident again afterwards."""+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ x = torch.randn(3, 64)+ ref = net.to("cuda")(x.cuda()).cpu()+ net.to("cpu")+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ record_stream = True,+ non_blocking = True,+ )+ pipe = types.SimpleNamespace(components = {"transformer": net})+ assert dm._keep_groups_resident(net, 1024, "cuda") > 0+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)+ assert torch.equal(net(x.cuda()).cpu(), ref)++ restore = dm.release_resident_groups(pipe, 1024)+ assert restore is not None+ # blocks first, from the last; the top-level group goes last+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)+ for _ in range(2):+ assert torch.equal(net(x.cuda()).cpu(), ref)+ assert all(next(b.parameters()).device.type == "cpu" for b in net.blocks)++ restore()+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)+ for _ in range(2):+ assert torch.equal(net(x.cuda()).cpu(), ref)+ assert dm.release_resident_groups(types.SimpleNamespace(components = {}), 1024) is None+++def test_partial_release_frees_only_what_the_request_needs(monkeypatch):+ torch, net = _cuda_offload_model()+ from diffusers.hooks import apply_group_offloading++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PARTIAL_RESIDENT", raising = False)+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ )+ dm._keep_groups_resident(net, 1024, "cuda")+ pipe = types.SimpleNamespace(components = {"transformer": net})+ restore = dm.release_resident_groups(pipe, 1) # one ~4 MiB block covers 1 MiB+ placed = [next(b.parameters()).device.type for b in net.blocks]+ assert placed == ["cuda"] * 5 + ["cpu"]+ restore()+ assert all(next(b.parameters()).device.type == "cuda" for b in net.blocks)+++def test_measured_request_extra(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DIFFUSION_MEASURED_ACTIVATION", raising = False)+ pipe = types.SimpleNamespace(_unsloth_measured_reserve = (2304, "qwen-image-2.1", "default"))+ # the default request is what the reserve was sized for+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024) == 0+ assert dm.measured_request_extra_mib(pipe, width = 512, height = 512) == 0+ # a bigger canvas, a batch and reference images each need more+ assert dm.measured_request_extra_mib(pipe, width = 2048, height = 2048) == 8704 - 2304+ assert dm.measured_request_extra_mib(pipe, width = 1024, height = 1024, batch_size = 2) > 0+ one_ref = int(1024 * 1024 * 0.32)+ assert dm.measured_request_extra_mib(+ pipe, width = 1024, height = 1024, condition_pixels = one_ref+ ) == int(8192 * 0.32 * 1.15)+ # no measured placement: never releases+ assert dm.measured_request_extra_mib(object(), width = 2048, height = 2048) == 0+++def test_generate_releases_and_restores_resident_groups():+ src = (__import__("pathlib").Path(dm.__file__).parent / "diffusion.py").read_text(+ encoding = "utf-8"+ )+ assert "restore_resident = release_resident_groups(state.pipe, extra_mib, logger)" in src+ finally_at = src.index("if restore_resident is not None:")+ assert "restore_resident()" in src[finally_at : finally_at + 200]+ assert "pipe._unsloth_measured_reserve = (" in srcdiff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.pyindex 03d8eb01808..721141fb3ab 100644--- a/studio/backend/tests/test_diffusion_memory.py+++ b/studio/backend/tests/test_diffusion_memory.py@@ -935,6 +935,14 @@ def test_apply_streaming_uses_block_and_leaf_hooks_with_bounded_cpu_memory(monke"vae": vae,},)+ import core.inference.diffusion_memory as mem++ # no pinnable host RAM: every streamed module stays unpinned (bounded CPU memory), and record_stream keeps the+ # offload from draining the compute stream+ monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 0)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)effective, _ = apply_memory_plan(pipe, _manual_plan(OFFLOAD_STREAMING, tiling = True), device = "cuda")@@ -945,7 +953,104 @@ def test_apply_streaming_uses_block_and_leaf_hooks_with_bounded_cpu_memory(monke("block_level", DEFAULT_GROUP_BLOCKS),("leaf_level", None),]- assert all(call[3:] == (True, True, False, True) for call in calls)+ assert all(call[3:] == (True, True, True, True) for call in calls)+++def _streaming_apply_kwargs(+ monkeypatch,+ budget,+ *,+ env = None,+ request_background = False,+):+ """{component: apply kwargs} and the modules handed to a deferred pinner, for one streaming apply."""+ import sys++ import core.inference.diffusion_memory as mem++ Module = _install_sized_torch(monkeypatch)+ seen: dict = {}+ deferred: list = []++ def _apply(+ module,+ *,+ onload_device,+ offload_device,+ offload_type,+ num_blocks_per_group = None,+ use_stream = False,+ non_blocking = False,+ record_stream = False,+ low_cpu_mem_usage = False,+ ):+ seen[module.name] = {+ "use_stream": use_stream,+ "record_stream": record_stream,+ "low_cpu_mem_usage": low_cpu_mem_usage,+ "offload_type": offload_type,+ }++ if "diffusers" not in sys.modules:+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))+ hooks = types.ModuleType("diffusers.hooks")+ hooks.apply_group_offloading = _apply+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ if env is None:+ monkeypatch.delenv(mem.STREAMING_PREFETCH_ENV, raising = False)+ else:+ monkeypatch.setenv(mem.STREAMING_PREFETCH_ENV, env)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.delenv(mem.BACKGROUND_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.setattr(mem, "install_group_pin_wait", lambda: True)+ monkeypatch.setattr(+ mem,+ "_defer_pinning",+ lambda pipe, module, device, logger: deferred.append(module.name) or True,+ )+ parts = {"transformer": Module(4000), "text_encoder": Module(7500), "vae": Module(200)}+ for name, module in parts.items():+ module.name = name+ pipe = types.SimpleNamespace(transformer = parts["transformer"], components = parts)+ if request_background:+ mem.request_background_pins(pipe)+ mem._apply_streaming_offload(pipe, "cuda", None)+ return seen, deferred+++def test_streaming_pins_the_transformer_within_the_host_budget(monkeypatch):+ # An unpinned host copy is re-pinned on the CPU at every onload, and record_stream=False synchronises the compute+ # stream after every group, so that pinning ran with the GPU idle (Wan2.2-5B streamed on an L4: see the PR).+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000)+ assert seen["transformer"]["low_cpu_mem_usage"] is False+ assert seen["transformer"]["record_stream"] is True+ # the encoder does not fit beside the transformer: it stays unpinned, still overlapped+ assert seen["text_encoder"]["low_cpu_mem_usage"] is True+ assert seen["text_encoder"]["record_stream"] is True+ assert deferred == []+++def test_streaming_pins_everything_on_a_ram_rich_host(monkeypatch):+ seen, _ = _streaming_apply_kwargs(monkeypatch, 40_000)+ assert all(kw["low_cpu_mem_usage"] is False for kw in seen.values()), seen+++def test_streaming_pins_off_the_load_path_when_asked(monkeypatch):+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 6000, request_background = True)+ # applied unpinned, then handed to the background pinner; the encoder is outside the budget so never pinned+ assert seen["transformer"]["low_cpu_mem_usage"] is True+ assert seen["transformer"]["record_stream"] is True+ assert deferred == ["transformer"]+++def test_streaming_prefetch_kill_switch_restores_the_old_kwargs(monkeypatch):+ seen, deferred = _streaming_apply_kwargs(monkeypatch, 40_000, env = "0", request_background = True)+ assert all(+ kw["low_cpu_mem_usage"] is True and kw["record_stream"] is False for kw in seen.values()+ ), seen+ assert deferred == []def test_apply_streaming_does_not_fall_back_to_known_oom_path(monkeypatch):@@ -2741,3 +2846,265 @@ def test_a_late_encoder_refusal_never_streams_a_torchao_transformer(monkeypatch))assert order == ["hook:text_encoder"]assert te2.placed is None+++def test_streamed_transformer_runs_pinned_and_overlapped_on_a_real_gpu(monkeypatch):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ pytest.importorskip("diffusers.hooks")+ import core.inference.diffusion_memory as mem++ monkeypatch.delenv("UNSLOTH_DIFFUSION_STREAMING_PREFETCH", raising = False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, 64)+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(6))++ def forward(self, x):+ x = self.proj_in(x)+ for block in self.blocks:+ x = x + torch.tanh(block(x))+ return x++ torch.manual_seed(0)+ net = Net().eval()+ x = torch.randn(8, 64)+ with torch.no_grad():+ want = net(x)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ mem._apply_streaming_offload(pipe, "cuda", None)+ groups = [g for g in mem._offload_groups(net) if g.stream is not None]+ assert groups and all(g.record_stream for g in groups)+ assert all(t.is_pinned() for g in groups for t in g.cpu_param_dict.values())+ with torch.no_grad():+ for _ in range(3): # the first forward traces the order, the later ones prefetch+ got = net(x.cuda())+ torch.cuda.synchronize()+ assert torch.allclose(got.cpu(), want, atol = 1e-5)+++def _top_group_module(+ monkeypatch,+ *,+ stream = None,+ torchao = False,+):+ """A module whose diffusers registry holds a top-level group, on a fake diffusers.hooks.group_offloading."""+ import sys++ torch = pytest.importorskip("torch")+ go = types.ModuleType("diffusers.hooks.group_offloading")+ go._GROUP_OFFLOADING = "group_offloading"+ go._is_torchao_tensor = lambda t: torchao+ hooks = types.ModuleType("diffusers.hooks")+ hooks.group_offloading = go+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)+ if "diffusers" not in sys.modules:+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))+ leaf = torch.nn.Linear(4, 4)+ group = types.SimpleNamespace(+ stream = stream,+ offload_to_disk_path = None,+ onload_device = types.SimpleNamespace(type = "cuda"),+ modules = [leaf],+ parameters = [],+ buffers = [],+ onload_ = "diffusers",+ offload_ = "diffusers",+ )+ hook = types.SimpleNamespace(group = group)+ module = types.SimpleNamespace(+ _diffusers_hook = types.SimpleNamespace(+ get_hook = lambda name: hook if name == "group_offloading" else None+ )+ )+ return module, group+++ "env", ["UNSLOTH_DIFFUSION_PIN_TOP_GROUP", "UNSLOTH_DIFFUSION_GROUP_OFFLOAD_PIN"]+)+def test_top_level_group_kill_switch(monkeypatch, env):+ import core.inference.diffusion_memory as mem++ module, group = _top_group_module(monkeypatch)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ monkeypatch.setenv(env, "0")+ assert mem._pin_top_level_group(module) is False+ assert group.onload_ == "diffusers" and group.offload_ == "diffusers"+++def test_top_level_group_respects_the_pinned_allocator_rounding(monkeypatch):+ import core.inference.diffusion_memory as mem++ module, group = _top_group_module(monkeypatch)+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ # a 3 MiB weight pins as a 4 MiB block: a 3 MiB budget must refuse it+ group.modules = [+ types.SimpleNamespace(+ parameters = lambda: [__import__("torch").empty(3 << 18)], buffers = lambda: []+ )+ ]+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 3)+ assert mem._pin_top_level_group(module) is False+ assert group.onload_ == "diffusers"+++def test_top_level_group_counts_against_the_running_pin_total(monkeypatch):+ import core.inference.diffusion_memory as mem++ module, group = _top_group_module(monkeypatch)+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ group.modules = [+ types.SimpleNamespace(+ parameters = lambda: [__import__("torch").empty(1 << 18)], buffers = lambda: []+ )+ ]+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 8)+ # 8 MiB budget, 8 MiB of encoders already pinned: the 1 MiB top group no longer fits+ assert mem._pin_top_level_group(module, None, [8]) is False+ assert group.onload_ == "diffusers"+ total = [4]+ assert mem._pin_top_level_group(module, None, total) is True+ assert total == [5]++[email protected]("kind", ["streamed_group", "torchao"])+def test_top_level_group_left_alone_when_not_the_streamless_top_group(monkeypatch, kind):+ import core.inference.diffusion_memory as mem++ module, group = _top_group_module(+ monkeypatch,+ stream = object() if kind == "streamed_group" else None,+ torchao = kind == "torchao",+ )+ monkeypatch.delenv(mem.PIN_TOP_GROUP_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)+ assert mem._pin_top_level_group(module) is False+ assert group.onload_ == "diffusers"++[email protected]("tier", ["group", "streaming"])+def test_top_level_weights_onload_from_one_pinned_copy_on_a_real_gpu(monkeypatch, tier):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ pytest.importorskip("diffusers.hooks")+ import core.inference.diffusion_memory as mem++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj_in = torch.nn.Linear(64, 64)+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(64, 64) for _ in range(4))+ self.proj_out = torch.nn.Linear(64, 64)++ def forward(self, x):+ x = self.proj_in(x)+ for block in self.blocks:+ x = x + torch.tanh(block(x))+ return self.proj_out(x)++ torch.manual_seed(0)+ net = Net().eval()+ x = torch.randn(8, 64)+ with torch.no_grad():+ want = net(x)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ if tier == "group":+ assert mem._apply_group_offload(pipe, "cuda", logger = None) is True+ else:+ mem._apply_streaming_offload(pipe, "cuda", None)+ with torch.no_grad():+ got = [net(x.cuda()).cpu() for _ in range(3)]+ torch.cuda.synchronize()+ # offloaded between forwards onto the same pinned host copy: no device-to-host copy into a fresh host buffer+ ptr = net.proj_out.weight.data_ptr()+ assert net.proj_out.weight.device.type == "cpu" and net.proj_out.weight.is_pinned()+ with torch.no_grad():+ got.append(net(x.cuda()).cpu())+ torch.cuda.synchronize()+ assert net.proj_out.weight.data_ptr() == ptr+ assert all(torch.allclose(g, want, atol = 1e-5) for g in got)+++def test_top_level_weights_replaced_while_offloaded_are_picked_up(monkeypatch):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ pytest.importorskip("diffusers.hooks")+ import copy++ import core.inference.diffusion_memory as mem++ monkeypatch.delenv("UNSLOTH_DIFFUSION_PIN_TOP_GROUP", raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: 1 << 20)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False)++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList(torch.nn.Linear(16, 16) for _ in range(3))+ self.proj_out = torch.nn.Linear(16, 16)++ def forward(self, x):+ for block in self.blocks:+ x = x + torch.tanh(block(x))+ return self.proj_out(x)++ torch.manual_seed(0)+ net = Net().eval()+ ref = copy.deepcopy(net)+ x = torch.randn(4, 16)+ pipe = types.SimpleNamespace(transformer = net, components = {"transformer": net})+ mem._apply_streaming_offload(pipe, "cuda", None)+ with torch.no_grad():+ for _ in range(2):+ net(x.cuda())+ torch.cuda.synchronize()+ # e.g. a .to() conversion or an adapter fused on the host while the weights sit offloaded+ net.proj_out.weight.data = net.proj_out.weight.data * 2+ ref.proj_out.weight.data = ref.proj_out.weight.data * 2+ got = net(x.cuda()).cpu()+ want = ref(x)+ assert torch.allclose(got, want, atol = 1e-5)+++def test_streaming_counts_pinned_encoders_before_a_torchao_denoiser_pins(monkeypatch):+ # The torchao denoiser pins within a running total; encoders the plan pins must already be on it (as on the group+ # tier), or the two together exceed the pinnable host budget.+ import core.inference.diffusion_memory as mem++ seen_totals: dict = {}+ real = mem._torchao_group_offload_kwargs++ def _spy(+ module,+ kwargs,+ pinned_mib = None,+ ):+ seen_totals[module.name] = pinned_mib[0] if pinned_mib else None+ return real(module, kwargs, pinned_mib)++ monkeypatch.setattr(mem, "_torchao_group_offload_kwargs", _spy)+ _streaming_apply_kwargs(monkeypatch, 40_000)+ # the 7500 MiB encoder, rounded up to a power of two like the pinned allocator+ assert seen_totals["transformer"] == 8192diff --git a/studio/backend/tests/test_diffusion_offload_precision.py b/studio/backend/tests/test_diffusion_offload_precision.pyindex 3abd8e186ca..e6fcf4e7143 100644--- a/studio/backend/tests/test_diffusion_offload_precision.py+++ b/studio/backend/tests/test_diffusion_offload_precision.py@@ -82,7 +82,8 @@ SURVIVAL = {s: {v: (True, OFFLOAD_GROUP) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")},"group_dit_streamed": {- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_GROUP)},+ # 0.17 streams int8 through its pinnable Int8Tensor (UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17)+ "int8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},"fp8": {T016: (False, None), T017: (True, OFFLOAD_GROUP), T018: (True, OFFLOAD_GROUP)},"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},},@@ -90,7 +91,11 @@ SURVIVAL = {s: {v: (True, OFFLOAD_MODEL) for v in (T016, T017, T018)} for s in ("int8", "fp8", "nvfp4")},"model_too_big": {- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},+ "int8": {+ T016: (False, None),+ T017: (True, OFFLOAD_STREAMING),+ T018: (True, OFFLOAD_STREAMING),+ },"fp8": {T016: (False, None),T017: (True, OFFLOAD_STREAMING),@@ -99,7 +104,11 @@ SURVIVAL = {"nvfp4": {T016: (False, None), T017: (False, None), T018: (False, None)},},"streaming": {- "int8": {T016: (False, None), T017: (False, None), T018: (True, OFFLOAD_STREAMING)},+ "int8": {+ T016: (False, None),+ T017: (True, OFFLOAD_STREAMING),+ T018: (True, OFFLOAD_STREAMING),+ },"fp8": {T016: (False, None),T017: (True, OFFLOAD_STREAMING),@@ -120,7 +129,10 @@ CASES = [@pytest.mark.parametrize("placement, scheme, version, survives, policy", CASES)-def test_survival_table(placement, scheme, version, survives, policy):+def test_survival_table(monkeypatch, placement, scheme, version, survives, policy):+ # the table is about versions, not about which torchao this test host happens to have+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)plan = PLACEMENTS[placement]assert torchao_survives_plan(plan, scheme, torchao_version = version) is survivesplaced = torchao_offload_plan(plan, scheme, torchao_version = version)@@ -246,14 +258,18 @@ NO_STREAM = {"onload_device": "cuda", "use_stream": False}(("Int8Tensor",), True, 10**7, "pinned"),(("Float8Tensor",), True, 10**7, "pinned"),(("Int8Tensor",), True, None, "no stream"),- # torchao 0.17 default int8: no aten.is_pinned, so no copy stream- (("LinearActivationQuantizedTensor",), False, 10**7, "no stream"),+ # torchao 0.17 default int8: no aten.is_pinned of its own; streams once Studio registers the pin ops+ (("LinearActivationQuantizedTensor",), False, 10**7, "as is"),+ (("LinearActivationQuantizedTensor",), False, 10**7, "no stream, no pin ops"),],)def test_group_offload_kwargs_per_weight_class(monkeypatch, classes, low_cpu_mem_usage, pin_budget, expected):monkeypatch.setattr(mem, "_pin_budget_mib", lambda: pin_budget)+ # whether this host's torchao ships the v1 classes must not decide the table+ pin_ops = expected != "no stream, no pin ops"+ monkeypatch.setattr(mem, "install_torchao_v1_int8_pin_ops", lambda: pin_ops, raising = False)kwargs = {**STREAM_KW, "low_cpu_mem_usage": low_cpu_mem_usage}out = mem._torchao_group_offload_kwargs(_Module(*classes, plain = True), kwargs)if expected == "as is":@@ -509,9 +525,12 @@ def test_auto_keeps_int8_on_the_offload_tier(planner, family, base, gib, expecteassert planner(family, base, gib) == expected[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted fp8")])+# 0.17 now streams int8 (pinnable Int8Tensor), so it takes the same hosted int8 as 0.18 instead of falling to fp8[email protected]("version, expected", [(T016, "bf16"), (T017, "hosted int8")])def test_older_torchao_streamed_seeds(planner, monkeypatch, version, expected):monkeypatch.setattr(mem, "_installed_torchao_version", lambda: version)+ monkeypatch.setattr(mem, "_int8_tensor_pinnable", lambda: True, raising = False)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_INT8_STREAM_TORCHAO17", raising = False)precision, _policy = planner("hunyuanimage-2.1", "hunyuanvideo-community/HunyuanImage-2.1-Diffusers", 16)@@ -587,6 +606,8 @@ def test_quantised_blocks_survive_group_offload_under_no_grad(want = resident(x)for _ in range(3):assert torch.equal(offloaded(x), want)+ if type(next(offloaded.parameters())).__name__ == "LinearActivationQuantizedTensor":+ return # torchao <= 0.17 v1 int8 (pin-op shim): its onload is a plain re-wrap, which inference_mode allowswith pytest.raises(Exception), torch.inference_mode():offloaded(x)diff --git a/studio/backend/tests/test_diffusion_offload_seed_balanced.py b/studio/backend/tests/test_diffusion_offload_seed_balanced.pynew file mode 100644index 00000000000..0b657a1a1f7--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_seed_balanced.py@@ -0,0 +1,276 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Two small-card load failures, Qwen-Image-2.1 sized, CPU only.++1. An offloading plan must seed a pre-quantized (int8 / fp8) denoiser on the host: seeding it onto the GPU whole+ (~7 GiB int8) left the 8 GB streaming hooks no room for their first block.+2. An explicit ``memory_mode=balanced`` is fit-checked against the loaded companions, walking down the streamed+ tiers, instead of loading every companion resident and OOMing at the first generate.+"""++from __future__ import annotations++import ast+from pathlib import Path++import pytest+import torch++import core.inference.diffusion_memory as dm+from core.inference.diffusion_memory import (+ BALANCED_FIT_CHECK_ENV,+ DeviceMemory,+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,+ OFFLOAD_NONE,+ OFFLOAD_STREAMING,+ PREQUANT_SEED_ON_HOST_ENV,+ plan_diffusion_memory,+ prequant_seed_device,+ refine_balanced_plan_for_components,+ refine_memory_plan_for_components,+)++MIB = 1024 * 1024+BACKEND = Path(__file__).resolve().parents[1]++# Qwen-Image-2.1 auto (int8 DiT + fp8 TE) as the planner prices it, and as it loads.+DENSE = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)+# The explicit transformer_quant=int8 path prices the encoder dense (16.7 GB) while it loads fp8 (9.0 GB).+DENSE_EXPLICIT = dict(+ model_dense_mib = 31566, companion_dense_mib = 18024, text_encoder_dense_mib = 16689+)+TE_LOADED_MIB = 8960+VAE_LOADED_MIB = 645+DIT_LOADED_MIB = 7448+++class _Target:+ supports_model_cpu_offload = True+++def _card(free_mib, total_mib):+ return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, total_mib)+++# (ballast free, simulated total) of the user's budget cells+CARD_24 = _card(24176, 24576)+CARD_16 = _card(15976, 16376)+CARD_12 = _card(11888, 12288)+CARD_8 = _card(7788, 8188)+++def _plan(+ card,+ mode = None,+ sizes = DENSE,+):+ return plan_diffusion_memory(+ target = _Target(),+ device_memory = card,+ runtime_headroom_mib = 8192,+ requested_mode = mode,+ **sizes,+ )+++def _module(mib):+ m = torch.nn.Module()+ # meta: sized like the loaded component, allocates nothing+ m.w = torch.nn.Parameter(+ torch.empty(mib * MIB, dtype = torch.uint8, device = "meta"), requires_grad = False+ )+ return m+++class _Pipe:+ def __init__(self):+ self.transformer = _module(DIT_LOADED_MIB)+ self.text_encoder = _module(TE_LOADED_MIB)+ self.vae = _module(VAE_LOADED_MIB)+ self.components = {+ "transformer": self.transformer,+ "text_encoder": self.text_encoder,+ "vae": self.vae,+ "tokenizer": object(),+ "scheduler": object(),+ }++[email protected](autouse = True)+def _clean_env(monkeypatch):+ monkeypatch.delenv(PREQUANT_SEED_ON_HOST_ENV, raising = False)+ monkeypatch.delenv(BALANCED_FIT_CHECK_ENV, raising = False)+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)+++# ---------------------------------------------------------------- 1. host seed for offloaded prequant denoisers++[email protected]("card", [CARD_8, CARD_12, CARD_16])+def test_offloading_auto_plan_seeds_int8_on_the_host(card):+ plan = (+ dm.torchao_streaming_plan(_plan(card))+ if _plan(card).offload_policy == OFFLOAD_MODEL+ else _plan(card)+ )+ assert plan.offload_policy != OFFLOAD_NONE+ assert not dm.plan_keeps_transformer_resident(plan)+ assert prequant_seed_device(plan, "cuda", "int8") == "cpu"+ assert prequant_seed_device(plan, "cuda:1", "fp8") == "cpu"+++def test_resident_plans_keep_the_gpu_seed():+ # 24 GB: transformer resident, encoders streamed -> unchanged (seed lands where it runs)+ plan24 = _plan(CARD_24)+ assert dm.plan_keeps_transformer_resident(plan24)+ assert prequant_seed_device(plan24, "cuda", "int8") == "cuda"+ full = _plan(_card(47000, 49140))+ assert full.offload_policy == OFFLOAD_NONE+ assert prequant_seed_device(full, "cuda:0", "int8") == "cuda:0"+++def test_seed_device_scheme_and_kill_switch(monkeypatch):+ plan = dm.torchao_streaming_plan(_plan(CARD_8))+ # schemes never measured under offload keep today's placement+ assert prequant_seed_device(plan, "cuda", "nvfp4") == "cuda"+ assert prequant_seed_device(plan, "cuda", None) == "cuda"+ monkeypatch.setenv(PREQUANT_SEED_ON_HOST_ENV, "0")+ assert prequant_seed_device(plan, "cuda", "int8") == "cuda"+++def test_denoiser_seed_forwards_the_placement(monkeypatch):+ import core.inference.diffusion_denoiser_prequant as dp+ import core.inference.diffusion_prequant as prequant+ import diffusers++ seen = {}++ def fake_load(cls, base, source, **kwargs):+ seen.update(kwargs)+ return torch.nn.Linear(2, 2)++ class _Fam:+ name = "qwen-image-2.1"+ transformer_class = "QwenImage21Transformer2DModel"++ monkeypatch.setattr(prequant, "load_prequantized_transformer", fake_load)+ monkeypatch.setattr(dp, "pipeline_seed_supported", lambda fam: True)+ monkeypatch.setattr(dp, "denoiser_prequant_source", lambda *a, **k: object())+ monkeypatch.setattr(diffusers, "QwenImage21Transformer2DModel", object, raising = False)+ out = dp.denoiser_prequant_pipe_kwargs(+ _Fam(),+ "Qwen/Qwen-Image-2.1",+ scheme = "int8",+ dtype = torch.bfloat16,+ device = "cuda",+ placement_device = "cpu",+ )+ assert out+ assert seen["device"] == "cuda" # kernels still selected for the compute device+ assert seen["placement_device"] == "cpu"+++def test_loader_materialises_on_the_placement_device():+ src = (BACKEND / "core/inference/diffusion_prequant.py").read_text()+ assert "transformer = transformer.to(placement_device or device)" in src+ assert "transformer = transformer.to(device)\n" not in src+++def test_pipeline_seed_call_passes_the_plan_placement():+ tree = ast.parse((BACKEND / "core/inference/diffusion.py").read_text())+ calls = [+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.Call)+ and getattr(n.func, "id", None) == "denoiser_prequant_pipe_kwargs"+ ]+ assert calls, "pipeline seed call not found"+ for call in calls:+ kw = {k.arg: k.value for k in call.keywords}+ assert "placement_device" in kw+ value = kw["placement_device"]+ assert (+ isinstance(value, ast.Call)+ and getattr(value.func, "id", None) == "prequant_seed_device"+ )+ assert [getattr(a, "id", None) for a in value.args] == [+ "plan",+ "device",+ "pipeline_seed_scheme",+ ]+++# ---------------------------------------------------------------- 2. balanced fit check+++def test_balanced_that_fits_is_unchanged():+ # 24 GB, explicit int8: the estimate (dense TE) says companions do not fit, the loaded ones do -> keep it+ plan = _plan(CARD_24, "balanced", DENSE_EXPLICIT)+ assert plan.offload_policy == OFFLOAD_GROUP and not plan.stream_text_encoders+ out = refine_balanced_plan_for_components(_Pipe(), plan)+ assert out.offload_policy == OFFLOAD_GROUP+ assert not out.stream_text_encoders+ assert out.reasons == plan.reasons++[email protected]("sizes", [DENSE, DENSE_EXPLICIT])+def test_balanced_16gb_streams_the_text_encoders(sizes):+ plan = _plan(CARD_16, "balanced", sizes)+ out = refine_balanced_plan_for_components(_Pipe(), plan)+ assert out.offload_policy == OFFLOAD_GROUP+ assert out.stream_text_encoders+ assert out.stream_transformer+++def test_balanced_12gb_drops_to_whole_module_offload():+ plan = _plan(CARD_12, "balanced", DENSE_EXPLICIT)+ out = refine_memory_plan_for_components(+ _Pipe(), refine_balanced_plan_for_components(_Pipe(), plan)+ )+ # the 9.0 GB encoder fits the 9.8 GB budget whole, so whole-module offload stays+ assert out.offload_policy == OFFLOAD_MODEL+ assert out.vae_tiling+++def test_balanced_8gb_streams_granularly():+ plan = _plan(CARD_8, "balanced", DENSE_EXPLICIT)+ out = refine_memory_plan_for_components(+ _Pipe(), refine_balanced_plan_for_components(_Pipe(), plan)+ )+ assert out.offload_policy == OFFLOAD_STREAMING++[email protected]("card", [CARD_12, CARD_8])+def test_balanced_torchao_denoiser_streams_instead_of_model_offload(monkeypatch, card):+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: True)+ out = refine_balanced_plan_for_components(_Pipe(), _plan(card, "balanced"))+ assert out.offload_policy == OFFLOAD_STREAMING+++def test_balanced_kill_switch_and_scope(monkeypatch):+ plan16 = _plan(CARD_16, "balanced")+ auto16 = _plan(CARD_16)+ # auto is not touched+ assert refine_balanced_plan_for_components(_Pipe(), auto16) is auto16+ monkeypatch.setenv(BALANCED_FIT_CHECK_ENV, "0")+ assert refine_balanced_plan_for_components(_Pipe(), plan16) is plan16+++def test_balanced_refinement_is_wired_before_component_refinement():+ src = (BACKEND / "core/inference/diffusion.py").read_text()+ assert (+ "refine_memory_plan_for_components(\n pipe, refine_balanced_plan_for_components(pipe, plan)\n"+ in src+ )+++def test_measured_placement_honours_the_legacy_cpu_offload_flag():+ # cpu_offload=True without a memory_mode asks for offload; the measured refinement must not make that load resident+ src = (BACKEND / "core/inference/diffusion.py").read_text(encoding = "utf-8")+ call = src.index("plan = refine_plan_from_loaded_weights(")+ guard = src.rindex("\n", 0, src.rindex("\n", 0, call))+ assert (+ "if not cpu_offload or normalize_memory_mode(memory_mode) is not None:" in src[guard:call]+ )diff --git a/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.py b/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.pynew file mode 100644index 00000000000..b4cea88761c--- /dev/null+++ b/studio/backend/tests/test_diffusion_offload_seed_balanced_behaviour.py@@ -0,0 +1,213 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Behaviour of the two small-card fixes through entry points that exist before and after them.++Imports only names present on main, so on main these fail by ASSERTION, not ImportError:++1. The pipeline seed call in diffusion.py, evaluated as written, drives ``load_prequantized_transformer`` (checkpoint+ I/O faked) under an offloading Qwen-Image-2.1 plan: the module must be materialised on the host, not the GPU.+2. The post-load refinement expression in diffusion.py, evaluated as written, applied to a balanced 16 GB plan and a+ pipe whose loaded fp8 text encoder is 8960 MiB: the encoder must not stay resident.+"""++from __future__ import annotations++import ast+import inspect+from pathlib import Path++import pytest+import torch++import core.inference.diffusion as diffusion+import core.inference.diffusion_memory as dm+import core.inference.diffusion_prequant as prequant+from core.inference.diffusion_memory import (+ DeviceMemory,+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,+ plan_diffusion_memory,+ torchao_streaming_plan,+)++BACKEND = Path(__file__).resolve().parents[1]+DIFFUSION_SRC = (BACKEND / "core/inference/diffusion.py").read_text()+MIB = 1024 * 1024+SIZES_AUTO = dict(model_dense_mib = 19630, companion_dense_mib = 12182, text_encoder_dense_mib = 10847)+SIZES_EXPLICIT_INT8 = dict(+ model_dense_mib = 31566, companion_dense_mib = 18024, text_encoder_dense_mib = 16689+)+++class _Target:+ supports_model_cpu_offload = True+++def _plan(+ free,+ total,+ mode = None,+ sizes = SIZES_AUTO,+):+ return plan_diffusion_memory(+ target = _Target(),+ device_memory = DeviceMemory("cuda", "cuda", "discrete_vram", free, total),+ runtime_headroom_mib = 8192,+ requested_mode = mode,+ **sizes,+ )+++def _call(name):+ calls = [+ n+ for n in ast.walk(ast.parse(DIFFUSION_SRC))+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == name+ ]+ assert len(calls) == 1, f"expected one {name}() call in diffusion.py"+ return calls[0]+++# ---------------------------------------------------------------- 1. seed placement+++class _FakeDiT(torch.nn.Module):+ placed: list = []++ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4)++ @classmethod+ def from_config(cls, config):+ return cls()++ def to(self, *args, **kwargs): # record, never move (CPU-only test host)+ _FakeDiT.placed.append(str(args[0] if args else kwargs.get("device")))+ return self+++def fake_checkpoint(monkeypatch):+ _FakeDiT.placed = []+ state = {k: v.clone() for k, v in torch.nn.Linear(4, 4).state_dict().items()}+ state = {f"proj.{k}": v for k, v in state.items()}+ monkeypatch.setattr(prequant, "_resolve_checkpoint_path", lambda *a, **k: "fake.safetensors")+ monkeypatch.setattr(+ prequant, "_load_prequant_checkpoint", lambda *a, **k: {"state_dict": state, "metadata": {}}+ )+ monkeypatch.setattr(prequant, "_validate_checkpoint", lambda *a, **k: True)+ monkeypatch.setattr(prequant, "_verify_packed_fingerprint", lambda *a, **k: True)+ monkeypatch.setattr(prequant, "_pin_kernel_preference", lambda *a, **k: 0)+ monkeypatch.setattr(prequant, "_load_transformer_config", lambda *a, **k: {})+++class _Source:+ kind = "repo"+ location = "unsloth/Qwen-Image-2.1-FP8"+ filename = "Qwen-Image-2.1-INT8.safetensors"+++def _seed_through_call_site(+ plan,+ device = "cuda",+ scheme = "int8",+):+ """Evaluate every keyword of diffusion.py's seed call, then run the loader with the ones it accepts."""+ call = _call("denoiser_prequant_pipe_kwargs")+ scope = dict(vars(diffusion))+ scope.update(plan = plan, device = device, pipeline_seed_scheme = scheme)+ kwargs = {}+ for kw in call.keywords:+ if kw.arg in ("device", "placement_device"):+ kwargs[kw.arg] = eval(compile(ast.Expression(kw.value), "<seed-call>", "eval"), scope)+ accepted = inspect.signature(prequant.load_prequantized_transformer).parameters+ kwargs = {k: v for k, v in kwargs.items() if k in accepted}+ module = prequant.load_prequantized_transformer(+ _FakeDiT, "Qwen/Qwen-Image-2.1", _Source(), dtype = torch.bfloat16, scheme = scheme, **kwargs+ )+ assert module is not None, "fake checkpoint load failed"+ return _FakeDiT.placed[-1]+++ "free,total", [(7788, 8188), (11888, 12288), (15976, 16376)], ids = ["8GB", "12GB", "16GB"]+)+def test_offloading_plan_materialises_the_seed_on_the_host(fake_checkpoint, free, total):+ plan = _plan(free, total)+ if plan.offload_policy == OFFLOAD_MODEL:+ plan = torchao_streaming_plan(plan) # what torchao_offload_plan makes of it for int8+ assert not dm.plan_keeps_transformer_resident(plan)+ assert _seed_through_call_site(plan) == "cpu"+++def test_resident_plan_keeps_the_seed_on_the_device(fake_checkpoint):+ plan = _plan(24176, 24576) # transformer resident, encoders streamed+ assert dm.plan_keeps_transformer_resident(plan)+ assert _seed_through_call_site(plan) == "cuda"+++# ---------------------------------------------------------------- 2. balanced refinement+++def _meta(mib):+ m = torch.nn.Module()+ m.w = torch.nn.Parameter(+ torch.empty(mib * MIB, dtype = torch.uint8, device = "meta"), requires_grad = False+ )+ return m+++class _Pipe:+ def __init__(self):+ self.transformer = _meta(7448)+ self.text_encoder = _meta(8960) # fp8 Qwen3-VL as loaded+ self.vae = _meta(645)+ self.components = {+ "transformer": self.transformer,+ "text_encoder": self.text_encoder,+ "vae": self.vae,+ }+++def _refine_as_loaded(pipe, plan):+ """The expression diffusion.py assigns to ``refined_plan`` after the load, evaluated as written."""+ for node in ast.walk(ast.parse(DIFFUSION_SRC)):+ if (+ isinstance(node, ast.Assign)+ and len(node.targets) == 1+ and getattr(node.targets[0], "id", None) == "refined_plan"+ ):+ scope = dict(vars(diffusion))+ scope.update(pipe = pipe, plan = plan)+ return eval(compile(ast.Expression(node.value), "<refine>", "eval"), scope)+ raise AssertionError("refined_plan assignment not found in diffusion.py")+++def no_torchao(monkeypatch):+ monkeypatch.setattr(dm, "_pipe_denoisers_hold_torchao", lambda pipe: False)++[email protected]("sizes", [SIZES_AUTO, SIZES_EXPLICIT_INT8], ids = ["auto_quant", "int8"])+def test_balanced_16gb_does_not_keep_the_9gib_encoder_resident(no_torchao, sizes):+ plan = _plan(15976, 16376, "balanced", sizes)+ assert (+ plan.offload_policy == OFFLOAD_GROUP and not plan.stream_text_encoders+ ) # the plan main ships+ final = _refine_as_loaded(_Pipe(), plan)+ encoder_resident = final.offload_policy == OFFLOAD_GROUP and not final.stream_text_encoders+ assert not encoder_resident++[email protected]("free,total", [(11888, 12288), (7788, 8188)], ids = ["12GB", "8GB"])+def test_balanced_small_cards_leave_group_offload(no_torchao, free, total):+ final = _refine_as_loaded(_Pipe(), _plan(free, total, "balanced", SIZES_EXPLICIT_INT8))+ assert final.offload_policy != OFFLOAD_GROUP+++def test_balanced_24gb_keeps_its_placement(no_torchao):+ plan = _plan(24176, 24576, "balanced", SIZES_EXPLICIT_INT8)+ final = _refine_as_loaded(_Pipe(), plan)+ assert final.offload_policy == OFFLOAD_GROUP and not final.stream_text_encodersdiff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.pyindex 75ef97e1032..eacd9ef23f0 100644--- a/studio/backend/tests/test_image_gallery.py+++ b/studio/backend/tests/test_image_gallery.py@@ -540,3 +540,55 @@ def test_thumbnail_is_a_downscaled_webp_of_the_png():with Image.open(io.BytesIO(data)) as im:assert im.format == "WEBP"assert im.size == (16, 8)+++def _noisy(size = 128):+ # Incompressible-ish content so the deflate level changes the encoded bytes.+ import random+ rng = random.Random(1234)+ return Image.frombytes(+ "RGB", (size, size), bytes(rng.randrange(256) for _ in range(size * size * 3))+ )+++def _encode(image, level):+ from PIL.PngImagePlugin import PngInfo++ info = PngInfo()+ info.add_text("unsloth", _json.dumps(_meta()))+ info.add_text("parameters", gallery._params_text(_meta()))+ buf = io.BytesIO()+ image.save(buf, format = "PNG", pnginfo = info, compress_level = level)+ return buf.getvalue()+++def test_gallery_png_uses_fast_deflate_by_default(monkeypatch):+ # The gallery encode is on the request path after the last denoise step; the default level must be the fast one.+ monkeypatch.delenv(gallery.PNG_COMPRESS_LEVEL_ENV, raising = False)+ image = _noisy()+ data = gallery._png_bytes(image, _meta())+ assert data == _encode(image, 1)+ assert data != _encode(image, 6)+++def test_gallery_png_level_kill_switch_restores_pillow_default(monkeypatch):+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, "6")+ image = _noisy()+ assert gallery._png_bytes(image, _meta()) == _encode(image, 6)++[email protected]("raw", ["", "fast", "-1", "10"])+def test_gallery_png_level_rejects_invalid_values(monkeypatch, raw):+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, raw)+ assert gallery.png_compress_level() == 1++[email protected]("level", ["0", "1", "6", "9"])+def test_gallery_png_is_lossless_at_any_level(monkeypatch, level):+ monkeypatch.setenv(gallery.PNG_COMPRESS_LEVEL_ENV, level)+ image = _noisy(64)+ record = gallery.save(image, _meta())+ raw = base64.b64decode(gallery.image_b64(record["id"]))+ with Image.open(io.BytesIO(raw)) as im:+ assert im.convert("RGB").tobytes() == image.tobytes()+ assert _json.loads(im.text["unsloth"])["seed"] == 7