Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio H3-Image: prompt cache, pinned streaming and the H3 attention levers (#17495)
Daniel Han5d5c1381b01c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex f65abb51e16..fbde3601429 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -11815,6 +11815,17 @@ class DiffusionBackend:)report = pipe._unsloth_h3_reportplan = report.plan+ prompt_cache.install(+ pipe,+ identity = {+ "family": fam.name,+ "repo": str(repo_id),+ "base": str(base or repo_id),+ "dtype": str(target.dtype),+ "te_quant": str(report.te_scheme),+ },+ logger = logger,+ )policy = (OFFLOAD_NONEif plan.placement == PLACE_RESIDENTdiff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex aef204ecbd0..07648c24d6d 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -4058,3 +4058,13 @@ def install_hunyuan_attention_trim(if engaged and logger is not None:logger.info("diffusion.attention: hunyuan padded-text trim engaged")return engaged+++# declared checkpoint attention -> Studio backend of the same arithmetic (still probe-gated)+CHECKPOINT_ATTENTION_BACKENDS = {"int8": "sage"}+++def checkpoint_attention_backend(declared: Any) -> Optional[str]:+ if not isinstance(declared, str):+ return None+ return CHECKPOINT_ATTENTION_BACKENDS.get(declared.strip().lower())diff --git a/studio/backend/core/inference/diffusion_minimax_h3_image.py b/studio/backend/core/inference/diffusion_minimax_h3_image.pyindex 50d20d74700..5dda42dbbe5 100644--- a/studio/backend/core/inference/diffusion_minimax_h3_image.py+++ b/studio/backend/core/inference/diffusion_minimax_h3_image.py@@ -113,6 +113,8 @@ ADALN_TABLE_ENV = "UNSLOTH_H3_IMAGE_ADALN_TABLE"PLACEMENT_ENV = "UNSLOTH_H3_IMAGE_PLACEMENT"VRAM_BUDGET_ENV = "UNSLOTH_H3_IMAGE_VRAM_GB"HOST_BUDGET_ENV = "UNSLOTH_H3_IMAGE_HOST_GB"+# "0": stream from pageable memory (ModuleListStreamer) instead of the pinned prefetch streamer+PINNED_STREAM_ENV = "UNSLOTH_H3_IMAGE_PINNED_STREAM"QUANT_SCHEMES = ("int8", "fp8")# ConvRot group: the one the H3 video denoiser and its hosted encoder use (divides 5376, 7168 and 14336).@@ -785,6 +787,8 @@ class H3ImagePlan:# False: the encoder is loaded for each encode and freed after (host RAM cannot hold it beside the DiT)te_kept_on_host: bool = Truereasons: list = field(default_factory = list)+ # host GB held once placed (host DiT blocks + encoder + 4 GB); None without a host budget+ host_need_gb: Optional[float] = Nonedef as_dict(self) -> dict:return {@@ -801,6 +805,17 @@ class H3ImagePlan:}+def pinned_stream_enabled() -> bool:+ return not _env_off(PINNED_STREAM_ENV)+++def stream_inflight() -> int:+ if not pinned_stream_enabled():+ return 1+ from .diffusion_offload_prefetch import LIST_STREAM_DEPTH+ return 1 + LIST_STREAM_DEPTH++def activation_gb(width: int,height: int,@@ -842,6 +857,7 @@ def plan_placement(num_blocks: int = 50,te_layers: int = 50,forced: Optional[str] = None,+ inflight: int = 1,) -> H3ImagePlan:"""Pure: VRAM and host budgets (GB, None = unknown/CPU) -> placement, best first:@@ -851,7 +867,8 @@ def plan_placement(- sequential: one big component on the card at a time (a DiT larger than the encoder room);- stream: DiT blocks stream past the ones that fit, the encoder streams every layer (L4 24 GB).- Host: the encoder is kept only when it fits beside the DiT, else it loads per prompt and is freed after."""+ Host: the encoder is kept only when it fits beside the DiT, else it loads per prompt and is freed after.+ ``inflight``: streamed members on the device at once (2 with the prefetch streamer: one runs, one copies)."""reasons = []act = activation_gb(width, height, references)margin = 1.5@@ -870,6 +887,7 @@ def plan_placement(return planlayer_gb = max(te_gb - TE_TOP_GB, 0.1) / max(1, te_layers)block_gb = max(dit_gb - DIT_TOP_GB, 0.1) / max(1, num_blocks)+ inflight = max(1, int(inflight))if forced in PLACEMENTS:choice = forcedreasons.append(f"placement forced to {forced} by {PLACEMENT_ENV}")@@ -878,7 +896,7 @@ def plan_placement(reasons.append(f"DiT + encoder + VAE ({dit_gb + te_gb + vae_gb:.1f} GB) fit {vram_gb:.1f} GB")- elif dit_gb + vae_gb + act + margin + TE_TOP_GB + layer_gb <= vram_gb:+ elif dit_gb + vae_gb + act + margin + TE_TOP_GB + layer_gb * inflight <= vram_gb:choice = PLACE_DIT_RESIDENTreasons.append(f"DiT + VAE ({dit_gb + vae_gb:.1f} GB) stay on {vram_gb:.1f} GB; the encoder streams"@@ -893,11 +911,12 @@ def plan_placement(reasons.append(f"DiT {dit_gb:.1f} GB + working set exceeds {vram_gb:.1f} GB: blocks stream")plan.placement = choiceif choice == PLACE_DIT_RESIDENT:- spare = vram_gb - dit_gb - vae_gb - act - margin - TE_TOP_GB - layer_gb+ spare = vram_gb - dit_gb - vae_gb - act - margin - TE_TOP_GB - layer_gb * inflightplan.te_resident_layers = max(0, min(te_layers, int(spare / layer_gb)))reasons.append(f"{plan.te_resident_layers}/{te_layers} encoder layers stay on the device")elif choice == PLACE_STREAM:spare = vram_gb - act - margin - vae_gb - DIT_TOP_GB - TE_TOP_GB - block_gb+ spare -= (inflight - 1) * max(block_gb, layer_gb)plan.dit_resident_blocks = max(0, min(num_blocks, int(spare / block_gb)))# The resident blocks hold their room through the encode, so every encoder layer streams.plan.te_resident_layers = 0@@ -912,6 +931,7 @@ def plan_placement(PLACE_STREAM: block_gb * (num_blocks - int(plan.dit_resident_blocks or 0)),}[choice]need = dit_host + te_gb + 4.0+ plan.host_need_gb = needplan.te_kept_on_host = host_gb >= need# Loading per prompt only frees the encoder between prompts: each load still holds it whole beside the# host-side DiT blocks, so a host below that peak would be OOM-killed on the first prompt.@@ -1138,8 +1158,13 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):"MiniMax-H3-Image editing needs the Qwen3-VL processor (processor/ of the repo).")- def _encode(self, prompt: str, vision_images: Optional[list]) -> tuple:- """(prompt_embeds (1, n, 5120), per-row text tags) for one presentation."""+ def encode_prompt(+ self,+ prompt: str,+ vision_images: Optional[list] = None,+ ) -> tuple:+ """(prompt_embeds, per-row text tags). Named like diffusers' encoder entry point so ``diffusion_prompt_cache``+ wraps it; VAE reference latents are never cached (seeded sample)."""import torchtokenizer = self.tokenizer@@ -1184,6 +1209,23 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):self._leave("text_encoder")return embeds, torch.tensor(tags, dtype = torch.long)+ def _encode(self, prompt: str, vision_images: Optional[list]) -> tuple:+ return self.encode_prompt(prompt, vision_images)++ def _streamer_stats(self) -> dict:+ """Per role: pinned prefetch counters, or {"pageable": n}."""+ out = {}+ for streamer in self._unsloth_h3_streamers:+ role = getattr(streamer, "_unsloth_role", "?")+ read = getattr(streamer, "stats", None)+ out[role] = read() if callable(read) else {"pageable": int(streamer.streamed)}+ return out++ def _prompt_cache_hits(self) -> int:+ cache = getattr(self, "_unsloth_prompt_cache", None)+ stats = getattr(cache, "stats", None)+ return int(stats.get("hits", 0)) if isinstance(stats, dict) else 0+def _encode_references(self, images: list, generator: Any) -> list:import numpy as npimport torch@@ -1332,8 +1374,10 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):device = self._device()canvas_h, canvas_w = canvas_size(height, width, min_generation_area)stats["canvas"] = (canvas_w, canvas_h)- embeds, text_tags = self._encode(prompt, vision_refs)+ hits = self._prompt_cache_hits()+ embeds, text_tags = self.encode_prompt(prompt, vision_refs)stats["encode_s"] = time.perf_counter() - t0+ stats["encode_cached"] = self._prompt_cache_hits() > hitspatch = tuple(self.transformer.config.patch_size)channels = int(self.vae.config.latent_channels) if self.vae is not None else 24lh, lw = canvas_h // H3_IMAGE_VAE_SCALE, canvas_w // H3_IMAGE_VAE_SCALE@@ -1416,6 +1460,7 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):images = self._decode(final, height, width, output_type)stats["decode_s"] = time.perf_counter() - t2stats["total_s"] = time.perf_counter() - t0+ stats["streamers"] = self._streamer_stats()self._unsloth_h3_last_stats = statsreturn images@@ -1879,6 +1924,10 @@ def h3_image_prepare_prequant_model(logger_: Any = None) -> Callable:def _prepare(transformer: Any, metadata: Optional[dict]) -> None:import torch+ declared = metadata.get("attention") if isinstance(metadata, dict) else None+ if isinstance(declared, str) and declared.strip():+ setattr(transformer, CHECKPOINT_ATTENTION_ATTR, declared.strip().lower())+from .video_minimax_h3_adaln import apply_h3_adaln_curve, is_curve_checkpointif is_curve_checkpoint(metadata):@@ -2087,6 +2136,7 @@ class H3ImageLoadReport:load_s: float = 0.0quantized_linears: int = 0bytes_read: int = 0+ speed_levers: tuple = ()def as_dict(self) -> dict:return {@@ -2099,6 +2149,7 @@ class H3ImageLoadReport:"load_s": round(self.load_s, 1),"quantized_linears": self.quantized_linears,"bytes_read": self.bytes_read,+ "speed_levers": list(self.speed_levers),**{f"plan_{k}": v for k, v in self.plan.as_dict().items()},}@@ -2316,21 +2367,209 @@ def load_h3_image_text_encoder(return encoder, None+def pin_room_bytes(plan: H3ImagePlan, pinned: Sequence[Any] = ()) -> Optional[int]:+ """Host bytes left for pins: host budget less what the plan holds less ``pinned`` streamers' locks. Pins count on+ top of the pageable copies they replace (those may stay cached). None without a host budget."""+ if plan.host_gb is None or plan.host_need_gb is None:+ return None+ room = int((float(plan.host_gb) - float(plan.host_need_gb)) * 1e9)+ room -= sum(int(getattr(s, "pinned_bytes", 0) or 0) for s in pinned if s is not None)+ return max(0, room)+++def make_list_streamer(+ owner: Any,+ members: Any,+ device: Any,+ resident: int,+ role: str,+ *,+ pinned: bool = True,+ reserve_bytes: int = 0,+ max_pin_bytes: Optional[int] = None,+) -> Any:+ """A streamer for ``members[resident:]``: the pinned prefetch one where it fits, else ``ModuleListStreamer``."""+ streamer = None+ if pinned and pinned_stream_enabled():+ from .diffusion_offload_prefetch import stream_module_list+ streamer = stream_module_list(+ owner,+ members,+ device,+ resident,+ logger = logger,+ reserve_bytes = reserve_bytes,+ max_pin_bytes = max_pin_bytes,+ )+ if streamer is None:+ streamer = ModuleListStreamer(members, device, resident)+ streamer._unsloth_role = role+ return streamer+++def stream_list(+ pipe: Any,+ owner: Any,+ members: Any,+ device: Any,+ resident: int,+ role: str,+ *,+ pinned: bool = True,+ reserve_bytes: int = 0,+ max_pin_bytes: Optional[int] = None,+) -> Any:+ """``make_list_streamer``, registered on the pipe by role."""+ streamer = make_list_streamer(+ owner,+ members,+ device,+ resident,+ role,+ pinned = pinned,+ reserve_bytes = reserve_bytes,+ max_pin_bytes = max_pin_bytes,+ )+ pipe._unsloth_h3_streamers.append(streamer)+ return streamer++def _place_streamed_dit(- transformer: Any, vae: Any, device: Any, plan: H3ImagePlan-) -> ModuleListStreamer:+ transformer: Any,+ vae: Any,+ device: Any,+ plan: H3ImagePlan,+ pinned_already: Sequence[Any] = (),+) -> Any:+ """Placed before the encoder so the DiT blocks (streamed every step) get the pin budget first."""vae.to(device)for name, child in transformer.named_children():if name != "transformer_blocks":child.to(device)resident = int(plan.dit_resident_blocks or 0)- streamer = ModuleListStreamer(transformer.transformer_blocks, device, resident)- streamer._unsloth_role = "transformer"+ streamer = make_list_streamer(+ transformer,+ transformer.transformer_blocks,+ device,+ resident,+ "transformer",+ max_pin_bytes = pin_room_bytes(plan, pinned_already),+ )streamer._unsloth_resident = residentreturn streamer-def _place_text_encoder(pipe: Any, encoder: Any, plan: H3ImagePlan, device: Any) -> Any:+# "0": native attention; "1": probe-gated SageAttention on the DiT blocks; unset: as the checkpoint declares+INT8_ATTENTION_ENV = "UNSLOTH_H3_IMAGE_INT8_ATTENTION"+CHECKPOINT_ATTENTION_ATTR = "_unsloth_checkpoint_attention"+# the INT8 DiT checkpoint runs INT8 attention on its DiT blocks unless its file says otherwise+SCHEME_ATTENTION = {"int8": "int8"}+ATTENTION_SAVED_ATTR = "_unsloth_h3_attention_saved"+_LOG = logger+++def checkpoint_attention_wanted(transformer: Any, scheme: Optional[str]) -> Optional[str]:+ """The DiT blocks' attention backend ("sage" or None) per the env switch or checkpoint."""+ from .diffusion_attention import checkpoint_attention_backend++ raw = (os.environ.get(INT8_ATTENTION_ENV) or "").strip().lower()+ if raw in ("0", "off", "false", "no"):+ return None+ if raw in ("1", "on", "true", "yes"):+ return "sage"+ declared = getattr(transformer, CHECKPOINT_ATTENTION_ATTR, None)+ if not declared:+ declared = SCHEME_ATTENTION.get(scheme or "")+ return checkpoint_attention_backend(declared)+++def _attention_processors(module: Any) -> list:+ return [+ m for m in module.modules() if hasattr(getattr(m, "processor", None), "_attention_backend")+ ]+++def install_checkpoint_attention(+ pipe: Any,+ device: Any,+ *,+ scheme: Optional[str] = None,+ logger: Any = None,+) -> Optional[str]:+ """Route the DiT blocks to the declared backend (SageAttention must pass its probe here first); the token refiner+ keeps its own. Returns the engaged backend, or None with every processor unchanged."""+ import torch++ log = logger if logger is not None else _LOG+ transformer = getattr(pipe, "transformer", None)+ if transformer is None or getattr(pipe, ATTENTION_SAVED_ATTR, None):+ return None+ want = checkpoint_attention_wanted(transformer, scheme)+ dev = torch.device(device)+ if want is None or dev.type != "cuda":+ return None+ from .diffusion_attention import ATTENTION_BACKEND_ATTR, apply_attention_backend++ modules = _attention_processors(transformer)+ saved = {m: m.processor._attention_backend for m in modules}+ tag = getattr(transformer, ATTENTION_BACKEND_ATTR, None)++ def restore(only: Any = None) -> None:+ for m, backend in saved.items():+ if only is None or m in only:+ m.processor._attention_backend = backend++ dtype = _module_dtype(transformer, skip_fp32 = True)+ target = types.SimpleNamespace(device = dev.type, dtype = dtype, torch_device = str(dev))+ try:+ engaged = apply_attention_backend(pipe, want, logger = log, target = target)+ except Exception as exc: # noqa: BLE001 - native attention stays+ engaged = None+ log.warning("diffusion.minimax_h3_image: INT8 attention unavailable (%s)", exc)+ if not engaged or "sage" not in str(engaged):+ restore()+ setattr(transformer, ATTENTION_BACKEND_ATTR, tag)+ return None+ blocks = {id(m) for b in getattr(transformer, "transformer_blocks", ()) for m in b.modules()}+ restore(only = [m for m in modules if id(m) not in blocks])+ setattr(pipe, ATTENTION_SAVED_ATTR, (saved, tag))+ log.info(+ "diffusion.minimax_h3_image: INT8 attention (%s) on %d DiT blocks, as the checkpoint declares",+ engaged,+ sum(1 for m in modules if id(m) in blocks),+ )+ return engaged+++def remove_checkpoint_attention(pipe: Any) -> int:+ """Every attention processor back to the backend it had before ``install_checkpoint_attention``."""+ entry = getattr(pipe, ATTENTION_SAVED_ATTR, None)+ if not entry:+ return 0+ from .diffusion_attention import ATTENTION_BACKEND_ATTR++ saved, tag = entry+ for m, backend in saved.items():+ m.processor._attention_backend = backend+ transformer = getattr(pipe, "transformer", None)+ if transformer is not None:+ setattr(transformer, ATTENTION_BACKEND_ATTR, tag)+ setattr(pipe, ATTENTION_SAVED_ATTR, None)+ return len(saved)+++def _place_text_encoder(+ pipe: Any,+ encoder: Any,+ plan: H3ImagePlan,+ device: Any,+ *,+ per_prompt: bool = False,+ reserve_bytes: int = 0,+ pinned_already: Sequence[Any] = (),+) -> Any:+ """``per_prompt`` (low-host tier, encoder reloaded each prompt) streams pageable: pinning would copy 27 GB per+ prompt. ``pinned_already``: streamers whose pins count against the host room first."""if encoder is None:return Noneif plan.placement == PLACE_RESIDENT:@@ -2343,9 +2582,17 @@ def _place_text_encoder(pipe: Any, encoder: Any, plan: H3ImagePlan, device: Any)for name, child in encoder.model.language_model.named_children():if name != "layers":child.to(device)- streamer = ModuleListStreamer(layers, device, int(plan.te_resident_layers or 0))- streamer._unsloth_role = "text_encoder"- pipe._unsloth_h3_streamers.append(streamer)+ stream_list(+ pipe,+ encoder.model.language_model,+ layers,+ device,+ int(plan.te_resident_layers or 0),+ "text_encoder",+ pinned = not per_prompt,+ reserve_bytes = reserve_bytes,+ max_pin_bytes = pin_room_bytes(plan, pinned_already),+ )return encoder@@ -2420,6 +2667,7 @@ def load_minimax_h3_image_pipeline("width": plan_width,"height": plan_height,"forced": (os.environ.get(PLACEMENT_ENV) or "").strip().lower() or None,+ "inflight": stream_inflight() if device.type == "cuda" else 1,}plan = plan_placement(te_gb = te_gb_for(te_scheme) if text_encoder is None else _module_gb(text_encoder),@@ -2576,11 +2824,11 @@ def load_minimax_h3_image_pipeline("for each prompt because host memory is short). Check the network or the Hugging Face cache ""and load the model again.")- return _place_text_encoder(pipe, encoder, plan, device)+ return _place_text_encoder(pipe, encoder, plan, device, per_prompt = True)pipe._unsloth_h3_te_loader = _te_loaderelse:- _place_text_encoder(pipe, text_encoder, plan, device)+ _place_text_encoder(pipe, text_encoder, plan, device, pinned_already = [early_streamer])if plan.placement != PLACE_SEQUENTIAL:vae.to(device)if plan.placement in (PLACE_RESIDENT, PLACE_DIT_RESIDENT):@@ -2592,11 +2840,19 @@ def load_minimax_h3_image_pipeline(if early_streamer is None or early_streamer._unsloth_resident != resident:if early_streamer is not None:early_streamer.remove()- early_streamer = _place_streamed_dit(transformer, vae, device, plan)+ early_streamer = _place_streamed_dit(+ transformer, vae, device, plan, pinned_already = pipe._unsloth_h3_streamers+ )pipe._unsloth_h3_streamers.append(early_streamer)early_streamer = Noneif early_streamer is not None:early_streamer.remove()+ from .diffusion_minimax_h3_image_speed import install_h3_image_speed++ speed_levers = install_h3_image_speed(pipe, plan, device, logger)+ int8_attention = install_checkpoint_attention(pipe, device, scheme = scheme, logger = logger)+ if int8_attention:+ speed_levers = tuple(speed_levers) + (f"checkpoint_attention={int8_attention}",)report = H3ImageLoadReport(scheme = scheme,scheme_reason = scheme_reason,@@ -2608,6 +2864,7 @@ def load_minimax_h3_image_pipeline(load_s = time.perf_counter() - t0,quantized_linears = int(stats.get("quantized_linears", 0)),bytes_read = int(stats.get("bytes_read", 0)),+ speed_levers = speed_levers,)pipe._unsloth_h3_report = reporttry:diff --git a/studio/backend/core/inference/diffusion_minimax_h3_image_speed.py b/studio/backend/core/inference/diffusion_minimax_h3_image_speed.pynew file mode 100644index 00000000000..a2d44e707f2--- /dev/null+++ b/studio/backend/core/inference/diffusion_minimax_h3_image_speed.py@@ -0,0 +1,182 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3-Image speed levers, reusing the MiniMax-H3 video ones (same attention layout).+``UNSLOTH_H3_IMAGE_SPEED=0`` disables all; each also honours its video switch.++Exact: h3_int8_gemm, h3_fused_qkv, h3_strided_attn, h3_shared_qkv_quant, the _native_flash pin on sm80 / sm89.+Tolerance-level: h3_qk_rope (last-bit rstd drift changes the 6-step image, so off by default on the streamed tier;+UNSLOTH_H3_IMAGE_QK_ROPE forces it), h3_fused_swiglu, h3_compiled (opt-in via UNSLOTH_H3_IMAGE_COMPILE=1: workspace+unmeasured beside the 37.5 GB A100-40 dit_resident peak). The streamed tier gets attention levers only: its blocks+move per forward, so resident-projection levers cannot engage."""++from __future__ import annotations++import os+import types+from typing import Any, Optional++SPEED_ENV = "UNSLOTH_H3_IMAGE_SPEED"+COMPILE_ENV = "UNSLOTH_H3_IMAGE_COMPILE"+QK_ROPE_ENV = "UNSLOTH_H3_IMAGE_QK_ROPE"+LEVERS_ATTR = "_unsloth_h3_speed_levers"+_RESIDENT_PLACEMENTS = ("resident", "dit_resident")+++def speed_enabled() -> bool:+ return (os.environ.get(SPEED_ENV) or "1").strip().lower() not in ("0", "off", "false", "no")+++def compile_enabled() -> bool:+ return (os.environ.get(COMPILE_ENV) or "").strip().lower() in ("1", "on", "true", "yes")+++def image_qk_rope_enabled(resident: bool = False) -> bool:+ """The fused q/k norm + rope for H3-Image: forced by UNSLOTH_H3_IMAGE_QK_ROPE, else on for the resident tiers."""+ value = (os.environ.get(QK_ROPE_ENV) or "").strip().lower()+ if value in ("1", "on", "true", "yes"):+ return True+ if value in ("0", "off", "false", "no"):+ return False+ return bool(resident)+++def _row_pads(transformer: Any) -> list:+ out = []+ for parent in transformer.modules():+ for leaf, child in list(parent._modules.items()):+ if type(child).__name__ == "H3ImageRowPad" and "inner" in child._modules:+ out.append((parent, leaf, child))+ return out+++def _unwrap(pads: list, keep: Any = None) -> int:+ n = 0+ for parent, leaf, wrapper in pads:+ inner = wrapper._modules["inner"]+ if keep is not None and not keep(inner):+ continue+ setattr(parent, leaf, inner)+ n += 1+ return n+++def _fuse_qkv(transformer: Any, logger: Any) -> int:+ """fuse_h3_qkv_ needs bare projections: unwrap, fuse, re-pad so the int8 path keeps its row multiple."""+ from .diffusion_minimax_h3_image import wrap_row_multiple+ from .video_minimax_h3_attn import fuse_h3_qkv_++ pads = [p for p in _row_pads(transformer) if p[1] in ("to_q", "to_k", "to_v")]+ _unwrap(pads)+ try:+ fused = fuse_h3_qkv_(transformer, logger = logger)+ finally:+ for parent, leaf, wrapper in pads:+ if leaf in parent._modules: # left unfused+ setattr(parent, leaf, wrapper)+ for parent, _leaf, _wrapper in pads:+ if getattr(parent, "fused_projections", False) and type(+ parent._modules.get("to_qkv")+ ).__name__ != ("H3ImageRowPad"):+ wrap_row_multiple(parent, ["to_qkv"])+ return fused+++def _fused_gemm_serves(inner: Any) -> bool:+ from .diffusion_int8_gemm import _REC+ rec = inner.__dict__.get(_REC)+ return bool(rec is not None and rec[4])+++def install_h3_image_speed(+ pipe: Any,+ plan: Any,+ device: Any,+ logger: Any = None,+) -> tuple:+ """Engage the levers after placement, before the first forward; returns the engaged names, never raises."""+ levers: list = []+ try:+ levers = _install(pipe, plan, device, logger)+ except Exception as exc: # noqa: BLE001 - an optimisation: the stock path stays+ if logger is not None:+ logger.warning("diffusion.minimax_h3_image: speed levers skipped: %s", exc)+ out = tuple(levers)+ try:+ setattr(pipe, LEVERS_ATTR, out)+ except Exception: # noqa: BLE001+ pass+ if logger is not None and out:+ logger.info("diffusion.minimax_h3_image: speed levers %s", ",".join(out))+ return out+++def _install(pipe: Any, plan: Any, device: Any, logger: Any) -> list:+ import torch++ transformer = getattr(pipe, "transformer", None)+ if transformer is None or not speed_enabled():+ return []+ dev = torch.device(device)+ if dev.type != "cuda" or not torch.cuda.is_available():+ return []+ levers: list = []+ resident = getattr(plan, "placement", None) in _RESIDENT_PLACEMENTS+ if resident and _fuse_qkv(transformer, logger):+ levers.append("h3_fused_qkv")+ if resident:+ from . import diffusion_int8_gemm as int8_gemm+ if int8_gemm.install(transformer, logger):+ _unwrap(_row_pads(transformer), keep = _fused_gemm_serves)+ levers.append("h3_int8_gemm")+ from .diffusion_attention import apply_attention_backend, select_attention_backend+ from .video_minimax_h3_attn import (+ h3_attention_backend,+ install_strided_attention,+ shared_qkv_count,+ )++ from .diffusion_minimax_h3_image import _module_dtype++ # the block stack's dtype: the fp32 embedders and heads come first in the parameter order+ dtype = _module_dtype(transformer, skip_fp32 = True)+ target = types.SimpleNamespace(device = dev.type, dtype = dtype, torch_device = str(dev))+ backend = select_attention_backend(target, None, speed_active = True)+ try:+ backend = h3_attention_backend(backend, requested = None, target = target)+ except Exception: # noqa: BLE001 - keep the generic pick+ pass+ engaged = apply_attention_backend(pipe, backend, logger = logger, target = target)+ if engaged:+ levers.append(f"attention={engaged}")+ if engaged in ("_native_cudnn", "_native_flash") and install_strided_attention(+ transformer, logger = logger, qk_rope = image_qk_rope_enabled(resident)+ ):+ levers.append("h3_strided_attn")+ from .video_minimax_h3_qknorm import qk_rope_enabled++ procs = [getattr(m, "processor", None) for m in transformer.modules()]+ if (+ image_qk_rope_enabled(resident)+ and qk_rope_enabled()+ and any(getattr(p, "_unsloth_qk_rope", False) for p in procs)+ ):+ levers.append("h3_qk_rope")+ if shared_qkv_count(transformer):+ levers.append("h3_shared_qkv_quant")+ if "h3_int8_gemm" in levers:+ from .video_minimax_h3_ffn import install as install_ffn+ if install_ffn(transformer, device = dev, logger = logger):+ levers.append("h3_fused_swiglu")+ if resident and compile_enabled():+ compiled = 0+ for block in getattr(transformer, "transformer_blocks", ()):+ block.compile()+ compiled += 1+ if compiled:+ levers.append("h3_compiled")+ return levers+++def speed_levers(pipe: Any) -> Optional[tuple]:+ return getattr(pipe, LEVERS_ATTR, None)diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.pyindex 80c87dc5160..e880684ef83 100644--- a/studio/backend/core/inference/diffusion_offload_prefetch.py+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py@@ -1346,3 +1346,233 @@ def cross_forward_step(pf.drop_carried() # the next step runs another expert: these copies would not be readelif nxt2 is not pf:pf.armed = False # the expert switches after the next step: carry nothing past it+++LIST_STREAM_DEPTH = 1+++class PrefetchedListStreamer:+ """Streams ``blocks[resident:]`` via pinned slabs and event-fenced copies ``depth`` members ahead, from the second+ forward on (the first records the order); a forward that raises still releases its queued copies."""++ def __init__(self) -> None:+ self.handles: list = []+ self.groups: list = []+ self.prefetcher: Optional[GroupPrefetcher] = None+ self.streamed = 0+ self.pinned_bytes = 0+ self.arena: Any = None+ # matches ModuleListStreamer, whose release path clears ``_host``+ self._host: dict = {}++ def stats(self) -> dict:+ pf = self.prefetcher+ return dict(pf.stats) if pf is not None else {}++ def remove(self) -> None:+ for handle in self.handles:+ handle.remove()+ self.handles = []+ pf = self.prefetcher+ if pf is not None and getattr(pf, "active", False):+ pf.end()+ for group in self.groups:+ for name in ("onload_", "offload_", "_unsloth_prefetch_onload", "_unsloth_prefetcher"):+ group.__dict__.pop(name, None)+ cpu = getattr(group, "cpu_param_dict", None)+ if isinstance(cpu, dict):+ cpu.clear()+ self.groups = []+ self.prefetcher = None+ self.arena = None+++def _list_stream_refusal(device: Any) -> Optional[str]:+ if not async_prefetch_enabled():+ return f"{ASYNC_PREFETCH_ENV}=0"+ import torch++ from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV++ if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return f"{GROUP_OFFLOAD_PIN_ENV}=0"+ if torch.device(device).type != "cuda" or not torch.cuda.is_available():+ return "not a CUDA device"+ from .host_pinning import host_pinning_allowed++ if not host_pinning_allowed(torch.device(device)):+ return "pinned host memory is not used on this device"+ from .diffusion_memory import _pinned_memory_capped++ if _pinned_memory_capped():+ return "pinned host memory is capped on this OS"+ return None+++def stream_module_list(+ owner: Any,+ blocks: Any,+ device: Any,+ resident: int,+ *,+ logger: Any = None,+ depth: int = LIST_STREAM_DEPTH,+ reserve_bytes: int = 0,+ max_pin_bytes: Optional[int] = None,+) -> Optional[PrefetchedListStreamer]:+ """A ``PrefetchedListStreamer`` over ``blocks``, or None (nothing changed) when it does not apply or cannot pin.+ Pins must fit ``_pin_budget_mib`` less ``reserve_bytes``, and ``max_pin_bytes`` if given."""+ import torch++ refused = _list_stream_refusal(device)+ blocks = list(blocks)+ resident = max(0, min(len(blocks), int(resident)))+ streamed = blocks[resident:]+ if refused is None and not streamed:+ refused = "nothing streams"+ need = 0+ if refused is None:+ from .diffusion_memory import _pin_budget_mib, _storage_nbytes++ need = sum(+ sum(_storage_nbytes(t))+ for b in streamed+ for t in list(b.parameters()) + list(b.buffers())+ )+ budget = _pin_budget_mib()+ if max_pin_bytes is not None and need > int(max_pin_bytes):+ refused = (+ f"{need / 1e9:.1f} GB to pin exceeds the {max(0, int(max_pin_bytes)) / 1e9:.1f} GB the host plan "+ "leaves for pinned copies"+ )+ elif budget is not None and need + int(reserve_bytes) > int(budget) << 20:+ refused = (+ f"{need / 1e9:.1f} GB to pin (+{int(reserve_bytes) / 1e9:.1f} GB reserved) exceeds the "+ f"{budget / 1024:.1f} GiB host pin budget"+ )+ if refused is None:+ try:+ go = _go()+ group_cls = go.ModuleGroup+ except Exception as exc: # noqa: BLE001 - an older diffusers+ refused = f"diffusers has no ModuleGroup ({type(exc).__name__})"+ if refused is not None:+ if logger is not None:+ logger.info(+ "diffusion.memory: %s keeps its pageable streamer (%s)",+ type(owner).__name__,+ refused,+ )+ return None+ onload = torch.device(device)+ streamer = PrefetchedListStreamer()+ try:+ from .diffusion_memory import install_group_offload_buffer_restore++ # diffusers restores only parameters; buffer weights (int8 encoder) would otherwise stay on the device+ install_group_offload_buffer_restore()+ from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload++ for block in streamed:+ block.to("cpu")+ tensors = [t for b in streamed for t in list(b.parameters()) + list(b.buffers())]+ try:+ expected = packable_bytes(tensors)+ except Exception: # noqa: BLE001 - whole slabs, as without a size+ expected = None+ stream = torch.cuda.Stream(device = onload)+ with pinned_arena_for_group_offload(expected_bytes = expected) as arena:+ for block in streamed:+ group = group_cls(+ modules = [block],+ offload_device = torch.device("cpu"),+ onload_device = onload,+ offload_leader = block,+ onload_leader = block,+ non_blocking = True,+ stream = stream,+ record_stream = True,+ low_cpu_mem_usage = False,+ )+ type(group).offload_(group)+ streamer.groups.append(group)+ streamer.arena = arena+ for block in blocks[:resident]:+ block.to(onload)+ pf = GroupPrefetcher(owner, streamer.groups, onload, depth)+ pf.fence_first = True+ disable = getattr(getattr(torch, "compiler", None), "disable", None)++ def _eager(fn: Any) -> Any:+ return disable(fn) if callable(disable) else fn++ for group, block in zip(streamer.groups, streamed):++ def onload_(+ *_a: Any,+ _g: Any = group,+ **_k: Any,+ ) -> None:+ pf.onload(_g)++ def offload_(+ *_a: Any,+ _g: Any = group,+ **_k: Any,+ ) -> None:+ pf.offload(_g)++ group.onload_ = _eager(onload_)+ group.offload_ = _eager(offload_)+ group._unsloth_prefetch_onload = group.onload_+ group._unsloth_prefetcher = pf+ streamer.handles.append(+ block.register_forward_pre_hook(_eager(lambda *_a, _g = group: _g.onload_()))+ )+ streamer.handles.append(+ block.register_forward_hook(_eager(lambda *_a, _g = group: _g.offload_()))+ )++ def _begin(*_a: Any, **_k: Any) -> None:+ pf.begin()+ pf.kick() # the first streamed members copy while the resident ones run++ streamer.handles.append(owner.register_forward_pre_hook(_eager(_begin)))+ streamer.handles.append(+ owner.register_forward_hook(_eager(lambda *_a, **_k: pf.end()), always_call = True)+ )+ streamer.prefetcher = pf+ streamer.streamed = len(streamed)+ streamer.pinned_bytes = int(need)+ bump_placement_epoch()+ except Exception as exc: # noqa: BLE001 - the caller's pageable streamer takes the members as they are now+ streamer.remove()+ for block in streamed:+ try:+ block.to("cpu")+ except Exception: # noqa: BLE001+ pass+ if logger is not None:+ logger.warning(+ "diffusion.memory: pinned prefetch streaming unavailable for %s (%s)",+ type(owner).__name__,+ exc,+ )+ return None+ if logger is not None:+ payload = getattr(arena, "payload_bytes", 0) if arena is not None else 0+ logger.info(+ "diffusion.memory: %s streams %d of %d members with pinned copies (%.2f GB) and event-fenced prefetch "+ "(%d ahead)",+ type(owner).__name__,+ len(streamed),+ len(blocks),+ payload / 1e9,+ depth,+ )+ return streamerdiff --git a/studio/backend/core/inference/video_minimax_h3_attn.py b/studio/backend/core/inference/video_minimax_h3_attn.pyindex 342e0ecfc6c..5c08917f85f 100644--- a/studio/backend/core/inference/video_minimax_h3_attn.py+++ b/studio/backend/core/inference/video_minimax_h3_attn.py@@ -90,18 +90,30 @@ def _backend_value(backend: Any) -> Optional[str]:return str(getattr(backend, "value", backend))+def _diffusers_h3_processor() -> Any:+ from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3AttnProcessor+ return MiniMaxH3AttnProcessor+++def strided_processor_class(base: Any = None) -> Any:+ """Strided subclass of the H3-layout processor ``base`` (diffusers' when None). One cached class per base: dynamo+ guards on the class id, so a class per call would retrace every block."""+ return _strided_processor_for(_diffusers_h3_processor() if base is None else base)++@lru_cache(maxsize = None)-def strided_processor_class() -> Any:- """Strided ``MiniMaxH3AttnProcessor`` subclass. Built once: dynamo guards on the class id, so a class per call- would retrace every block. Masks, context parallelism and other backends take the stock ``__call__``."""+def _strided_processor_for(base: Any) -> Any:+ import sys+import torchimport torch.nn.functional as F- from diffusers.models.transformers.transformer_minimax_h3 import (- MiniMaxH3AttnProcessor,- _apply_rotary_emb,- )from torch.nn.attention import SDPBackend, sdpa_kernel+ MiniMaxH3AttnProcessor = base+ _apply_rotary_emb = getattr(sys.modules.get(base.__module__), "_apply_rotary_emb", None)+ if not callable(_apply_rotary_emb):+ raise TypeError(f"{base.__name__}: its module has no _apply_rotary_emb")+from . import diffusion_int8_gemm as int8_gemmdef _fusable_norm(norm: Any) -> bool:@@ -189,30 +201,72 @@ def strided_processor_class() -> Any:return UnslothH3StridedAttnProcessor-def _h3_attention_modules(transformer: Any) -> list:+_H3_ATTN_FORWARD = ("self", "hidden_states", "rotary_emb", "attention_mask")+_H3_PROC_CALL = ("self", "attn", "hidden_states", "rotary_emb", "attention_mask")+++@lru_cache(maxsize = None)+def _params(fn: Any) -> tuple:+ import inspecttry:- from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3Attention- except Exception: # noqa: BLE001- return []+ return tuple(inspect.signature(fn).parameters)+ except (TypeError, ValueError):+ return ()+++def h3_processor_base(module: Any) -> Any:+ """The stock H3-layout processor class of ``module``, or None. Gated on structure (forward signature, norms, QKV,+ a processor module defining ``_apply_rotary_emb``), not class, so the vendored H3-Image attention matches too."""+ import sys++ from torch import nn++ cls = type(module)+ base = getattr(cls, "_default_processor_cls", None)+ if not isinstance(module, nn.Module) or not isinstance(base, type):+ return None+ if _params(cls.forward) != _H3_ATTN_FORWARD or _params(base.__call__) != _H3_PROC_CALL:+ return None+ if not callable(getattr(sys.modules.get(base.__module__), "_apply_rotary_emb", None)):+ return None+ if not all(isinstance(getattr(module, n, None), nn.RMSNorm) for n in ("norm_q", "norm_k")):+ return None+ if not isinstance(getattr(module, "to_out", None), nn.ModuleList) or not callable(+ getattr(module, "set_processor", None)+ ):+ return None+ fused = getattr(module, "fused_projections", False)+ if not (fused and hasattr(module, "to_qkv")) and not all(+ hasattr(module, n) for n in ("to_q", "to_k", "to_v")+ ):+ return None+ if not isinstance(getattr(module, "processor", None), base):+ return None+ return base+++def _h3_attention_modules(transformer: Any) -> list:modules = getattr(transformer, "modules", None)if not callable(modules):return []- return [m for m in modules() if isinstance(m, MiniMaxH3Attention)]+ return [m for m in modules() if h3_processor_base(m) is not None]-def install_strided_attention(transformer: Any, logger: Any = None) -> int:- """Swap every H3 attention processor for the strided one, keeping its backend. Returns the count."""+def install_strided_attention(+ transformer: Any,+ logger: Any = None,+ qk_rope: Optional[bool] = None,+) -> int:+ """Swap every H3 attention processor for the strided one, keeping its backend. Returns the count.+ ``qk_rope=False`` keeps the stock q/k norm + rope; ``None`` follows UNSLOTH_H3_QK_ROPE."""if not _enabled(STRIDED_ATTN_ENV):return 0- try:- cls = strided_processor_class()- except Exception as exc: # noqa: BLE001 -- older diffusers without the H3 processor- if logger is not None:- logger.info("video.h3_attn: strided processor unavailable, keeping stock: %s", exc)+ modules = _h3_attention_modules(transformer)+ if not modules:return 0from .video_minimax_h3_qknorm import qk_norm_rope_op, qk_rope_enabled- qk_rope_on = qk_rope_enabled()+ qk_rope_on = qk_rope_enabled() and qk_rope is not Falseif qk_rope_on:try:qk_norm_rope_op()@@ -221,8 +275,14 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:if logger is not None:logger.info("video.h3_attn: fused q/k norm+rope unavailable: %s", exc)swapped = 0- for module in _h3_attention_modules(transformer):+ for module in modules:old = getattr(module, "processor", None)+ try:+ cls = strided_processor_class(h3_processor_base(module))+ except Exception as exc: # noqa: BLE001 -- older diffusers without the H3 processor+ if logger is not None:+ logger.info("video.h3_attn: strided processor unavailable, keeping stock: %s", exc)+ return swappedif old is None or isinstance(old, cls):continuenew = cls()@@ -239,15 +299,14 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:def strided_attention_count(transformer: Any) -> int:"""Engagement census."""- try:- cls = strided_processor_class()- except Exception: # noqa: BLE001- return 0- return sum(- 1- for m in _h3_attention_modules(transformer)- if isinstance(getattr(m, "processor", None), cls)- )+ count = 0+ for m in _h3_attention_modules(transformer):+ try:+ cls = strided_processor_class(h3_processor_base(m))+ except Exception: # noqa: BLE001+ return 0+ count += isinstance(getattr(m, "processor", None), cls)+ return countdef shared_qkv_count(transformer: Any) -> int:diff --git a/studio/backend/tests/test_diffusion_list_streamer.py b/studio/backend/tests/test_diffusion_list_streamer.pynew file mode 100644index 00000000000..9a0a86d19e0--- /dev/null+++ b/studio/backend/tests/test_diffusion_list_streamer.py@@ -0,0 +1,279 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The shared pinned ModuleList streamer: CPU refusals, plus CUDA parity / prefetch / remove() checks."""++from __future__ import annotations++import pytest+import torch++import core.inference.diffusion_offload_prefetch as op+++class _Stop(Exception):+ pass+++class _BufferLinear(torch.nn.Module):+ def __init__(self, width: int) -> None:+ super().__init__()+ self.register_buffer("weight", torch.randint(-127, 127, (width, width), dtype = torch.int8))+ self.register_buffer("weight_scale", torch.rand(width, 1) / 64)++ def forward(self, x):+ return x @ (self.weight.to(x.dtype) * self.weight_scale.to(x.dtype)).t()+++class _Stack(torch.nn.Module):+ def __init__(+ self,+ n: int = 6,+ width: int = 64,+ ) -> None:+ super().__init__()+ torch.manual_seed(0)+ self.inp = torch.nn.Linear(width, width)+ self.layers = torch.nn.ModuleList(+ torch.nn.Sequential(+ torch.nn.Linear(width, width * 2),+ torch.nn.GELU(),+ torch.nn.Linear(width * 2, width),+ _BufferLinear(width),+ )+ for _ in range(n)+ )+ self.stop_after = None++ def forward(self, x):+ x = self.inp(x)+ for i, layer in enumerate(self.layers):+ x = x + layer(x)+ if self.stop_after is not None and i == self.stop_after:+ raise _Stop()+ return x+++def test_cpu_device_is_refused_and_nothing_changes():+ stack = _Stack()+ before = [p.data_ptr() for p in stack.parameters()]+ assert op.stream_module_list(stack, stack.layers, "cpu", 2) is None+ assert [p.data_ptr() for p in stack.parameters()] == before+ assert not stack._forward_pre_hooks and not stack.layers[3]._forward_pre_hooks+++def test_kill_switch_refuses(monkeypatch):+ monkeypatch.setenv(op.ASYNC_PREFETCH_ENV, "0")+ assert op._list_stream_refusal("cuda").startswith(op.ASYNC_PREFETCH_ENV)+++def test_h3_image_falls_back_to_the_pageable_streamer_off_cuda(monkeypatch):+ pytest.importorskip("diffusers")+ from core.inference import diffusion_minimax_h3_image as h3i++ class _Pipe:+ _unsloth_h3_streamers: list = []++ pipe = _Pipe()+ pipe._unsloth_h3_streamers = []+ stack = _Stack()+ streamer = h3i.stream_list(pipe, stack, stack.layers, "cpu", 2, "transformer")+ assert isinstance(streamer, h3i.ModuleListStreamer) and streamer.streamed == 4+ assert pipe._unsloth_h3_streamers == [streamer] and streamer._unsloth_role == "transformer"+ monkeypatch.setenv(h3i.PINNED_STREAM_ENV, "0")+ assert h3i.stream_inflight() == 1+ monkeypatch.delenv(h3i.PINNED_STREAM_ENV)+ assert h3i.stream_inflight() == 1 + op.LIST_STREAM_DEPTH+++def test_a_stream_larger_than_the_pin_budget_is_refused(monkeypatch):+ import core.inference.diffusion_memory as dm++ stack = _Stack()+ went_on = []+ monkeypatch.setattr(op, "_list_stream_refusal", lambda device: None)+ # no diffusers needed: the budget check runs before ModuleGroup is looked up+ monkeypatch.setattr(op, "_go", lambda: type("_Go", (), {"ModuleGroup": object}))+ # the refusal must come from the budget, before any pinning+ monkeypatch.setattr(+ dm, "install_group_offload_buffer_restore", lambda: went_on.append(1) or 1 / 0+ )+ # compared in bytes, not rounded MiB+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 0)+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None+ assert not went_on and not stack.layers[3]._forward_pre_hooks+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 1)+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None+ assert went_on == [1] and not stack.layers[3]._forward_pre_hooks+++def test_the_plan_keeps_room_for_the_prefetched_member():+ pytest.importorskip("diffusers")+ from core.inference import diffusion_minimax_h3_image as h3i++ args = dict(vram_gb = 23.46, host_gb = 53.97, dit_gb = 21.9, te_gb = 27.2)+ one = h3i.plan_placement(**args)+ two = h3i.plan_placement(**args, inflight = 2)+ assert one.placement == two.placement == h3i.PLACE_STREAM+ assert one.dit_resident_blocks == 28 # the #17453 L4 plan, unchanged at one in flight+ assert two.dit_resident_blocks in (26, 27)+ a100 = dict(vram_gb = 41.96, host_gb = 86.55, dit_gb = 21.9, te_gb = 27.2)+ assert h3i.plan_placement(**a100).te_resident_layers == 20+ assert h3i.plan_placement(**a100, inflight = 2).te_resident_layers == 19+++cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+++@cuda[email protected]("resident", [0, 2, 5])+def test_streamed_stack_is_bit_identical_and_prefetches(resident):+ ref = _Stack().cuda().eval()+ stack = _Stack().eval()+ stack.inp.cuda()+ streamer = op.stream_module_list(stack, stack.layers, "cuda", resident)+ assert streamer is not None and streamer.streamed == 6 - resident+ for g in streamer.groups:+ assert all(t.is_pinned() for t in g.cpu_param_dict.values())+ x = torch.randn(8, 64, device = "cuda")+ with torch.no_grad():+ want = ref(x)+ torch.cuda.synchronize()+ before = torch.cuda.memory_allocated()+ for _ in range(3):+ assert torch.equal(stack(x), want)+ stack.stop_after = 3+ with pytest.raises(_Stop):+ stack(x)+ stack.stop_after = None+ assert torch.equal(stack(x), want)+ torch.cuda.synchronize()+ stats = streamer.stats()+ assert stats["forwards"] == 5 and stats["prefetched"] > 0+ assert streamer.prefetcher.inflight_bytes == 0 and not streamer.prefetcher.ready+ for layer in stack.layers[resident:]:+ assert all(p.device.type == "cpu" for p in layer.parameters())+ assert all(b.device.type == "cpu" for b in layer.buffers())+ assert torch.cuda.memory_allocated() <= before + (1 << 20)+ streamer.remove()+ assert not stack._forward_pre_hooks and not stack._forward_hooks+ with torch.no_grad():+ stack.cuda()+ assert torch.equal(stack(x), want)+++def test_a_stream_over_the_callers_host_cap_is_refused_before_pinning(monkeypatch):+ import core.inference.diffusion_memory as dm++ stack = _Stack()+ went_on = []+ monkeypatch.setattr(op, "_list_stream_refusal", lambda device: None)+ monkeypatch.setattr(+ dm, "install_group_offload_buffer_restore", lambda: went_on.append(1) or 1 / 0+ )+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 1 << 20)+ assert op.stream_module_list(stack, stack.layers, "cuda", 2, max_pin_bytes = 0) is None+ assert not went_on+++def _h3():+ pytest.importorskip("diffusers")+ from core.inference import diffusion_minimax_h3_image as h3i+ return h3i+++ "host, dit_room, te_pinned",+ [+ (+ 86.55,+ True,+ True,+ ), # A100 VM: DiT resident, the encoder's streamed layers fit the plan's room+ (+ 53.97,+ True,+ False,+ ), # L4 VM: the streamed DiT blocks pin, the 24 GB encoder stays pageable+ (+ 40.0,+ False,+ False,+ ), # 40 GB host (the smallest that loads on L4): nothing pins, everything streams pageable+ ],+)+def test_pins_fit_the_plans_host_room(host, dit_room, te_pinned):+ """Pinned copies are counted on top of what the plan holds on the host, so a host the plan fits still fits."""+ h3i = _h3()+ vram = 41.96 if host > 80 else 23.46+ plan = h3i.plan_placement(vram_gb = vram, host_gb = host, dit_gb = 21.9, te_gb = 27.2, inflight = 2)+ room = h3i.pin_room_bytes(plan)+ assert room is not None and room >= 0+ assert room == max(0, int((host - plan.host_need_gb) * 1e9))+ dit_stream = (+ (50 - int(plan.dit_resident_blocks or 50)) * 0.398e9+ if plan.placement == h3i.PLACE_STREAM+ else 0+ )+ te_stream = (51 - int(plan.te_resident_layers or 0)) * 0.478e9+ if plan.placement == h3i.PLACE_STREAM:+ assert (dit_stream <= room) == dit_room++ class _Pinned:+ pinned_bytes = int(dit_stream) if dit_room else 0++ left = h3i.pin_room_bytes(plan, [_Pinned()])+ assert (plan.te_kept_on_host and te_stream <= left) == te_pinned+ assert (+ h3i.pin_room_bytes(h3i.plan_placement(vram_gb = vram, host_gb = None, dit_gb = 21.9, te_gb = 27.2))+ is None+ )+++def test_a_small_host_streams_pageable_and_still_renders(monkeypatch):+ """40 GB host, stream tier (about 37 GB is the planner floor): the DiT gets no pin room and runs pageable."""+ h3i = _h3()+ import core.inference.diffusion_offload_prefetch as opf++ seen = []+ real = opf.stream_module_list+ monkeypatch.setattr(+ opf,+ "stream_module_list",+ lambda *a, **k: seen.append(k.get("max_pin_bytes")) or real(*a, **k),+ )+ with pytest.raises(RuntimeError, match = "free system memory"):+ h3i.plan_placement(vram_gb = 23.46, host_gb = 32.0, dit_gb = 21.9, te_gb = 27.2, inflight = 2)+ plan = h3i.plan_placement(vram_gb = 23.46, host_gb = 40.0, dit_gb = 21.9, te_gb = 27.2, inflight = 2)+ assert plan.placement == h3i.PLACE_STREAM and not plan.te_kept_on_host+ stack = _Stack()++ class _T(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.inp = stack.inp+ self.transformer_blocks = stack.layers++ class _V(torch.nn.Module):+ pass++ t = _T()+ streamer = h3i._place_streamed_dit(t, _V(), "cpu", plan)+ assert seen == [0] and isinstance(streamer, h3i.ModuleListStreamer)+ x = torch.randn(4, 64)+ with torch.no_grad():+ out = stack(x)+ streamer.remove()+ with torch.no_grad():+ assert torch.equal(out, stack(x))++[email protected]("value", ["0", "off"])+def test_the_global_pin_opt_out_keeps_the_pageable_streamer(monkeypatch, value):+ import core.inference.diffusion_memory as dm++ monkeypatch.setenv(dm.GROUP_OFFLOAD_PIN_ENV, value)+ assert op._list_stream_refusal("cuda") == f"{dm.GROUP_OFFLOAD_PIN_ENV}=0"+ stack = _Stack()+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None+ assert not stack.layers[3]._forward_pre_hooksdiff --git a/studio/backend/tests/test_diffusion_minimax_h3_image.py b/studio/backend/tests/test_diffusion_minimax_h3_image.pyindex 400cb56a1bc..361dfdae04f 100644--- a/studio/backend/tests/test_diffusion_minimax_h3_image.py+++ b/studio/backend/tests/test_diffusion_minimax_h3_image.py@@ -611,8 +611,18 @@ def test_backend_loads_and_generates(h3i, release, tmp_path, monkeypatch):status = backend.load_pipeline(str(root), model_kind = "pipeline")assert backend._state is not None and backend._state.family.name == "minimax-h3-image"assert backend._state.pipe._unsloth_h3_report.layout == "native"+ from core.inference import diffusion_prompt_cache as prompt_cache++ cache = prompt_cache.cache_for(backend._state.pipe)+ assert cache is not None, "the H3-Image load must install the prompt cache"out = backend.generate(prompt = "a cat", width = 256, height = 256, steps = 6, guidance = 1.0, seed = 3)assert out["images"], out+ assert cache.stats["misses"] == 1 and cache.stats["hits"] == 0+ again = backend.generate(+ prompt = "a cat", width = 256, height = 256, steps = 6, guidance = 1.0, seed = 4+ )+ assert again["images"] and cache.stats["hits"] == 1+ assert backend._state.pipe._unsloth_h3_last_stats["encode_cached"] is Trueraw = out["images"][0]data = raw.get("b64") if isinstance(raw, dict) else rawif isinstance(data, str):@@ -1139,3 +1149,163 @@ def test_the_per_prompt_encoder_refuses_a_bf16_fallback(h3i, release, tmp_path,with pytest.raises(RuntimeError, match = "bf16 encoder it would fall back to"):pipe._unsloth_h3_te_loader()pipe.release()+++def _cached_pipe(h3i, release):+ from core.inference import diffusion_prompt_cache as prompt_cache++ root, _ref, te = release+ p = h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = te)+ assert prompt_cache.install(p, identity = {"family": "minimax-h3-image"})+ return p, prompt_cache.cache_for(p)+++def _latents(+ p,+ prompt,+ seed,+ image = None,+):+ import torch+ out = p(+ prompt = prompt,+ image = image,+ height = 256,+ width = 256,+ generator = torch.Generator("cpu").manual_seed(seed),+ output_type = "latent",+ )+ return out.images[0]+++def test_a_repeated_prompt_reuses_the_conditioning_bitwise(h3i, release, monkeypatch):+ import torch++ p, cache = _cached_pipe(h3i, release)+ calls = []+ real = h3i.qwen3vl_hidden_state+ monkeypatch.setattr(+ h3i, "qwen3vl_hidden_state", lambda *a, **k: calls.append(1) or real(*a, **k)+ )+ first = _latents(p, "a red bicycle", 5)+ hit = _latents(p, "a red bicycle", 5)+ assert (+ len(calls) == 1 and cache.stats["hits"] == 1 and p._unsloth_h3_last_stats["encode_cached"]+ )+ assert torch.equal(first, hit)+ _latents(p, "a blue bicycle", 5)+ assert len(calls) == 2 and not p._unsloth_h3_last_stats["encode_cached"]+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PROMPT_CACHE", "0")+ from core.inference import diffusion_prompt_cache as prompt_cache++ q = h3i.load_minimax_h3_image_pipeline(str(release[0]), device = "cpu", text_encoder = release[2])+ assert not prompt_cache.install(q, identity = {"family": "minimax-h3-image"})+ assert torch.equal(_latents(q, "a red bicycle", 5), first)+++def test_the_conditioning_cache_keys_on_the_reference_pixels(h3i, release, monkeypatch):+ from PIL import Image++ p, cache = _cached_pipe(h3i, release)+ seen = []+ real = h3i.qwen3vl_hidden_state+ monkeypatch.setattr(+ h3i, "qwen3vl_hidden_state", lambda *a, **k: seen.append(1) or real(*a, **k)+ )+ if p.processor is None:+ pytest.skip("tiny release has no processor")+ red = Image.new("RGB", (128, 128), (200, 10, 10))+ blue = Image.new("RGB", (128, 128), (10, 10, 200))+ _latents(p, "<Picture 1> at night", 1, image = [red])+ _latents(p, "<Picture 1> at night", 2, image = [red.copy()])+ assert len(seen) == 1 and cache.stats["hits"] == 1+ _latents(p, "<Picture 1> at night", 1, image = [blue])+ assert len(seen) == 2+++def test_every_backend_loader_installs_the_prompt_cache():+ """Each DiffusionBackend method that commits a load (builds a _LoadState) installs the conditioning cache."""+ import ast++ path = _BACKEND / "core" / "inference" / "diffusion.py"+ src = path.read_text(encoding = "utf-8")+ loaders = []+ for node in ast.walk(ast.parse(src)):+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ continue+ calls = {ast.unparse(c.func) for c in ast.walk(node) if isinstance(c, ast.Call)}+ if "_LoadState" in calls:+ loaders.append(node.name)+ assert (+ "prompt_cache.install" in calls+ ), f"{node.name} commits a load without the prompt cache"+ assert {"load_pipeline", "_load_h3_image_locked"} <= set(loaders), loaders+++def test_a_declared_int8_attention_maps_to_sage():+ from core.inference import diffusion_attention as da++ assert da.checkpoint_attention_backend("int8") == "sage"+ assert da.checkpoint_attention_backend(" INT8 ") == "sage"+ assert da.checkpoint_attention_backend("fp4") is None+ assert da.checkpoint_attention_backend(None) is None+++def test_int8_attention_follows_the_checkpoint_and_the_switch(h3i, monkeypatch):+ import types++ t = types.SimpleNamespace()+ monkeypatch.delenv(h3i.INT8_ATTENTION_ENV, raising = False)+ assert h3i.checkpoint_attention_wanted(t, "int8") == "sage"+ assert h3i.checkpoint_attention_wanted(t, "fp8") is None+ assert h3i.checkpoint_attention_wanted(t, None) is None+ setattr(t, h3i.CHECKPOINT_ATTENTION_ATTR, "int8")+ assert h3i.checkpoint_attention_wanted(t, "fp8") == "sage"+ monkeypatch.setenv(h3i.INT8_ATTENTION_ENV, "0")+ assert h3i.checkpoint_attention_wanted(t, "int8") is None+ monkeypatch.setenv(h3i.INT8_ATTENTION_ENV, "1")+ assert h3i.checkpoint_attention_wanted(types.SimpleNamespace(), None) == "sage"+++def test_int8_attention_engages_on_the_blocks_only_and_comes_off(h3i, release, monkeypatch):+ from core.inference import diffusion_attention as da++ root, _ref, te = release+ p = h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = te)+ mods = h3i._attention_processors(p.transformer)+ blocks = {id(m) for b in p.transformer.transformer_blocks for m in b.modules()}+ before = {id(m): m.processor._attention_backend for m in mods}++ def fake_apply(+ pipe,+ backend,+ logger = None,+ target = None,+ ):+ for m in h3i._attention_processors(pipe.transformer):+ m.processor._attention_backend = backend+ return backend++ monkeypatch.setattr(da, "apply_attention_backend", fake_apply)+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") == "sage"+ for m in mods:+ want = "sage" if id(m) in blocks else before[id(m)]+ assert m.processor._attention_backend == want+ assert any(id(m) not in blocks for m in mods)+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") is None+ assert h3i.remove_checkpoint_attention(p) == len(mods)+ assert {id(m): m.processor._attention_backend for m in mods} == before+ monkeypatch.setattr(+ da, "apply_attention_backend", lambda *a, **k: fake_apply(a[0], None) and None+ )+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") is None+ assert {id(m): m.processor._attention_backend for m in mods} == before+ assert h3i.install_checkpoint_attention(p, "cpu", scheme = "int8") is None+++def test_hosted_metadata_attention_is_recorded(h3i):+ import types++ t = types.SimpleNamespace()+ h3i.h3_image_prepare_prequant_model()(t, {"attention": "INT8"})+ assert getattr(t, h3i.CHECKPOINT_ATTENTION_ATTR) == "int8"diff --git a/studio/backend/tests/test_diffusion_minimax_h3_image_speed.py b/studio/backend/tests/test_diffusion_minimax_h3_image_speed.pynew file mode 100644index 00000000000..a5aa2a19881--- /dev/null+++ b/studio/backend/tests/test_diffusion_minimax_h3_image_speed.py@@ -0,0 +1,234 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3-Image speed levers on a tiny random H3-Image DiT (CPU)."""++from __future__ import annotations++import copy+import sys+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parents[1]+for p in (_BACKEND, Path(__file__).resolve().parent):+ if str(p) not in sys.path:+ sys.path.insert(0, str(p))++torch = pytest.importorskip("torch")+pytest.importorskip("diffusers")+pytest.importorskip("accelerate")++from core.inference import diffusion_minimax_h3_image as h3i # noqa: E402+from core.inference import diffusion_minimax_h3_image_speed as S # noqa: E402+from core.inference import video_minimax_h3_attn as A # noqa: E402++[email protected](autouse = True)+def _restore_backend():+ from diffusers.models.attention_dispatch import _AttentionBackendRegistry++ active = _AttentionBackendRegistry._active_backend+ yield+ _AttentionBackendRegistry.set_active_backend(active)+++def _model(dtype = torch.float32):+ import _minimax_h3_image_tiny as tiny++ cfg = h3i.transformer_config_from_native(tiny.TINY_TRANSFORMER_NATIVE)+ torch.manual_seed(0)+ model = h3i.transformer_class()(**cfg).eval()+ tiny.randomize(model, 0)+ return model.to(dtype)+++def _inputs(+ model,+ dtype = torch.float32,+ n_text = 5,+ lh = 8,+ lw = 8,+):+ g = torch.Generator().manual_seed(1)+ patch = tuple(model.config.patch_size)+ tags = torch.full((n_text,), h3i.H3_IMAGE_TEXT_TAG, dtype = torch.long)+ pos, tags, image_idx, text_idx, _n_ref = h3i.build_packed_layout(tags, [], lh, lw, patch)+ seq = int(image_idx.numel() + text_idx.numel())+ unique, index = h3i.row_timestep_plan(torch.tensor([0.75]), seq, image_idx[:0])[0]+ noise = torch.randn((1, 24, 1, lh, lw), generator = g)+ text_dim = int(model.context_embedder.in_features)+ return dict(+ hidden_states = h3i.patchify(noise, patch)[None].to(dtype),+ encoder_hidden_states = torch.randn(1, n_text, text_dim, generator = g).to(dtype),+ timestep = unique,+ timestep_indices = index,+ token_tags = tags,+ position_ids = pos,+ image_indices = image_idx,+ text_indices = text_idx,+ pdd_step_index = 0,+ return_dict = False,+ )+++def _run(model, inputs):+ with torch.no_grad():+ return model(**inputs)[0]+++def _quantized_padded(model):+ pytest.importorskip("torchao")+ from torchao.quantization import quantize_++ from core.inference.diffusion_transformer_quant import _make_quant_config++ model = model.to(torch.bfloat16)+ quantize_(+ model,+ _make_quant_config("int8"),+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear)+ and fqn.startswith("transformer_blocks.")+ and fqn.endswith(h3i.QUANT_LINEAR_SUFFIXES),+ )+ wrapped = sum(+ h3i.wrap_row_multiple(b, h3i.QUANT_LINEAR_SUFFIXES) for b in model.transformer_blocks+ )+ if not wrapped:+ pytest.skip("this torchao gives no per-row int8 Linears")+ return model+++def test_the_vendored_attention_is_recognised_by_structure():+ model = _model()+ mods = A._h3_attention_modules(model)+ assert len(mods) == 4+ base = A.h3_processor_base(mods[0])+ assert base is type(model.transformer_blocks[0].attn.processor)+ cls = A.strided_processor_class(base)+ assert issubclass(cls, base) and cls is A.strided_processor_class(base)+ assert A.h3_processor_base(model.transformer_blocks[0].ff) is None+ assert A.h3_processor_base(torch.nn.Linear(2, 2)) is None++[email protected]("qk_rope", ["1", "0"])+def test_strided_processor_matches_stock_bit_for_bit(monkeypatch, qk_rope):+ monkeypatch.setenv("UNSLOTH_H3_QK_ROPE", qk_rope)+ model = _model()+ model.set_attention_backend("_native_math")+ stock = copy.deepcopy(model)+ assert A.install_strided_attention(model) == 4+ assert A.strided_attention_count(model) == 4+ proc = model.transformer_blocks[0].attn.processor+ assert A._backend_value(proc._attention_backend) == "_native_math"+ assert proc._unsloth_qk_rope is (qk_rope == "1")+ inputs = _inputs(model)+ assert torch.equal(_run(stock, inputs), _run(model, inputs))+++ "resident, image_env, video_env, want",+ [+ (False, None, None, False),+ (True, None, None, True),+ (False, "1", None, True),+ (True, "0", None, False),+ (True, None, "0", False),+ (False, "1", "0", False),+ ],+)+def test_fused_qk_rope_default_follows_the_tier(monkeypatch, resident, image_env, video_env, want):+ for env, value in ((S.QK_ROPE_ENV, image_env), ("UNSLOTH_H3_QK_ROPE", video_env)):+ if value is None:+ monkeypatch.delenv(env, raising = False)+ else:+ monkeypatch.setenv(env, value)+ monkeypatch.setattr(+ "core.inference.video_minimax_h3_qknorm.qk_norm_rope_op", lambda: None, raising = False+ )+ model = _model()+ assert A.install_strided_attention(model, qk_rope = S.image_qk_rope_enabled(resident)) == 4+ assert model.transformer_blocks[0].attn.processor._unsloth_qk_rope is want+++def test_strided_processor_defers_to_stock_for_an_unpinned_backend():+ model = _model()+ stock = copy.deepcopy(model)+ A.install_strided_attention(model)+ for m in A._h3_attention_modules(model):+ m.processor._attention_backend = None+ inputs = _inputs(model)+ assert torch.equal(_run(stock, inputs), _run(model, inputs))+++def test_strided_kill_switch(monkeypatch):+ monkeypatch.setenv(A.STRIDED_ATTN_ENV, "0")+ model = _model()+ assert A.install_strided_attention(model) == 0+ assert A.strided_attention_count(model) == 0+++def test_int8_qkv_fuses_through_the_row_pads_exactly():+ model = _quantized_padded(_model())+ stock = copy.deepcopy(model)+ fused = S._fuse_qkv(model, None)+ assert fused == 2+ attn = model.transformer_blocks[0].attn+ assert attn.fused_projections and not hasattr(attn, "to_q")+ assert type(attn.to_qkv).__name__ == "H3ImageRowPad"+ assert type(attn.to_out[0]).__name__ == "H3ImageRowPad"+ assert not model.token_refiner.refiner_blocks[0].attn.fused_projections+ inputs = _inputs(model, torch.bfloat16)+ assert torch.equal(_run(stock, inputs), _run(model, inputs))+++def test_fused_qkv_kill_switch_keeps_the_pads(monkeypatch):+ monkeypatch.setenv(A.FUSED_QKV_ENV, "0")+ model = _quantized_padded(_model())+ assert S._fuse_qkv(model, None) == 0+ attn = model.transformer_blocks[0].attn+ assert not attn.fused_projections+ assert all(type(getattr(attn, n)).__name__ == "H3ImageRowPad" for n in ("to_q", "to_k", "to_v"))+++def test_shared_qkv_needs_the_int8_swap():+ model = _quantized_padded(_model())+ A.install_strided_attention(model)+ assert A.shared_qkv_count(model) == 0+++def test_install_is_off_off_cuda_and_under_the_umbrella_switch(monkeypatch):+ model = _model()+ pipe = type("P", (), {})()+ pipe.transformer = model+ plan = type("Plan", (), {"placement": "dit_resident"})()+ assert S.install_h3_image_speed(pipe, plan, "cpu") == ()+ assert pipe._unsloth_h3_speed_levers == ()+ monkeypatch.setenv(S.SPEED_ENV, "0")+ assert not S.speed_enabled()+ assert S.install_h3_image_speed(pipe, plan, "cuda") == ()+ assert A.strided_attention_count(model) == 0+++def test_compile_is_opt_in(monkeypatch):+ monkeypatch.delenv(S.COMPILE_ENV, raising = False)+ assert not S.compile_enabled()+ monkeypatch.setenv(S.COMPILE_ENV, "1")+ assert S.compile_enabled()+++def test_the_loader_installs_the_levers_once_and_reports_them():+ import ast+ import inspect++ src = inspect.getsource(h3i.load_minimax_h3_image_pipeline)+ calls = [+ n+ for n in ast.walk(ast.parse(src))+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_h3_image_speed"+ ]+ assert len(calls) == 1+ assert src.index("install_h3_image_speed(") > src.rindex("early_streamer.remove()")+ assert src.index("install_h3_image_speed(") < src.index("H3ImageLoadReport(")+ assert "speed_levers" in h3i.H3ImageLoadReport.__dataclass_fields__