Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio H3-Image: prompt cache, pinned streaming and the H3 attention levers (#17495)

Daniel Han5d5c1381b01c
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index f65abb51e16..fbde3601429 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -11815,6 +11815,17 @@ class DiffusionBackend:
)
report = pipe._unsloth_h3_report
plan = report.plan
+ prompt_cache.install(
+ pipe,
+ identity = {
+ "family": fam.name,
+ "repo": str(repo_id),
+ "base": str(base or repo_id),
+ "dtype": str(target.dtype),
+ "te_quant": str(report.te_scheme),
+ },
+ logger = logger,
+ )
policy = (
OFFLOAD_NONE
if plan.placement == PLACE_RESIDENT
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index aef204ecbd0..07648c24d6d 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -4058,3 +4058,13 @@ def install_hunyuan_attention_trim(
if engaged and logger is not None:
logger.info("diffusion.attention: hunyuan padded-text trim engaged")
return engaged
+
+
+# declared checkpoint attention -> Studio backend of the same arithmetic (still probe-gated)
+CHECKPOINT_ATTENTION_BACKENDS = {"int8": "sage"}
+
+
+def checkpoint_attention_backend(declared: Any) -> Optional[str]:
+ if not isinstance(declared, str):
+ return None
+ return CHECKPOINT_ATTENTION_BACKENDS.get(declared.strip().lower())
diff --git a/studio/backend/core/inference/diffusion_minimax_h3_image.py b/studio/backend/core/inference/diffusion_minimax_h3_image.py
index 50d20d74700..5dda42dbbe5 100644
--- a/studio/backend/core/inference/diffusion_minimax_h3_image.py
+++ b/studio/backend/core/inference/diffusion_minimax_h3_image.py
@@ -113,6 +113,8 @@ ADALN_TABLE_ENV = "UNSLOTH_H3_IMAGE_ADALN_TABLE"
PLACEMENT_ENV = "UNSLOTH_H3_IMAGE_PLACEMENT"
VRAM_BUDGET_ENV = "UNSLOTH_H3_IMAGE_VRAM_GB"
HOST_BUDGET_ENV = "UNSLOTH_H3_IMAGE_HOST_GB"
+# "0": stream from pageable memory (ModuleListStreamer) instead of the pinned prefetch streamer
+PINNED_STREAM_ENV = "UNSLOTH_H3_IMAGE_PINNED_STREAM"
QUANT_SCHEMES = ("int8", "fp8")
# ConvRot group: the one the H3 video denoiser and its hosted encoder use (divides 5376, 7168 and 14336).
@@ -785,6 +787,8 @@ class H3ImagePlan:
# False: the encoder is loaded for each encode and freed after (host RAM cannot hold it beside the DiT)
te_kept_on_host: bool = True
reasons: list = field(default_factory = list)
+ # host GB held once placed (host DiT blocks + encoder + 4 GB); None without a host budget
+ host_need_gb: Optional[float] = None
def as_dict(self) -> dict:
return {
@@ -801,6 +805,17 @@ class H3ImagePlan:
}
+def pinned_stream_enabled() -> bool:
+ return not _env_off(PINNED_STREAM_ENV)
+
+
+def stream_inflight() -> int:
+ if not pinned_stream_enabled():
+ return 1
+ from .diffusion_offload_prefetch import LIST_STREAM_DEPTH
+ return 1 + LIST_STREAM_DEPTH
+
+
def activation_gb(
width: int,
height: int,
@@ -842,6 +857,7 @@ def plan_placement(
num_blocks: int = 50,
te_layers: int = 50,
forced: Optional[str] = None,
+ inflight: int = 1,
) -> H3ImagePlan:
"""Pure: VRAM and host budgets (GB, None = unknown/CPU) -> placement, best first:
@@ -851,7 +867,8 @@ def plan_placement(
- sequential: one big component on the card at a time (a DiT larger than the encoder room);
- stream: DiT blocks stream past the ones that fit, the encoder streams every layer (L4 24 GB).
- Host: the encoder is kept only when it fits beside the DiT, else it loads per prompt and is freed after."""
+ Host: the encoder is kept only when it fits beside the DiT, else it loads per prompt and is freed after.
+ ``inflight``: streamed members on the device at once (2 with the prefetch streamer: one runs, one copies)."""
reasons = []
act = activation_gb(width, height, references)
margin = 1.5
@@ -870,6 +887,7 @@ def plan_placement(
return plan
layer_gb = max(te_gb - TE_TOP_GB, 0.1) / max(1, te_layers)
block_gb = max(dit_gb - DIT_TOP_GB, 0.1) / max(1, num_blocks)
+ inflight = max(1, int(inflight))
if forced in PLACEMENTS:
choice = forced
reasons.append(f"placement forced to {forced} by {PLACEMENT_ENV}")
@@ -878,7 +896,7 @@ def plan_placement(
reasons.append(
f"DiT + encoder + VAE ({dit_gb + te_gb + vae_gb:.1f} GB) fit {vram_gb:.1f} GB"
)
- elif dit_gb + vae_gb + act + margin + TE_TOP_GB + layer_gb <= vram_gb:
+ elif dit_gb + vae_gb + act + margin + TE_TOP_GB + layer_gb * inflight <= vram_gb:
choice = PLACE_DIT_RESIDENT
reasons.append(
f"DiT + VAE ({dit_gb + vae_gb:.1f} GB) stay on {vram_gb:.1f} GB; the encoder streams"
@@ -893,11 +911,12 @@ def plan_placement(
reasons.append(f"DiT {dit_gb:.1f} GB + working set exceeds {vram_gb:.1f} GB: blocks stream")
plan.placement = choice
if choice == PLACE_DIT_RESIDENT:
- spare = vram_gb - dit_gb - vae_gb - act - margin - TE_TOP_GB - layer_gb
+ spare = vram_gb - dit_gb - vae_gb - act - margin - TE_TOP_GB - layer_gb * inflight
plan.te_resident_layers = max(0, min(te_layers, int(spare / layer_gb)))
reasons.append(f"{plan.te_resident_layers}/{te_layers} encoder layers stay on the device")
elif choice == PLACE_STREAM:
spare = vram_gb - act - margin - vae_gb - DIT_TOP_GB - TE_TOP_GB - block_gb
+ spare -= (inflight - 1) * max(block_gb, layer_gb)
plan.dit_resident_blocks = max(0, min(num_blocks, int(spare / block_gb)))
# The resident blocks hold their room through the encode, so every encoder layer streams.
plan.te_resident_layers = 0
@@ -912,6 +931,7 @@ def plan_placement(
PLACE_STREAM: block_gb * (num_blocks - int(plan.dit_resident_blocks or 0)),
}[choice]
need = dit_host + te_gb + 4.0
+ plan.host_need_gb = need
plan.te_kept_on_host = host_gb >= need
# Loading per prompt only frees the encoder between prompts: each load still holds it whole beside the
# host-side DiT blocks, so a host below that peak would be OOM-killed on the first prompt.
@@ -1138,8 +1158,13 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):
"MiniMax-H3-Image editing needs the Qwen3-VL processor (processor/ of the repo)."
)
- def _encode(self, prompt: str, vision_images: Optional[list]) -> tuple:
- """(prompt_embeds (1, n, 5120), per-row text tags) for one presentation."""
+ def encode_prompt(
+ self,
+ prompt: str,
+ vision_images: Optional[list] = None,
+ ) -> tuple:
+ """(prompt_embeds, per-row text tags). Named like diffusers' encoder entry point so ``diffusion_prompt_cache``
+ wraps it; VAE reference latents are never cached (seeded sample)."""
import torch
tokenizer = self.tokenizer
@@ -1184,6 +1209,23 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):
self._leave("text_encoder")
return embeds, torch.tensor(tags, dtype = torch.long)
+ def _encode(self, prompt: str, vision_images: Optional[list]) -> tuple:
+ return self.encode_prompt(prompt, vision_images)
+
+ def _streamer_stats(self) -> dict:
+ """Per role: pinned prefetch counters, or {"pageable": n}."""
+ out = {}
+ for streamer in self._unsloth_h3_streamers:
+ role = getattr(streamer, "_unsloth_role", "?")
+ read = getattr(streamer, "stats", None)
+ out[role] = read() if callable(read) else {"pageable": int(streamer.streamed)}
+ return out
+
+ def _prompt_cache_hits(self) -> int:
+ cache = getattr(self, "_unsloth_prompt_cache", None)
+ stats = getattr(cache, "stats", None)
+ return int(stats.get("hits", 0)) if isinstance(stats, dict) else 0
+
def _encode_references(self, images: list, generator: Any) -> list:
import numpy as np
import torch
@@ -1332,8 +1374,10 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):
device = self._device()
canvas_h, canvas_w = canvas_size(height, width, min_generation_area)
stats["canvas"] = (canvas_w, canvas_h)
- embeds, text_tags = self._encode(prompt, vision_refs)
+ hits = self._prompt_cache_hits()
+ embeds, text_tags = self.encode_prompt(prompt, vision_refs)
stats["encode_s"] = time.perf_counter() - t0
+ stats["encode_cached"] = self._prompt_cache_hits() > hits
patch = tuple(self.transformer.config.patch_size)
channels = int(self.vae.config.latent_channels) if self.vae is not None else 24
lh, lw = canvas_h // H3_IMAGE_VAE_SCALE, canvas_w // H3_IMAGE_VAE_SCALE
@@ -1416,6 +1460,7 @@ class MiniMaxH3ImagePipeline(_pipeline_base()):
images = self._decode(final, height, width, output_type)
stats["decode_s"] = time.perf_counter() - t2
stats["total_s"] = time.perf_counter() - t0
+ stats["streamers"] = self._streamer_stats()
self._unsloth_h3_last_stats = stats
return images
@@ -1879,6 +1924,10 @@ def h3_image_prepare_prequant_model(logger_: Any = None) -> Callable:
def _prepare(transformer: Any, metadata: Optional[dict]) -> None:
import torch
+ declared = metadata.get("attention") if isinstance(metadata, dict) else None
+ if isinstance(declared, str) and declared.strip():
+ setattr(transformer, CHECKPOINT_ATTENTION_ATTR, declared.strip().lower())
+
from .video_minimax_h3_adaln import apply_h3_adaln_curve, is_curve_checkpoint
if is_curve_checkpoint(metadata):
@@ -2087,6 +2136,7 @@ class H3ImageLoadReport:
load_s: float = 0.0
quantized_linears: int = 0
bytes_read: int = 0
+ speed_levers: tuple = ()
def as_dict(self) -> dict:
return {
@@ -2099,6 +2149,7 @@ class H3ImageLoadReport:
"load_s": round(self.load_s, 1),
"quantized_linears": self.quantized_linears,
"bytes_read": self.bytes_read,
+ "speed_levers": list(self.speed_levers),
**{f"plan_{k}": v for k, v in self.plan.as_dict().items()},
}
@@ -2316,21 +2367,209 @@ def load_h3_image_text_encoder(
return encoder, None
+def pin_room_bytes(plan: H3ImagePlan, pinned: Sequence[Any] = ()) -> Optional[int]:
+ """Host bytes left for pins: host budget less what the plan holds less ``pinned`` streamers' locks. Pins count on
+ top of the pageable copies they replace (those may stay cached). None without a host budget."""
+ if plan.host_gb is None or plan.host_need_gb is None:
+ return None
+ room = int((float(plan.host_gb) - float(plan.host_need_gb)) * 1e9)
+ room -= sum(int(getattr(s, "pinned_bytes", 0) or 0) for s in pinned if s is not None)
+ return max(0, room)
+
+
+def make_list_streamer(
+ owner: Any,
+ members: Any,
+ device: Any,
+ resident: int,
+ role: str,
+ *,
+ pinned: bool = True,
+ reserve_bytes: int = 0,
+ max_pin_bytes: Optional[int] = None,
+) -> Any:
+ """A streamer for ``members[resident:]``: the pinned prefetch one where it fits, else ``ModuleListStreamer``."""
+ streamer = None
+ if pinned and pinned_stream_enabled():
+ from .diffusion_offload_prefetch import stream_module_list
+ streamer = stream_module_list(
+ owner,
+ members,
+ device,
+ resident,
+ logger = logger,
+ reserve_bytes = reserve_bytes,
+ max_pin_bytes = max_pin_bytes,
+ )
+ if streamer is None:
+ streamer = ModuleListStreamer(members, device, resident)
+ streamer._unsloth_role = role
+ return streamer
+
+
+def stream_list(
+ pipe: Any,
+ owner: Any,
+ members: Any,
+ device: Any,
+ resident: int,
+ role: str,
+ *,
+ pinned: bool = True,
+ reserve_bytes: int = 0,
+ max_pin_bytes: Optional[int] = None,
+) -> Any:
+ """``make_list_streamer``, registered on the pipe by role."""
+ streamer = make_list_streamer(
+ owner,
+ members,
+ device,
+ resident,
+ role,
+ pinned = pinned,
+ reserve_bytes = reserve_bytes,
+ max_pin_bytes = max_pin_bytes,
+ )
+ pipe._unsloth_h3_streamers.append(streamer)
+ return streamer
+
+
def _place_streamed_dit(
- transformer: Any, vae: Any, device: Any, plan: H3ImagePlan
-) -> ModuleListStreamer:
+ transformer: Any,
+ vae: Any,
+ device: Any,
+ plan: H3ImagePlan,
+ pinned_already: Sequence[Any] = (),
+) -> Any:
+ """Placed before the encoder so the DiT blocks (streamed every step) get the pin budget first."""
vae.to(device)
for name, child in transformer.named_children():
if name != "transformer_blocks":
child.to(device)
resident = int(plan.dit_resident_blocks or 0)
- streamer = ModuleListStreamer(transformer.transformer_blocks, device, resident)
- streamer._unsloth_role = "transformer"
+ streamer = make_list_streamer(
+ transformer,
+ transformer.transformer_blocks,
+ device,
+ resident,
+ "transformer",
+ max_pin_bytes = pin_room_bytes(plan, pinned_already),
+ )
streamer._unsloth_resident = resident
return streamer
-def _place_text_encoder(pipe: Any, encoder: Any, plan: H3ImagePlan, device: Any) -> Any:
+# "0": native attention; "1": probe-gated SageAttention on the DiT blocks; unset: as the checkpoint declares
+INT8_ATTENTION_ENV = "UNSLOTH_H3_IMAGE_INT8_ATTENTION"
+CHECKPOINT_ATTENTION_ATTR = "_unsloth_checkpoint_attention"
+# the INT8 DiT checkpoint runs INT8 attention on its DiT blocks unless its file says otherwise
+SCHEME_ATTENTION = {"int8": "int8"}
+ATTENTION_SAVED_ATTR = "_unsloth_h3_attention_saved"
+_LOG = logger
+
+
+def checkpoint_attention_wanted(transformer: Any, scheme: Optional[str]) -> Optional[str]:
+ """The DiT blocks' attention backend ("sage" or None) per the env switch or checkpoint."""
+ from .diffusion_attention import checkpoint_attention_backend
+
+ raw = (os.environ.get(INT8_ATTENTION_ENV) or "").strip().lower()
+ if raw in ("0", "off", "false", "no"):
+ return None
+ if raw in ("1", "on", "true", "yes"):
+ return "sage"
+ declared = getattr(transformer, CHECKPOINT_ATTENTION_ATTR, None)
+ if not declared:
+ declared = SCHEME_ATTENTION.get(scheme or "")
+ return checkpoint_attention_backend(declared)
+
+
+def _attention_processors(module: Any) -> list:
+ return [
+ m for m in module.modules() if hasattr(getattr(m, "processor", None), "_attention_backend")
+ ]
+
+
+def install_checkpoint_attention(
+ pipe: Any,
+ device: Any,
+ *,
+ scheme: Optional[str] = None,
+ logger: Any = None,
+) -> Optional[str]:
+ """Route the DiT blocks to the declared backend (SageAttention must pass its probe here first); the token refiner
+ keeps its own. Returns the engaged backend, or None with every processor unchanged."""
+ import torch
+
+ log = logger if logger is not None else _LOG
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None or getattr(pipe, ATTENTION_SAVED_ATTR, None):
+ return None
+ want = checkpoint_attention_wanted(transformer, scheme)
+ dev = torch.device(device)
+ if want is None or dev.type != "cuda":
+ return None
+ from .diffusion_attention import ATTENTION_BACKEND_ATTR, apply_attention_backend
+
+ modules = _attention_processors(transformer)
+ saved = {m: m.processor._attention_backend for m in modules}
+ tag = getattr(transformer, ATTENTION_BACKEND_ATTR, None)
+
+ def restore(only: Any = None) -> None:
+ for m, backend in saved.items():
+ if only is None or m in only:
+ m.processor._attention_backend = backend
+
+ dtype = _module_dtype(transformer, skip_fp32 = True)
+ target = types.SimpleNamespace(device = dev.type, dtype = dtype, torch_device = str(dev))
+ try:
+ engaged = apply_attention_backend(pipe, want, logger = log, target = target)
+ except Exception as exc: # noqa: BLE001 - native attention stays
+ engaged = None
+ log.warning("diffusion.minimax_h3_image: INT8 attention unavailable (%s)", exc)
+ if not engaged or "sage" not in str(engaged):
+ restore()
+ setattr(transformer, ATTENTION_BACKEND_ATTR, tag)
+ return None
+ blocks = {id(m) for b in getattr(transformer, "transformer_blocks", ()) for m in b.modules()}
+ restore(only = [m for m in modules if id(m) not in blocks])
+ setattr(pipe, ATTENTION_SAVED_ATTR, (saved, tag))
+ log.info(
+ "diffusion.minimax_h3_image: INT8 attention (%s) on %d DiT blocks, as the checkpoint declares",
+ engaged,
+ sum(1 for m in modules if id(m) in blocks),
+ )
+ return engaged
+
+
+def remove_checkpoint_attention(pipe: Any) -> int:
+ """Every attention processor back to the backend it had before ``install_checkpoint_attention``."""
+ entry = getattr(pipe, ATTENTION_SAVED_ATTR, None)
+ if not entry:
+ return 0
+ from .diffusion_attention import ATTENTION_BACKEND_ATTR
+
+ saved, tag = entry
+ for m, backend in saved.items():
+ m.processor._attention_backend = backend
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is not None:
+ setattr(transformer, ATTENTION_BACKEND_ATTR, tag)
+ setattr(pipe, ATTENTION_SAVED_ATTR, None)
+ return len(saved)
+
+
+def _place_text_encoder(
+ pipe: Any,
+ encoder: Any,
+ plan: H3ImagePlan,
+ device: Any,
+ *,
+ per_prompt: bool = False,
+ reserve_bytes: int = 0,
+ pinned_already: Sequence[Any] = (),
+) -> Any:
+ """``per_prompt`` (low-host tier, encoder reloaded each prompt) streams pageable: pinning would copy 27 GB per
+ prompt. ``pinned_already``: streamers whose pins count against the host room first."""
if encoder is None:
return None
if plan.placement == PLACE_RESIDENT:
@@ -2343,9 +2582,17 @@ def _place_text_encoder(pipe: Any, encoder: Any, plan: H3ImagePlan, device: Any)
for name, child in encoder.model.language_model.named_children():
if name != "layers":
child.to(device)
- streamer = ModuleListStreamer(layers, device, int(plan.te_resident_layers or 0))
- streamer._unsloth_role = "text_encoder"
- pipe._unsloth_h3_streamers.append(streamer)
+ stream_list(
+ pipe,
+ encoder.model.language_model,
+ layers,
+ device,
+ int(plan.te_resident_layers or 0),
+ "text_encoder",
+ pinned = not per_prompt,
+ reserve_bytes = reserve_bytes,
+ max_pin_bytes = pin_room_bytes(plan, pinned_already),
+ )
return encoder
@@ -2420,6 +2667,7 @@ def load_minimax_h3_image_pipeline(
"width": plan_width,
"height": plan_height,
"forced": (os.environ.get(PLACEMENT_ENV) or "").strip().lower() or None,
+ "inflight": stream_inflight() if device.type == "cuda" else 1,
}
plan = plan_placement(
te_gb = te_gb_for(te_scheme) if text_encoder is None else _module_gb(text_encoder),
@@ -2576,11 +2824,11 @@ def load_minimax_h3_image_pipeline(
"for each prompt because host memory is short). Check the network or the Hugging Face cache "
"and load the model again."
)
- return _place_text_encoder(pipe, encoder, plan, device)
+ return _place_text_encoder(pipe, encoder, plan, device, per_prompt = True)
pipe._unsloth_h3_te_loader = _te_loader
else:
- _place_text_encoder(pipe, text_encoder, plan, device)
+ _place_text_encoder(pipe, text_encoder, plan, device, pinned_already = [early_streamer])
if plan.placement != PLACE_SEQUENTIAL:
vae.to(device)
if plan.placement in (PLACE_RESIDENT, PLACE_DIT_RESIDENT):
@@ -2592,11 +2840,19 @@ def load_minimax_h3_image_pipeline(
if early_streamer is None or early_streamer._unsloth_resident != resident:
if early_streamer is not None:
early_streamer.remove()
- early_streamer = _place_streamed_dit(transformer, vae, device, plan)
+ early_streamer = _place_streamed_dit(
+ transformer, vae, device, plan, pinned_already = pipe._unsloth_h3_streamers
+ )
pipe._unsloth_h3_streamers.append(early_streamer)
early_streamer = None
if early_streamer is not None:
early_streamer.remove()
+ from .diffusion_minimax_h3_image_speed import install_h3_image_speed
+
+ speed_levers = install_h3_image_speed(pipe, plan, device, logger)
+ int8_attention = install_checkpoint_attention(pipe, device, scheme = scheme, logger = logger)
+ if int8_attention:
+ speed_levers = tuple(speed_levers) + (f"checkpoint_attention={int8_attention}",)
report = H3ImageLoadReport(
scheme = scheme,
scheme_reason = scheme_reason,
@@ -2608,6 +2864,7 @@ def load_minimax_h3_image_pipeline(
load_s = time.perf_counter() - t0,
quantized_linears = int(stats.get("quantized_linears", 0)),
bytes_read = int(stats.get("bytes_read", 0)),
+ speed_levers = speed_levers,
)
pipe._unsloth_h3_report = report
try:
diff --git a/studio/backend/core/inference/diffusion_minimax_h3_image_speed.py b/studio/backend/core/inference/diffusion_minimax_h3_image_speed.py
new file mode 100644
index 00000000000..a2d44e707f2
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_minimax_h3_image_speed.py
@@ -0,0 +1,182 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3-Image speed levers, reusing the MiniMax-H3 video ones (same attention layout).
+``UNSLOTH_H3_IMAGE_SPEED=0`` disables all; each also honours its video switch.
+
+Exact: h3_int8_gemm, h3_fused_qkv, h3_strided_attn, h3_shared_qkv_quant, the _native_flash pin on sm80 / sm89.
+Tolerance-level: h3_qk_rope (last-bit rstd drift changes the 6-step image, so off by default on the streamed tier;
+UNSLOTH_H3_IMAGE_QK_ROPE forces it), h3_fused_swiglu, h3_compiled (opt-in via UNSLOTH_H3_IMAGE_COMPILE=1: workspace
+unmeasured beside the 37.5 GB A100-40 dit_resident peak). The streamed tier gets attention levers only: its blocks
+move per forward, so resident-projection levers cannot engage."""
+
+from __future__ import annotations
+
+import os
+import types
+from typing import Any, Optional
+
+SPEED_ENV = "UNSLOTH_H3_IMAGE_SPEED"
+COMPILE_ENV = "UNSLOTH_H3_IMAGE_COMPILE"
+QK_ROPE_ENV = "UNSLOTH_H3_IMAGE_QK_ROPE"
+LEVERS_ATTR = "_unsloth_h3_speed_levers"
+_RESIDENT_PLACEMENTS = ("resident", "dit_resident")
+
+
+def speed_enabled() -> bool:
+ return (os.environ.get(SPEED_ENV) or "1").strip().lower() not in ("0", "off", "false", "no")
+
+
+def compile_enabled() -> bool:
+ return (os.environ.get(COMPILE_ENV) or "").strip().lower() in ("1", "on", "true", "yes")
+
+
+def image_qk_rope_enabled(resident: bool = False) -> bool:
+ """The fused q/k norm + rope for H3-Image: forced by UNSLOTH_H3_IMAGE_QK_ROPE, else on for the resident tiers."""
+ value = (os.environ.get(QK_ROPE_ENV) or "").strip().lower()
+ if value in ("1", "on", "true", "yes"):
+ return True
+ if value in ("0", "off", "false", "no"):
+ return False
+ return bool(resident)
+
+
+def _row_pads(transformer: Any) -> list:
+ out = []
+ for parent in transformer.modules():
+ for leaf, child in list(parent._modules.items()):
+ if type(child).__name__ == "H3ImageRowPad" and "inner" in child._modules:
+ out.append((parent, leaf, child))
+ return out
+
+
+def _unwrap(pads: list, keep: Any = None) -> int:
+ n = 0
+ for parent, leaf, wrapper in pads:
+ inner = wrapper._modules["inner"]
+ if keep is not None and not keep(inner):
+ continue
+ setattr(parent, leaf, inner)
+ n += 1
+ return n
+
+
+def _fuse_qkv(transformer: Any, logger: Any) -> int:
+ """fuse_h3_qkv_ needs bare projections: unwrap, fuse, re-pad so the int8 path keeps its row multiple."""
+ from .diffusion_minimax_h3_image import wrap_row_multiple
+ from .video_minimax_h3_attn import fuse_h3_qkv_
+
+ pads = [p for p in _row_pads(transformer) if p[1] in ("to_q", "to_k", "to_v")]
+ _unwrap(pads)
+ try:
+ fused = fuse_h3_qkv_(transformer, logger = logger)
+ finally:
+ for parent, leaf, wrapper in pads:
+ if leaf in parent._modules: # left unfused
+ setattr(parent, leaf, wrapper)
+ for parent, _leaf, _wrapper in pads:
+ if getattr(parent, "fused_projections", False) and type(
+ parent._modules.get("to_qkv")
+ ).__name__ != ("H3ImageRowPad"):
+ wrap_row_multiple(parent, ["to_qkv"])
+ return fused
+
+
+def _fused_gemm_serves(inner: Any) -> bool:
+ from .diffusion_int8_gemm import _REC
+ rec = inner.__dict__.get(_REC)
+ return bool(rec is not None and rec[4])
+
+
+def install_h3_image_speed(
+ pipe: Any,
+ plan: Any,
+ device: Any,
+ logger: Any = None,
+) -> tuple:
+ """Engage the levers after placement, before the first forward; returns the engaged names, never raises."""
+ levers: list = []
+ try:
+ levers = _install(pipe, plan, device, logger)
+ except Exception as exc: # noqa: BLE001 - an optimisation: the stock path stays
+ if logger is not None:
+ logger.warning("diffusion.minimax_h3_image: speed levers skipped: %s", exc)
+ out = tuple(levers)
+ try:
+ setattr(pipe, LEVERS_ATTR, out)
+ except Exception: # noqa: BLE001
+ pass
+ if logger is not None and out:
+ logger.info("diffusion.minimax_h3_image: speed levers %s", ",".join(out))
+ return out
+
+
+def _install(pipe: Any, plan: Any, device: Any, logger: Any) -> list:
+ import torch
+
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None or not speed_enabled():
+ return []
+ dev = torch.device(device)
+ if dev.type != "cuda" or not torch.cuda.is_available():
+ return []
+ levers: list = []
+ resident = getattr(plan, "placement", None) in _RESIDENT_PLACEMENTS
+ if resident and _fuse_qkv(transformer, logger):
+ levers.append("h3_fused_qkv")
+ if resident:
+ from . import diffusion_int8_gemm as int8_gemm
+ if int8_gemm.install(transformer, logger):
+ _unwrap(_row_pads(transformer), keep = _fused_gemm_serves)
+ levers.append("h3_int8_gemm")
+ from .diffusion_attention import apply_attention_backend, select_attention_backend
+ from .video_minimax_h3_attn import (
+ h3_attention_backend,
+ install_strided_attention,
+ shared_qkv_count,
+ )
+
+ from .diffusion_minimax_h3_image import _module_dtype
+
+ # the block stack's dtype: the fp32 embedders and heads come first in the parameter order
+ dtype = _module_dtype(transformer, skip_fp32 = True)
+ target = types.SimpleNamespace(device = dev.type, dtype = dtype, torch_device = str(dev))
+ backend = select_attention_backend(target, None, speed_active = True)
+ try:
+ backend = h3_attention_backend(backend, requested = None, target = target)
+ except Exception: # noqa: BLE001 - keep the generic pick
+ pass
+ engaged = apply_attention_backend(pipe, backend, logger = logger, target = target)
+ if engaged:
+ levers.append(f"attention={engaged}")
+ if engaged in ("_native_cudnn", "_native_flash") and install_strided_attention(
+ transformer, logger = logger, qk_rope = image_qk_rope_enabled(resident)
+ ):
+ levers.append("h3_strided_attn")
+ from .video_minimax_h3_qknorm import qk_rope_enabled
+
+ procs = [getattr(m, "processor", None) for m in transformer.modules()]
+ if (
+ image_qk_rope_enabled(resident)
+ and qk_rope_enabled()
+ and any(getattr(p, "_unsloth_qk_rope", False) for p in procs)
+ ):
+ levers.append("h3_qk_rope")
+ if shared_qkv_count(transformer):
+ levers.append("h3_shared_qkv_quant")
+ if "h3_int8_gemm" in levers:
+ from .video_minimax_h3_ffn import install as install_ffn
+ if install_ffn(transformer, device = dev, logger = logger):
+ levers.append("h3_fused_swiglu")
+ if resident and compile_enabled():
+ compiled = 0
+ for block in getattr(transformer, "transformer_blocks", ()):
+ block.compile()
+ compiled += 1
+ if compiled:
+ levers.append("h3_compiled")
+ return levers
+
+
+def speed_levers(pipe: Any) -> Optional[tuple]:
+ return getattr(pipe, LEVERS_ATTR, None)
diff --git a/studio/backend/core/inference/diffusion_offload_prefetch.py b/studio/backend/core/inference/diffusion_offload_prefetch.py
index 80c87dc5160..e880684ef83 100644
--- a/studio/backend/core/inference/diffusion_offload_prefetch.py
+++ b/studio/backend/core/inference/diffusion_offload_prefetch.py
@@ -1346,3 +1346,233 @@ def cross_forward_step(
pf.drop_carried() # the next step runs another expert: these copies would not be read
elif nxt2 is not pf:
pf.armed = False # the expert switches after the next step: carry nothing past it
+
+
+LIST_STREAM_DEPTH = 1
+
+
+class PrefetchedListStreamer:
+ """Streams ``blocks[resident:]`` via pinned slabs and event-fenced copies ``depth`` members ahead, from the second
+ forward on (the first records the order); a forward that raises still releases its queued copies."""
+
+ def __init__(self) -> None:
+ self.handles: list = []
+ self.groups: list = []
+ self.prefetcher: Optional[GroupPrefetcher] = None
+ self.streamed = 0
+ self.pinned_bytes = 0
+ self.arena: Any = None
+ # matches ModuleListStreamer, whose release path clears ``_host``
+ self._host: dict = {}
+
+ def stats(self) -> dict:
+ pf = self.prefetcher
+ return dict(pf.stats) if pf is not None else {}
+
+ def remove(self) -> None:
+ for handle in self.handles:
+ handle.remove()
+ self.handles = []
+ pf = self.prefetcher
+ if pf is not None and getattr(pf, "active", False):
+ pf.end()
+ for group in self.groups:
+ for name in ("onload_", "offload_", "_unsloth_prefetch_onload", "_unsloth_prefetcher"):
+ group.__dict__.pop(name, None)
+ cpu = getattr(group, "cpu_param_dict", None)
+ if isinstance(cpu, dict):
+ cpu.clear()
+ self.groups = []
+ self.prefetcher = None
+ self.arena = None
+
+
+def _list_stream_refusal(device: Any) -> Optional[str]:
+ if not async_prefetch_enabled():
+ return f"{ASYNC_PREFETCH_ENV}=0"
+ import torch
+
+ from .diffusion_memory import GROUP_OFFLOAD_PIN_ENV
+
+ if str(os.environ.get(GROUP_OFFLOAD_PIN_ENV, "")).strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return f"{GROUP_OFFLOAD_PIN_ENV}=0"
+ if torch.device(device).type != "cuda" or not torch.cuda.is_available():
+ return "not a CUDA device"
+ from .host_pinning import host_pinning_allowed
+
+ if not host_pinning_allowed(torch.device(device)):
+ return "pinned host memory is not used on this device"
+ from .diffusion_memory import _pinned_memory_capped
+
+ if _pinned_memory_capped():
+ return "pinned host memory is capped on this OS"
+ return None
+
+
+def stream_module_list(
+ owner: Any,
+ blocks: Any,
+ device: Any,
+ resident: int,
+ *,
+ logger: Any = None,
+ depth: int = LIST_STREAM_DEPTH,
+ reserve_bytes: int = 0,
+ max_pin_bytes: Optional[int] = None,
+) -> Optional[PrefetchedListStreamer]:
+ """A ``PrefetchedListStreamer`` over ``blocks``, or None (nothing changed) when it does not apply or cannot pin.
+ Pins must fit ``_pin_budget_mib`` less ``reserve_bytes``, and ``max_pin_bytes`` if given."""
+ import torch
+
+ refused = _list_stream_refusal(device)
+ blocks = list(blocks)
+ resident = max(0, min(len(blocks), int(resident)))
+ streamed = blocks[resident:]
+ if refused is None and not streamed:
+ refused = "nothing streams"
+ need = 0
+ if refused is None:
+ from .diffusion_memory import _pin_budget_mib, _storage_nbytes
+
+ need = sum(
+ sum(_storage_nbytes(t))
+ for b in streamed
+ for t in list(b.parameters()) + list(b.buffers())
+ )
+ budget = _pin_budget_mib()
+ if max_pin_bytes is not None and need > int(max_pin_bytes):
+ refused = (
+ f"{need / 1e9:.1f} GB to pin exceeds the {max(0, int(max_pin_bytes)) / 1e9:.1f} GB the host plan "
+ "leaves for pinned copies"
+ )
+ elif budget is not None and need + int(reserve_bytes) > int(budget) << 20:
+ refused = (
+ f"{need / 1e9:.1f} GB to pin (+{int(reserve_bytes) / 1e9:.1f} GB reserved) exceeds the "
+ f"{budget / 1024:.1f} GiB host pin budget"
+ )
+ if refused is None:
+ try:
+ go = _go()
+ group_cls = go.ModuleGroup
+ except Exception as exc: # noqa: BLE001 - an older diffusers
+ refused = f"diffusers has no ModuleGroup ({type(exc).__name__})"
+ if refused is not None:
+ if logger is not None:
+ logger.info(
+ "diffusion.memory: %s keeps its pageable streamer (%s)",
+ type(owner).__name__,
+ refused,
+ )
+ return None
+ onload = torch.device(device)
+ streamer = PrefetchedListStreamer()
+ try:
+ from .diffusion_memory import install_group_offload_buffer_restore
+
+ # diffusers restores only parameters; buffer weights (int8 encoder) would otherwise stay on the device
+ install_group_offload_buffer_restore()
+ from .diffusion_pinned_arena import packable_bytes, pinned_arena_for_group_offload
+
+ for block in streamed:
+ block.to("cpu")
+ tensors = [t for b in streamed for t in list(b.parameters()) + list(b.buffers())]
+ try:
+ expected = packable_bytes(tensors)
+ except Exception: # noqa: BLE001 - whole slabs, as without a size
+ expected = None
+ stream = torch.cuda.Stream(device = onload)
+ with pinned_arena_for_group_offload(expected_bytes = expected) as arena:
+ for block in streamed:
+ group = group_cls(
+ modules = [block],
+ offload_device = torch.device("cpu"),
+ onload_device = onload,
+ offload_leader = block,
+ onload_leader = block,
+ non_blocking = True,
+ stream = stream,
+ record_stream = True,
+ low_cpu_mem_usage = False,
+ )
+ type(group).offload_(group)
+ streamer.groups.append(group)
+ streamer.arena = arena
+ for block in blocks[:resident]:
+ block.to(onload)
+ pf = GroupPrefetcher(owner, streamer.groups, onload, depth)
+ pf.fence_first = True
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+
+ def _eager(fn: Any) -> Any:
+ return disable(fn) if callable(disable) else fn
+
+ for group, block in zip(streamer.groups, streamed):
+
+ def onload_(
+ *_a: Any,
+ _g: Any = group,
+ **_k: Any,
+ ) -> None:
+ pf.onload(_g)
+
+ def offload_(
+ *_a: Any,
+ _g: Any = group,
+ **_k: Any,
+ ) -> None:
+ pf.offload(_g)
+
+ group.onload_ = _eager(onload_)
+ group.offload_ = _eager(offload_)
+ group._unsloth_prefetch_onload = group.onload_
+ group._unsloth_prefetcher = pf
+ streamer.handles.append(
+ block.register_forward_pre_hook(_eager(lambda *_a, _g = group: _g.onload_()))
+ )
+ streamer.handles.append(
+ block.register_forward_hook(_eager(lambda *_a, _g = group: _g.offload_()))
+ )
+
+ def _begin(*_a: Any, **_k: Any) -> None:
+ pf.begin()
+ pf.kick() # the first streamed members copy while the resident ones run
+
+ streamer.handles.append(owner.register_forward_pre_hook(_eager(_begin)))
+ streamer.handles.append(
+ owner.register_forward_hook(_eager(lambda *_a, **_k: pf.end()), always_call = True)
+ )
+ streamer.prefetcher = pf
+ streamer.streamed = len(streamed)
+ streamer.pinned_bytes = int(need)
+ bump_placement_epoch()
+ except Exception as exc: # noqa: BLE001 - the caller's pageable streamer takes the members as they are now
+ streamer.remove()
+ for block in streamed:
+ try:
+ block.to("cpu")
+ except Exception: # noqa: BLE001
+ pass
+ if logger is not None:
+ logger.warning(
+ "diffusion.memory: pinned prefetch streaming unavailable for %s (%s)",
+ type(owner).__name__,
+ exc,
+ )
+ return None
+ if logger is not None:
+ payload = getattr(arena, "payload_bytes", 0) if arena is not None else 0
+ logger.info(
+ "diffusion.memory: %s streams %d of %d members with pinned copies (%.2f GB) and event-fenced prefetch "
+ "(%d ahead)",
+ type(owner).__name__,
+ len(streamed),
+ len(blocks),
+ payload / 1e9,
+ depth,
+ )
+ return streamer
diff --git a/studio/backend/core/inference/video_minimax_h3_attn.py b/studio/backend/core/inference/video_minimax_h3_attn.py
index 342e0ecfc6c..5c08917f85f 100644
--- a/studio/backend/core/inference/video_minimax_h3_attn.py
+++ b/studio/backend/core/inference/video_minimax_h3_attn.py
@@ -90,18 +90,30 @@ def _backend_value(backend: Any) -> Optional[str]:
return str(getattr(backend, "value", backend))
+def _diffusers_h3_processor() -> Any:
+ from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3AttnProcessor
+ return MiniMaxH3AttnProcessor
+
+
+def strided_processor_class(base: Any = None) -> Any:
+ """Strided subclass of the H3-layout processor ``base`` (diffusers' when None). One cached class per base: dynamo
+ guards on the class id, so a class per call would retrace every block."""
+ return _strided_processor_for(_diffusers_h3_processor() if base is None else base)
+
+
@lru_cache(maxsize = None)
-def strided_processor_class() -> Any:
- """Strided ``MiniMaxH3AttnProcessor`` subclass. Built once: dynamo guards on the class id, so a class per call
- would retrace every block. Masks, context parallelism and other backends take the stock ``__call__``."""
+def _strided_processor_for(base: Any) -> Any:
+ import sys
+
import torch
import torch.nn.functional as F
- from diffusers.models.transformers.transformer_minimax_h3 import (
- MiniMaxH3AttnProcessor,
- _apply_rotary_emb,
- )
from torch.nn.attention import SDPBackend, sdpa_kernel
+ MiniMaxH3AttnProcessor = base
+ _apply_rotary_emb = getattr(sys.modules.get(base.__module__), "_apply_rotary_emb", None)
+ if not callable(_apply_rotary_emb):
+ raise TypeError(f"{base.__name__}: its module has no _apply_rotary_emb")
+
from . import diffusion_int8_gemm as int8_gemm
def _fusable_norm(norm: Any) -> bool:
@@ -189,30 +201,72 @@ def strided_processor_class() -> Any:
return UnslothH3StridedAttnProcessor
-def _h3_attention_modules(transformer: Any) -> list:
+_H3_ATTN_FORWARD = ("self", "hidden_states", "rotary_emb", "attention_mask")
+_H3_PROC_CALL = ("self", "attn", "hidden_states", "rotary_emb", "attention_mask")
+
+
+@lru_cache(maxsize = None)
+def _params(fn: Any) -> tuple:
+ import inspect
try:
- from diffusers.models.transformers.transformer_minimax_h3 import MiniMaxH3Attention
- except Exception: # noqa: BLE001
- return []
+ return tuple(inspect.signature(fn).parameters)
+ except (TypeError, ValueError):
+ return ()
+
+
+def h3_processor_base(module: Any) -> Any:
+ """The stock H3-layout processor class of ``module``, or None. Gated on structure (forward signature, norms, QKV,
+ a processor module defining ``_apply_rotary_emb``), not class, so the vendored H3-Image attention matches too."""
+ import sys
+
+ from torch import nn
+
+ cls = type(module)
+ base = getattr(cls, "_default_processor_cls", None)
+ if not isinstance(module, nn.Module) or not isinstance(base, type):
+ return None
+ if _params(cls.forward) != _H3_ATTN_FORWARD or _params(base.__call__) != _H3_PROC_CALL:
+ return None
+ if not callable(getattr(sys.modules.get(base.__module__), "_apply_rotary_emb", None)):
+ return None
+ if not all(isinstance(getattr(module, n, None), nn.RMSNorm) for n in ("norm_q", "norm_k")):
+ return None
+ if not isinstance(getattr(module, "to_out", None), nn.ModuleList) or not callable(
+ getattr(module, "set_processor", None)
+ ):
+ return None
+ fused = getattr(module, "fused_projections", False)
+ if not (fused and hasattr(module, "to_qkv")) and not all(
+ hasattr(module, n) for n in ("to_q", "to_k", "to_v")
+ ):
+ return None
+ if not isinstance(getattr(module, "processor", None), base):
+ return None
+ return base
+
+
+def _h3_attention_modules(transformer: Any) -> list:
modules = getattr(transformer, "modules", None)
if not callable(modules):
return []
- return [m for m in modules() if isinstance(m, MiniMaxH3Attention)]
+ return [m for m in modules() if h3_processor_base(m) is not None]
-def install_strided_attention(transformer: Any, logger: Any = None) -> int:
- """Swap every H3 attention processor for the strided one, keeping its backend. Returns the count."""
+def install_strided_attention(
+ transformer: Any,
+ logger: Any = None,
+ qk_rope: Optional[bool] = None,
+) -> int:
+ """Swap every H3 attention processor for the strided one, keeping its backend. Returns the count.
+ ``qk_rope=False`` keeps the stock q/k norm + rope; ``None`` follows UNSLOTH_H3_QK_ROPE."""
if not _enabled(STRIDED_ATTN_ENV):
return 0
- try:
- cls = strided_processor_class()
- except Exception as exc: # noqa: BLE001 -- older diffusers without the H3 processor
- if logger is not None:
- logger.info("video.h3_attn: strided processor unavailable, keeping stock: %s", exc)
+ modules = _h3_attention_modules(transformer)
+ if not modules:
return 0
from .video_minimax_h3_qknorm import qk_norm_rope_op, qk_rope_enabled
- qk_rope_on = qk_rope_enabled()
+ qk_rope_on = qk_rope_enabled() and qk_rope is not False
if qk_rope_on:
try:
qk_norm_rope_op()
@@ -221,8 +275,14 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:
if logger is not None:
logger.info("video.h3_attn: fused q/k norm+rope unavailable: %s", exc)
swapped = 0
- for module in _h3_attention_modules(transformer):
+ for module in modules:
old = getattr(module, "processor", None)
+ try:
+ cls = strided_processor_class(h3_processor_base(module))
+ except Exception as exc: # noqa: BLE001 -- older diffusers without the H3 processor
+ if logger is not None:
+ logger.info("video.h3_attn: strided processor unavailable, keeping stock: %s", exc)
+ return swapped
if old is None or isinstance(old, cls):
continue
new = cls()
@@ -239,15 +299,14 @@ def install_strided_attention(transformer: Any, logger: Any = None) -> int:
def strided_attention_count(transformer: Any) -> int:
"""Engagement census."""
- try:
- cls = strided_processor_class()
- except Exception: # noqa: BLE001
- return 0
- return sum(
- 1
- for m in _h3_attention_modules(transformer)
- if isinstance(getattr(m, "processor", None), cls)
- )
+ count = 0
+ for m in _h3_attention_modules(transformer):
+ try:
+ cls = strided_processor_class(h3_processor_base(m))
+ except Exception: # noqa: BLE001
+ return 0
+ count += isinstance(getattr(m, "processor", None), cls)
+ return count
def shared_qkv_count(transformer: Any) -> int:
diff --git a/studio/backend/tests/test_diffusion_list_streamer.py b/studio/backend/tests/test_diffusion_list_streamer.py
new file mode 100644
index 00000000000..9a0a86d19e0
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_list_streamer.py
@@ -0,0 +1,279 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The shared pinned ModuleList streamer: CPU refusals, plus CUDA parity / prefetch / remove() checks."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+import core.inference.diffusion_offload_prefetch as op
+
+
+class _Stop(Exception):
+ pass
+
+
+class _BufferLinear(torch.nn.Module):
+ def __init__(self, width: int) -> None:
+ super().__init__()
+ self.register_buffer("weight", torch.randint(-127, 127, (width, width), dtype = torch.int8))
+ self.register_buffer("weight_scale", torch.rand(width, 1) / 64)
+
+ def forward(self, x):
+ return x @ (self.weight.to(x.dtype) * self.weight_scale.to(x.dtype)).t()
+
+
+class _Stack(torch.nn.Module):
+ def __init__(
+ self,
+ n: int = 6,
+ width: int = 64,
+ ) -> None:
+ super().__init__()
+ torch.manual_seed(0)
+ self.inp = torch.nn.Linear(width, width)
+ self.layers = torch.nn.ModuleList(
+ torch.nn.Sequential(
+ torch.nn.Linear(width, width * 2),
+ torch.nn.GELU(),
+ torch.nn.Linear(width * 2, width),
+ _BufferLinear(width),
+ )
+ for _ in range(n)
+ )
+ self.stop_after = None
+
+ def forward(self, x):
+ x = self.inp(x)
+ for i, layer in enumerate(self.layers):
+ x = x + layer(x)
+ if self.stop_after is not None and i == self.stop_after:
+ raise _Stop()
+ return x
+
+
+def test_cpu_device_is_refused_and_nothing_changes():
+ stack = _Stack()
+ before = [p.data_ptr() for p in stack.parameters()]
+ assert op.stream_module_list(stack, stack.layers, "cpu", 2) is None
+ assert [p.data_ptr() for p in stack.parameters()] == before
+ assert not stack._forward_pre_hooks and not stack.layers[3]._forward_pre_hooks
+
+
+def test_kill_switch_refuses(monkeypatch):
+ monkeypatch.setenv(op.ASYNC_PREFETCH_ENV, "0")
+ assert op._list_stream_refusal("cuda").startswith(op.ASYNC_PREFETCH_ENV)
+
+
+def test_h3_image_falls_back_to_the_pageable_streamer_off_cuda(monkeypatch):
+ pytest.importorskip("diffusers")
+ from core.inference import diffusion_minimax_h3_image as h3i
+
+ class _Pipe:
+ _unsloth_h3_streamers: list = []
+
+ pipe = _Pipe()
+ pipe._unsloth_h3_streamers = []
+ stack = _Stack()
+ streamer = h3i.stream_list(pipe, stack, stack.layers, "cpu", 2, "transformer")
+ assert isinstance(streamer, h3i.ModuleListStreamer) and streamer.streamed == 4
+ assert pipe._unsloth_h3_streamers == [streamer] and streamer._unsloth_role == "transformer"
+ monkeypatch.setenv(h3i.PINNED_STREAM_ENV, "0")
+ assert h3i.stream_inflight() == 1
+ monkeypatch.delenv(h3i.PINNED_STREAM_ENV)
+ assert h3i.stream_inflight() == 1 + op.LIST_STREAM_DEPTH
+
+
+def test_a_stream_larger_than_the_pin_budget_is_refused(monkeypatch):
+ import core.inference.diffusion_memory as dm
+
+ stack = _Stack()
+ went_on = []
+ monkeypatch.setattr(op, "_list_stream_refusal", lambda device: None)
+ # no diffusers needed: the budget check runs before ModuleGroup is looked up
+ monkeypatch.setattr(op, "_go", lambda: type("_Go", (), {"ModuleGroup": object}))
+ # the refusal must come from the budget, before any pinning
+ monkeypatch.setattr(
+ dm, "install_group_offload_buffer_restore", lambda: went_on.append(1) or 1 / 0
+ )
+ # compared in bytes, not rounded MiB
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 0)
+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None
+ assert not went_on and not stack.layers[3]._forward_pre_hooks
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 1)
+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None
+ assert went_on == [1] and not stack.layers[3]._forward_pre_hooks
+
+
+def test_the_plan_keeps_room_for_the_prefetched_member():
+ pytest.importorskip("diffusers")
+ from core.inference import diffusion_minimax_h3_image as h3i
+
+ args = dict(vram_gb = 23.46, host_gb = 53.97, dit_gb = 21.9, te_gb = 27.2)
+ one = h3i.plan_placement(**args)
+ two = h3i.plan_placement(**args, inflight = 2)
+ assert one.placement == two.placement == h3i.PLACE_STREAM
+ assert one.dit_resident_blocks == 28 # the #17453 L4 plan, unchanged at one in flight
+ assert two.dit_resident_blocks in (26, 27)
+ a100 = dict(vram_gb = 41.96, host_gb = 86.55, dit_gb = 21.9, te_gb = 27.2)
+ assert h3i.plan_placement(**a100).te_resident_layers == 20
+ assert h3i.plan_placement(**a100, inflight = 2).te_resident_layers == 19
+
+
+cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+
+@cuda
[email protected]("resident", [0, 2, 5])
+def test_streamed_stack_is_bit_identical_and_prefetches(resident):
+ ref = _Stack().cuda().eval()
+ stack = _Stack().eval()
+ stack.inp.cuda()
+ streamer = op.stream_module_list(stack, stack.layers, "cuda", resident)
+ assert streamer is not None and streamer.streamed == 6 - resident
+ for g in streamer.groups:
+ assert all(t.is_pinned() for t in g.cpu_param_dict.values())
+ x = torch.randn(8, 64, device = "cuda")
+ with torch.no_grad():
+ want = ref(x)
+ torch.cuda.synchronize()
+ before = torch.cuda.memory_allocated()
+ for _ in range(3):
+ assert torch.equal(stack(x), want)
+ stack.stop_after = 3
+ with pytest.raises(_Stop):
+ stack(x)
+ stack.stop_after = None
+ assert torch.equal(stack(x), want)
+ torch.cuda.synchronize()
+ stats = streamer.stats()
+ assert stats["forwards"] == 5 and stats["prefetched"] > 0
+ assert streamer.prefetcher.inflight_bytes == 0 and not streamer.prefetcher.ready
+ for layer in stack.layers[resident:]:
+ assert all(p.device.type == "cpu" for p in layer.parameters())
+ assert all(b.device.type == "cpu" for b in layer.buffers())
+ assert torch.cuda.memory_allocated() <= before + (1 << 20)
+ streamer.remove()
+ assert not stack._forward_pre_hooks and not stack._forward_hooks
+ with torch.no_grad():
+ stack.cuda()
+ assert torch.equal(stack(x), want)
+
+
+def test_a_stream_over_the_callers_host_cap_is_refused_before_pinning(monkeypatch):
+ import core.inference.diffusion_memory as dm
+
+ stack = _Stack()
+ went_on = []
+ monkeypatch.setattr(op, "_list_stream_refusal", lambda device: None)
+ monkeypatch.setattr(
+ dm, "install_group_offload_buffer_restore", lambda: went_on.append(1) or 1 / 0
+ )
+ monkeypatch.setattr(dm, "_pin_budget_mib", lambda: 1 << 20)
+ assert op.stream_module_list(stack, stack.layers, "cuda", 2, max_pin_bytes = 0) is None
+ assert not went_on
+
+
+def _h3():
+ pytest.importorskip("diffusers")
+ from core.inference import diffusion_minimax_h3_image as h3i
+ return h3i
+
+
+ "host, dit_room, te_pinned",
+ [
+ (
+ 86.55,
+ True,
+ True,
+ ), # A100 VM: DiT resident, the encoder's streamed layers fit the plan's room
+ (
+ 53.97,
+ True,
+ False,
+ ), # L4 VM: the streamed DiT blocks pin, the 24 GB encoder stays pageable
+ (
+ 40.0,
+ False,
+ False,
+ ), # 40 GB host (the smallest that loads on L4): nothing pins, everything streams pageable
+ ],
+)
+def test_pins_fit_the_plans_host_room(host, dit_room, te_pinned):
+ """Pinned copies are counted on top of what the plan holds on the host, so a host the plan fits still fits."""
+ h3i = _h3()
+ vram = 41.96 if host > 80 else 23.46
+ plan = h3i.plan_placement(vram_gb = vram, host_gb = host, dit_gb = 21.9, te_gb = 27.2, inflight = 2)
+ room = h3i.pin_room_bytes(plan)
+ assert room is not None and room >= 0
+ assert room == max(0, int((host - plan.host_need_gb) * 1e9))
+ dit_stream = (
+ (50 - int(plan.dit_resident_blocks or 50)) * 0.398e9
+ if plan.placement == h3i.PLACE_STREAM
+ else 0
+ )
+ te_stream = (51 - int(plan.te_resident_layers or 0)) * 0.478e9
+ if plan.placement == h3i.PLACE_STREAM:
+ assert (dit_stream <= room) == dit_room
+
+ class _Pinned:
+ pinned_bytes = int(dit_stream) if dit_room else 0
+
+ left = h3i.pin_room_bytes(plan, [_Pinned()])
+ assert (plan.te_kept_on_host and te_stream <= left) == te_pinned
+ assert (
+ h3i.pin_room_bytes(h3i.plan_placement(vram_gb = vram, host_gb = None, dit_gb = 21.9, te_gb = 27.2))
+ is None
+ )
+
+
+def test_a_small_host_streams_pageable_and_still_renders(monkeypatch):
+ """40 GB host, stream tier (about 37 GB is the planner floor): the DiT gets no pin room and runs pageable."""
+ h3i = _h3()
+ import core.inference.diffusion_offload_prefetch as opf
+
+ seen = []
+ real = opf.stream_module_list
+ monkeypatch.setattr(
+ opf,
+ "stream_module_list",
+ lambda *a, **k: seen.append(k.get("max_pin_bytes")) or real(*a, **k),
+ )
+ with pytest.raises(RuntimeError, match = "free system memory"):
+ h3i.plan_placement(vram_gb = 23.46, host_gb = 32.0, dit_gb = 21.9, te_gb = 27.2, inflight = 2)
+ plan = h3i.plan_placement(vram_gb = 23.46, host_gb = 40.0, dit_gb = 21.9, te_gb = 27.2, inflight = 2)
+ assert plan.placement == h3i.PLACE_STREAM and not plan.te_kept_on_host
+ stack = _Stack()
+
+ class _T(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.inp = stack.inp
+ self.transformer_blocks = stack.layers
+
+ class _V(torch.nn.Module):
+ pass
+
+ t = _T()
+ streamer = h3i._place_streamed_dit(t, _V(), "cpu", plan)
+ assert seen == [0] and isinstance(streamer, h3i.ModuleListStreamer)
+ x = torch.randn(4, 64)
+ with torch.no_grad():
+ out = stack(x)
+ streamer.remove()
+ with torch.no_grad():
+ assert torch.equal(out, stack(x))
+
+
[email protected]("value", ["0", "off"])
+def test_the_global_pin_opt_out_keeps_the_pageable_streamer(monkeypatch, value):
+ import core.inference.diffusion_memory as dm
+
+ monkeypatch.setenv(dm.GROUP_OFFLOAD_PIN_ENV, value)
+ assert op._list_stream_refusal("cuda") == f"{dm.GROUP_OFFLOAD_PIN_ENV}=0"
+ stack = _Stack()
+ assert op.stream_module_list(stack, stack.layers, "cuda", 2) is None
+ assert not stack.layers[3]._forward_pre_hooks
diff --git a/studio/backend/tests/test_diffusion_minimax_h3_image.py b/studio/backend/tests/test_diffusion_minimax_h3_image.py
index 400cb56a1bc..361dfdae04f 100644
--- a/studio/backend/tests/test_diffusion_minimax_h3_image.py
+++ b/studio/backend/tests/test_diffusion_minimax_h3_image.py
@@ -611,8 +611,18 @@ def test_backend_loads_and_generates(h3i, release, tmp_path, monkeypatch):
status = backend.load_pipeline(str(root), model_kind = "pipeline")
assert backend._state is not None and backend._state.family.name == "minimax-h3-image"
assert backend._state.pipe._unsloth_h3_report.layout == "native"
+ from core.inference import diffusion_prompt_cache as prompt_cache
+
+ cache = prompt_cache.cache_for(backend._state.pipe)
+ assert cache is not None, "the H3-Image load must install the prompt cache"
out = backend.generate(prompt = "a cat", width = 256, height = 256, steps = 6, guidance = 1.0, seed = 3)
assert out["images"], out
+ assert cache.stats["misses"] == 1 and cache.stats["hits"] == 0
+ again = backend.generate(
+ prompt = "a cat", width = 256, height = 256, steps = 6, guidance = 1.0, seed = 4
+ )
+ assert again["images"] and cache.stats["hits"] == 1
+ assert backend._state.pipe._unsloth_h3_last_stats["encode_cached"] is True
raw = out["images"][0]
data = raw.get("b64") if isinstance(raw, dict) else raw
if isinstance(data, str):
@@ -1139,3 +1149,163 @@ def test_the_per_prompt_encoder_refuses_a_bf16_fallback(h3i, release, tmp_path,
with pytest.raises(RuntimeError, match = "bf16 encoder it would fall back to"):
pipe._unsloth_h3_te_loader()
pipe.release()
+
+
+def _cached_pipe(h3i, release):
+ from core.inference import diffusion_prompt_cache as prompt_cache
+
+ root, _ref, te = release
+ p = h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = te)
+ assert prompt_cache.install(p, identity = {"family": "minimax-h3-image"})
+ return p, prompt_cache.cache_for(p)
+
+
+def _latents(
+ p,
+ prompt,
+ seed,
+ image = None,
+):
+ import torch
+ out = p(
+ prompt = prompt,
+ image = image,
+ height = 256,
+ width = 256,
+ generator = torch.Generator("cpu").manual_seed(seed),
+ output_type = "latent",
+ )
+ return out.images[0]
+
+
+def test_a_repeated_prompt_reuses_the_conditioning_bitwise(h3i, release, monkeypatch):
+ import torch
+
+ p, cache = _cached_pipe(h3i, release)
+ calls = []
+ real = h3i.qwen3vl_hidden_state
+ monkeypatch.setattr(
+ h3i, "qwen3vl_hidden_state", lambda *a, **k: calls.append(1) or real(*a, **k)
+ )
+ first = _latents(p, "a red bicycle", 5)
+ hit = _latents(p, "a red bicycle", 5)
+ assert (
+ len(calls) == 1 and cache.stats["hits"] == 1 and p._unsloth_h3_last_stats["encode_cached"]
+ )
+ assert torch.equal(first, hit)
+ _latents(p, "a blue bicycle", 5)
+ assert len(calls) == 2 and not p._unsloth_h3_last_stats["encode_cached"]
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_PROMPT_CACHE", "0")
+ from core.inference import diffusion_prompt_cache as prompt_cache
+
+ q = h3i.load_minimax_h3_image_pipeline(str(release[0]), device = "cpu", text_encoder = release[2])
+ assert not prompt_cache.install(q, identity = {"family": "minimax-h3-image"})
+ assert torch.equal(_latents(q, "a red bicycle", 5), first)
+
+
+def test_the_conditioning_cache_keys_on_the_reference_pixels(h3i, release, monkeypatch):
+ from PIL import Image
+
+ p, cache = _cached_pipe(h3i, release)
+ seen = []
+ real = h3i.qwen3vl_hidden_state
+ monkeypatch.setattr(
+ h3i, "qwen3vl_hidden_state", lambda *a, **k: seen.append(1) or real(*a, **k)
+ )
+ if p.processor is None:
+ pytest.skip("tiny release has no processor")
+ red = Image.new("RGB", (128, 128), (200, 10, 10))
+ blue = Image.new("RGB", (128, 128), (10, 10, 200))
+ _latents(p, "<Picture 1> at night", 1, image = [red])
+ _latents(p, "<Picture 1> at night", 2, image = [red.copy()])
+ assert len(seen) == 1 and cache.stats["hits"] == 1
+ _latents(p, "<Picture 1> at night", 1, image = [blue])
+ assert len(seen) == 2
+
+
+def test_every_backend_loader_installs_the_prompt_cache():
+ """Each DiffusionBackend method that commits a load (builds a _LoadState) installs the conditioning cache."""
+ import ast
+
+ path = _BACKEND / "core" / "inference" / "diffusion.py"
+ src = path.read_text(encoding = "utf-8")
+ loaders = []
+ for node in ast.walk(ast.parse(src)):
+ if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ calls = {ast.unparse(c.func) for c in ast.walk(node) if isinstance(c, ast.Call)}
+ if "_LoadState" in calls:
+ loaders.append(node.name)
+ assert (
+ "prompt_cache.install" in calls
+ ), f"{node.name} commits a load without the prompt cache"
+ assert {"load_pipeline", "_load_h3_image_locked"} <= set(loaders), loaders
+
+
+def test_a_declared_int8_attention_maps_to_sage():
+ from core.inference import diffusion_attention as da
+
+ assert da.checkpoint_attention_backend("int8") == "sage"
+ assert da.checkpoint_attention_backend(" INT8 ") == "sage"
+ assert da.checkpoint_attention_backend("fp4") is None
+ assert da.checkpoint_attention_backend(None) is None
+
+
+def test_int8_attention_follows_the_checkpoint_and_the_switch(h3i, monkeypatch):
+ import types
+
+ t = types.SimpleNamespace()
+ monkeypatch.delenv(h3i.INT8_ATTENTION_ENV, raising = False)
+ assert h3i.checkpoint_attention_wanted(t, "int8") == "sage"
+ assert h3i.checkpoint_attention_wanted(t, "fp8") is None
+ assert h3i.checkpoint_attention_wanted(t, None) is None
+ setattr(t, h3i.CHECKPOINT_ATTENTION_ATTR, "int8")
+ assert h3i.checkpoint_attention_wanted(t, "fp8") == "sage"
+ monkeypatch.setenv(h3i.INT8_ATTENTION_ENV, "0")
+ assert h3i.checkpoint_attention_wanted(t, "int8") is None
+ monkeypatch.setenv(h3i.INT8_ATTENTION_ENV, "1")
+ assert h3i.checkpoint_attention_wanted(types.SimpleNamespace(), None) == "sage"
+
+
+def test_int8_attention_engages_on_the_blocks_only_and_comes_off(h3i, release, monkeypatch):
+ from core.inference import diffusion_attention as da
+
+ root, _ref, te = release
+ p = h3i.load_minimax_h3_image_pipeline(str(root), device = "cpu", text_encoder = te)
+ mods = h3i._attention_processors(p.transformer)
+ blocks = {id(m) for b in p.transformer.transformer_blocks for m in b.modules()}
+ before = {id(m): m.processor._attention_backend for m in mods}
+
+ def fake_apply(
+ pipe,
+ backend,
+ logger = None,
+ target = None,
+ ):
+ for m in h3i._attention_processors(pipe.transformer):
+ m.processor._attention_backend = backend
+ return backend
+
+ monkeypatch.setattr(da, "apply_attention_backend", fake_apply)
+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") == "sage"
+ for m in mods:
+ want = "sage" if id(m) in blocks else before[id(m)]
+ assert m.processor._attention_backend == want
+ assert any(id(m) not in blocks for m in mods)
+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") is None
+ assert h3i.remove_checkpoint_attention(p) == len(mods)
+ assert {id(m): m.processor._attention_backend for m in mods} == before
+ monkeypatch.setattr(
+ da, "apply_attention_backend", lambda *a, **k: fake_apply(a[0], None) and None
+ )
+ assert h3i.install_checkpoint_attention(p, "cuda", scheme = "int8") is None
+ assert {id(m): m.processor._attention_backend for m in mods} == before
+ assert h3i.install_checkpoint_attention(p, "cpu", scheme = "int8") is None
+
+
+def test_hosted_metadata_attention_is_recorded(h3i):
+ import types
+
+ t = types.SimpleNamespace()
+ h3i.h3_image_prepare_prequant_model()(t, {"attention": "INT8"})
+ assert getattr(t, h3i.CHECKPOINT_ATTENTION_ATTR) == "int8"
diff --git a/studio/backend/tests/test_diffusion_minimax_h3_image_speed.py b/studio/backend/tests/test_diffusion_minimax_h3_image_speed.py
new file mode 100644
index 00000000000..a5aa2a19881
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_minimax_h3_image_speed.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""MiniMax-H3-Image speed levers on a tiny random H3-Image DiT (CPU)."""
+
+from __future__ import annotations
+
+import copy
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parents[1]
+for p in (_BACKEND, Path(__file__).resolve().parent):
+ if str(p) not in sys.path:
+ sys.path.insert(0, str(p))
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("diffusers")
+pytest.importorskip("accelerate")
+
+from core.inference import diffusion_minimax_h3_image as h3i # noqa: E402
+from core.inference import diffusion_minimax_h3_image_speed as S # noqa: E402
+from core.inference import video_minimax_h3_attn as A # noqa: E402
+
+
[email protected](autouse = True)
+def _restore_backend():
+ from diffusers.models.attention_dispatch import _AttentionBackendRegistry
+
+ active = _AttentionBackendRegistry._active_backend
+ yield
+ _AttentionBackendRegistry.set_active_backend(active)
+
+
+def _model(dtype = torch.float32):
+ import _minimax_h3_image_tiny as tiny
+
+ cfg = h3i.transformer_config_from_native(tiny.TINY_TRANSFORMER_NATIVE)
+ torch.manual_seed(0)
+ model = h3i.transformer_class()(**cfg).eval()
+ tiny.randomize(model, 0)
+ return model.to(dtype)
+
+
+def _inputs(
+ model,
+ dtype = torch.float32,
+ n_text = 5,
+ lh = 8,
+ lw = 8,
+):
+ g = torch.Generator().manual_seed(1)
+ patch = tuple(model.config.patch_size)
+ tags = torch.full((n_text,), h3i.H3_IMAGE_TEXT_TAG, dtype = torch.long)
+ pos, tags, image_idx, text_idx, _n_ref = h3i.build_packed_layout(tags, [], lh, lw, patch)
+ seq = int(image_idx.numel() + text_idx.numel())
+ unique, index = h3i.row_timestep_plan(torch.tensor([0.75]), seq, image_idx[:0])[0]
+ noise = torch.randn((1, 24, 1, lh, lw), generator = g)
+ text_dim = int(model.context_embedder.in_features)
+ return dict(
+ hidden_states = h3i.patchify(noise, patch)[None].to(dtype),
+ encoder_hidden_states = torch.randn(1, n_text, text_dim, generator = g).to(dtype),
+ timestep = unique,
+ timestep_indices = index,
+ token_tags = tags,
+ position_ids = pos,
+ image_indices = image_idx,
+ text_indices = text_idx,
+ pdd_step_index = 0,
+ return_dict = False,
+ )
+
+
+def _run(model, inputs):
+ with torch.no_grad():
+ return model(**inputs)[0]
+
+
+def _quantized_padded(model):
+ pytest.importorskip("torchao")
+ from torchao.quantization import quantize_
+
+ from core.inference.diffusion_transformer_quant import _make_quant_config
+
+ model = model.to(torch.bfloat16)
+ quantize_(
+ model,
+ _make_quant_config("int8"),
+ filter_fn = lambda m, fqn: isinstance(m, torch.nn.Linear)
+ and fqn.startswith("transformer_blocks.")
+ and fqn.endswith(h3i.QUANT_LINEAR_SUFFIXES),
+ )
+ wrapped = sum(
+ h3i.wrap_row_multiple(b, h3i.QUANT_LINEAR_SUFFIXES) for b in model.transformer_blocks
+ )
+ if not wrapped:
+ pytest.skip("this torchao gives no per-row int8 Linears")
+ return model
+
+
+def test_the_vendored_attention_is_recognised_by_structure():
+ model = _model()
+ mods = A._h3_attention_modules(model)
+ assert len(mods) == 4
+ base = A.h3_processor_base(mods[0])
+ assert base is type(model.transformer_blocks[0].attn.processor)
+ cls = A.strided_processor_class(base)
+ assert issubclass(cls, base) and cls is A.strided_processor_class(base)
+ assert A.h3_processor_base(model.transformer_blocks[0].ff) is None
+ assert A.h3_processor_base(torch.nn.Linear(2, 2)) is None
+
+
[email protected]("qk_rope", ["1", "0"])
+def test_strided_processor_matches_stock_bit_for_bit(monkeypatch, qk_rope):
+ monkeypatch.setenv("UNSLOTH_H3_QK_ROPE", qk_rope)
+ model = _model()
+ model.set_attention_backend("_native_math")
+ stock = copy.deepcopy(model)
+ assert A.install_strided_attention(model) == 4
+ assert A.strided_attention_count(model) == 4
+ proc = model.transformer_blocks[0].attn.processor
+ assert A._backend_value(proc._attention_backend) == "_native_math"
+ assert proc._unsloth_qk_rope is (qk_rope == "1")
+ inputs = _inputs(model)
+ assert torch.equal(_run(stock, inputs), _run(model, inputs))
+
+
+ "resident, image_env, video_env, want",
+ [
+ (False, None, None, False),
+ (True, None, None, True),
+ (False, "1", None, True),
+ (True, "0", None, False),
+ (True, None, "0", False),
+ (False, "1", "0", False),
+ ],
+)
+def test_fused_qk_rope_default_follows_the_tier(monkeypatch, resident, image_env, video_env, want):
+ for env, value in ((S.QK_ROPE_ENV, image_env), ("UNSLOTH_H3_QK_ROPE", video_env)):
+ if value is None:
+ monkeypatch.delenv(env, raising = False)
+ else:
+ monkeypatch.setenv(env, value)
+ monkeypatch.setattr(
+ "core.inference.video_minimax_h3_qknorm.qk_norm_rope_op", lambda: None, raising = False
+ )
+ model = _model()
+ assert A.install_strided_attention(model, qk_rope = S.image_qk_rope_enabled(resident)) == 4
+ assert model.transformer_blocks[0].attn.processor._unsloth_qk_rope is want
+
+
+def test_strided_processor_defers_to_stock_for_an_unpinned_backend():
+ model = _model()
+ stock = copy.deepcopy(model)
+ A.install_strided_attention(model)
+ for m in A._h3_attention_modules(model):
+ m.processor._attention_backend = None
+ inputs = _inputs(model)
+ assert torch.equal(_run(stock, inputs), _run(model, inputs))
+
+
+def test_strided_kill_switch(monkeypatch):
+ monkeypatch.setenv(A.STRIDED_ATTN_ENV, "0")
+ model = _model()
+ assert A.install_strided_attention(model) == 0
+ assert A.strided_attention_count(model) == 0
+
+
+def test_int8_qkv_fuses_through_the_row_pads_exactly():
+ model = _quantized_padded(_model())
+ stock = copy.deepcopy(model)
+ fused = S._fuse_qkv(model, None)
+ assert fused == 2
+ attn = model.transformer_blocks[0].attn
+ assert attn.fused_projections and not hasattr(attn, "to_q")
+ assert type(attn.to_qkv).__name__ == "H3ImageRowPad"
+ assert type(attn.to_out[0]).__name__ == "H3ImageRowPad"
+ assert not model.token_refiner.refiner_blocks[0].attn.fused_projections
+ inputs = _inputs(model, torch.bfloat16)
+ assert torch.equal(_run(stock, inputs), _run(model, inputs))
+
+
+def test_fused_qkv_kill_switch_keeps_the_pads(monkeypatch):
+ monkeypatch.setenv(A.FUSED_QKV_ENV, "0")
+ model = _quantized_padded(_model())
+ assert S._fuse_qkv(model, None) == 0
+ attn = model.transformer_blocks[0].attn
+ assert not attn.fused_projections
+ assert all(type(getattr(attn, n)).__name__ == "H3ImageRowPad" for n in ("to_q", "to_k", "to_v"))
+
+
+def test_shared_qkv_needs_the_int8_swap():
+ model = _quantized_padded(_model())
+ A.install_strided_attention(model)
+ assert A.shared_qkv_count(model) == 0
+
+
+def test_install_is_off_off_cuda_and_under_the_umbrella_switch(monkeypatch):
+ model = _model()
+ pipe = type("P", (), {})()
+ pipe.transformer = model
+ plan = type("Plan", (), {"placement": "dit_resident"})()
+ assert S.install_h3_image_speed(pipe, plan, "cpu") == ()
+ assert pipe._unsloth_h3_speed_levers == ()
+ monkeypatch.setenv(S.SPEED_ENV, "0")
+ assert not S.speed_enabled()
+ assert S.install_h3_image_speed(pipe, plan, "cuda") == ()
+ assert A.strided_attention_count(model) == 0
+
+
+def test_compile_is_opt_in(monkeypatch):
+ monkeypatch.delenv(S.COMPILE_ENV, raising = False)
+ assert not S.compile_enabled()
+ monkeypatch.setenv(S.COMPILE_ENV, "1")
+ assert S.compile_enabled()
+
+
+def test_the_loader_installs_the_levers_once_and_reports_them():
+ import ast
+ import inspect
+
+ src = inspect.getsource(h3i.load_minimax_h3_image_pipeline)
+ calls = [
+ n
+ for n in ast.walk(ast.parse(src))
+ if isinstance(n, ast.Call) and getattr(n.func, "id", None) == "install_h3_image_speed"
+ ]
+ assert len(calls) == 1
+ assert src.index("install_h3_image_speed(") > src.rindex("early_streamer.remove()")
+ assert src.index("install_h3_image_speed(") < src.index("H3ImageLoadReport(")
+ assert "speed_levers" in h3i.H3ImageLoadReport.__dataclass_fields__