Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion training: no per-micro-step host syncs, one decode per cached image, streamed int8 base load (#16763)

Daniel Handa412c072cd9
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 6274b36fd54..c46f60bf3dd 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -97,7 +97,9 @@ from core.training.diffusion_samples import (
from core.training.diffusion_train_extras import (
BucketBatchSampler,
LoRAEMA,
+ accumulate_loss,
cover_resize_dims,
+ loss_value,
PersistentConditioningCache,
save_ema_adapter,
)
@@ -183,7 +185,7 @@ class _FamilySpec:
def _gather_sigmas(sigma_table, indices, device, dtype, n_dim):
"""Gathers by index, so sigma_table must stay aligned with the timesteps the indices were drawn from."""
- sigma = sigma_table[indices].to(device = device, dtype = dtype).flatten()
+ sigma = sigma_table[indices].to(device = device, non_blocking = True).to(dtype = dtype).flatten()
while sigma.ndim < n_dim:
sigma = sigma.unsqueeze(-1)
return sigma
@@ -253,10 +255,19 @@ def _sample_timesteps(scheduler, batch_size, device):
)
num_train = scheduler.config.num_train_timesteps
indices = (u * num_train).long().clamp(0, num_train - 1)
- timesteps = scheduler.timesteps.to(device)[indices].to(device)
+ # Gather on host then one async copy; device-table indexing with host indices blocks.
+ timesteps = scheduler.timesteps[indices].to(device, non_blocking = True)
return timesteps, indices
+def _empty_device_cache(device) -> None:
+ import torch
+ if device == "cuda":
+ torch.cuda.empty_cache()
+ elif device == "xpu":
+ torch.xpu.empty_cache()
+
+
def _encoders_to_device(pipe, device) -> None:
"""A QLoRA load leaves text encoders on CPU, so move them; a 4-bit encoder already placed is skipped."""
for attr in ("text_encoder", "text_encoder_2", "text_encoder_3"):
@@ -310,7 +321,8 @@ def _load_pipe_without_transformer(pipe_cls, cfg, device):
torch_dtype = torch.bfloat16,
token = cfg.hf_token,
)
- pipe.vae.to(device, dtype = torch.float32)
+ # Stays on host until the text encoders are freed (never resident beside them).
+ pipe.vae.to(dtype = torch.float32)
return pipe, pipe.vae
@@ -344,12 +356,14 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):
# int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear, whose peft-0.18
# constructor rejects the torchao-0.16 config API (peft 0.19+ accepts it; the order works for both).
- return transformer_cls.from_pretrained(
+ transformer = transformer_cls.from_pretrained(
cfg.base_model,
subfolder = "transformer",
torch_dtype = torch.bfloat16,
token = cfg.hf_token,
- ).to(device)
+ )
+ # int8 stays on host: _int8_quantize_base streams Linears to device so the bf16 base is never whole in VRAM.
+ return transformer if base_precision == "int8" else transformer.to(device)
def _load_fp8_prequant_transformer(transformer_cls, cfg, device):
@@ -378,8 +392,13 @@ def _load_fp8_prequant_transformer(transformer_cls, cfg, device):
return transformer
-def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:
- """Passing family keeps training and inference on one exclusion list, else the first forward raises."""
+def _int8_quantize_base(
+ transformer,
+ family: Optional[str] = None,
+ device = None,
+) -> None:
+ """Passing family keeps training and inference on one exclusion list, else the first forward raises. With device,
+ a host-resident base moves to it one module at a time as it quantizes (peak = int8 base + one bf16 Linear)."""
from core.inference.diffusion_transformer_quant import (
_quiet_config,
apply_small_m_padding,
@@ -395,9 +414,15 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:
filter_fn = make_filter_fn(
512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family, baked = True)
),
+ **({} if device is None else {"device": device}),
)
# After quantize_: the helper reparents the Linears it wraps. Not best-effort: a raise leaves the base quantized but not safely runnable.
apply_small_m_padding(transformer, "int8", family)
+ if device is not None:
+ # quantize_ moves every module it visits; this catches anything outside its walk (root params / buffers).
+ transformer.to(device)
+ gc.collect()
+ _empty_device_cache(device)
# Small-M linears (conditioning vectors, LTX-2 one-token audio) stay bf16: the float8 / MX casts cost more than the GEMM.
@@ -853,9 +878,9 @@ def _flux_collate(
import torch
# FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared across prompts.
- pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
- pooled = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)
- text_ids = entries[0][2].to(device = device, dtype = torch.float32)
+ pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
+ pooled = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)
+ text_ids = entries[0][2].to(device = device, non_blocking = True).to(dtype = torch.float32)
return (pe, pooled, text_ids)
@@ -994,17 +1019,21 @@ def _qwen_collate(
seqs = [e[0].shape[1] for e in entries]
target = max(pad_to or 0, max(seqs))
pes, masks = [], []
+ # Pad/cat on device: a host pad/cat is pageable and its copy blocks.
for pe, mask in entries:
s = pe.shape[1]
+ pe = pe.to(device, non_blocking = True)
if mask is None:
- mask = torch.ones((1, s), dtype = torch.int64)
+ mask = torch.ones((1, s), dtype = torch.int64, device = pe.device)
+ else:
+ mask = mask.to(device, non_blocking = True)
if s < target:
pe = F.pad(pe, (0, 0, 0, target - s))
mask = F.pad(mask, (0, target - s))
pes.append(pe)
masks.append(mask)
- pe_b = torch.cat(pes).to(device = device, dtype = weight_dtype)
- mask_b = torch.cat(masks).to(device)
+ pe_b = (pes[0] if len(pes) == 1 else torch.cat(pes)).to(dtype = weight_dtype)
+ mask_b = masks[0] if len(masks) == 1 else torch.cat(masks)
# A single unpadded sample keeps the legacy None mask, avoiding a behaviour change for existing single-image runs.
if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:
mask_b = None
@@ -1088,7 +1117,7 @@ def _zimage_collate(
weight_dtype,
pad_to = None,
):
- caps = [e[0].to(device = device, dtype = weight_dtype) for e in entries]
+ caps = [e[0].to(device = device, non_blocking = True).to(dtype = weight_dtype) for e in entries]
return (caps,)
@@ -1120,7 +1149,8 @@ def _krea2_load_conditioners(cfg, device, weight_dtype):
pipe = load_krea2_pipeline(
cfg.base_model, torch.bfloat16, hf_token = cfg.hf_token, with_transformer = False
)
- pipe.vae.to(device, dtype = torch.float32)
+ # Host until the text encoders are freed, as _load_pipe_without_transformer.
+ pipe.vae.to(dtype = torch.float32)
return pipe, pipe.vae
@@ -1159,11 +1189,9 @@ def _krea2_collate(
weight_dtype,
pad_to = None,
):
- import torch
-
# Fixed-length embeds, so collation is a plain concat with the mask riding along; ``pad_to`` is moot.
- pe_b = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
- mask_b = torch.cat([e[1] for e in entries]).to(device)
+ pe_b = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
+ mask_b = _to_device_cat([e[1] for e in entries], device)
return (pe_b, mask_b)
@@ -1280,8 +1308,8 @@ def _flux2_collate(
import torch
# text_ids are PER-SAMPLE (unlike FLUX.1's shared grid).
- pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
- text_ids = torch.cat([e[1] for e in entries]).to(device = device, dtype = torch.float32)
+ pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
+ text_ids = _to_device_cat([e[1] for e in entries], device).to(dtype = torch.float32)
return (pe, text_ids)
@@ -1425,8 +1453,6 @@ def _ltx2_encode_prompts(pipe, captions, device):
def _ltx2_latent_affine(vae, ref):
- import torch
-
mean = vae.latents_mean.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)
std = vae.latents_std.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)
# scaling_factor is 1.0 on the shipped checkpoint but is applied for fidelity to _normalize_latents.
@@ -1460,12 +1486,10 @@ def _ltx2_collate(
weight_dtype,
pad_to = None,
):
- import torch
-
# Connector embeds are fixed at [1, 1024, 3840] via max_sequence_length, so pad_to is moot.
- video = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)
- audio = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)
- mask = torch.cat([e[2] for e in entries]).to(device)
+ video = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
+ audio = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)
+ mask = _to_device_cat([e[2] for e in entries], device)
return (video, audio, mask)
@@ -1966,12 +1990,21 @@ def _load_pixel_tensor(path, resolution, center_crop, random_flip, rng):
return _to_unit_tensor(img)
-def _load_pixel_tensor_planned(path, resolution, center_crop, u_left, u_top, flip):
- """Deterministic pixel-crop variant for the latent cache: crop offsets are unit fractions over the randint range; center_crop matches the uncached path."""
+def _load_pixel_tensor_planned(
+ path,
+ resolution,
+ center_crop,
+ u_left,
+ u_top,
+ flip,
+ resized = None,
+):
+ """Deterministic pixel-crop variant for the latent cache: crop offsets are unit fractions over the randint range; center_crop matches the uncached path.
+ ``resized`` is a prior ``_open_resized`` result, reused across variants."""
from PIL import Image
cw, ch = _canvas(resolution)
- img, rw, rh = _open_resized(path, resolution)
+ img, rw, rh = resized if resized is not None else _open_resized(path, resolution)
if center_crop:
left, top = (rw - cw) // 2, (rh - ch) // 2
else:
@@ -2027,6 +2060,7 @@ def _build_latent_cache(
gated = False
for i, path in enumerate(image_paths):
variants = []
+ resized = None
for u_left, u_top, flip in plan[i]:
canvas = cfg.resolution if canvases is None else canvases[i]
key = (
@@ -2038,8 +2072,12 @@ def _build_latent_cache(
if entry is not None:
a, b = entry
else:
+ if resized is None:
+ resized = _open_resized(path, canvas)
px = (
- _load_pixel_tensor_planned(path, canvas, cfg.center_crop, u_left, u_top, flip)
+ _load_pixel_tensor_planned(
+ path, canvas, cfg.center_crop, u_left, u_top, flip, resized = resized
+ )
.unsqueeze(0)
.to(device)
)
@@ -2186,6 +2224,34 @@ def _to_device_cat(parts, device):
return moved[0] if len(moved) == 1 else torch.cat(moved)
+# Pinned (page-locked) host memory is not swappable, so a huge caption set stays pageable; the copies stay async.
+_CAPTION_PIN_BUDGET_BYTES = 1024**3
+
+
+def _pin_caption_embeds(caption_embeds: dict, device) -> dict:
+ import torch
+
+ from core.inference.host_pinning import host_pinning_allowed
+
+ if device != "cuda" or not caption_embeds or not host_pinning_allowed():
+ return caption_embeds
+ tensors = [t for entry in caption_embeds.values() for t in entry if isinstance(t, torch.Tensor)]
+ if any(t.device.type != "cpu" for t in tensors):
+ return caption_embeds
+ if sum(t.numel() * t.element_size() for t in tensors) > _CAPTION_PIN_BUDGET_BYTES:
+ return caption_embeds
+
+ def _pin(t):
+ if not isinstance(t, torch.Tensor):
+ return t
+ try:
+ return t.contiguous().pin_memory()
+ except RuntimeError:
+ return t
+
+ return {cap: type(entry)(_pin(t) for t in entry) for cap, entry in caption_embeds.items()}
+
+
def _should_compile(
cfg,
base_is_bnb,
@@ -2544,7 +2610,6 @@ def _train_dit(
torch.cuda.empty_cache()
elif device == "xpu":
torch.xpu.empty_cache()
- # Some families keep the VAE on host while text encoders are resident.
if vae is not None:
vae.to(device)
@@ -2596,6 +2661,7 @@ def _train_dit(
torch.xpu.empty_cache()
# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.
variant_rng = random.Random(cfg.seed + 1)
+ caption_embeds = _pin_caption_embeds(caption_embeds, device)
# Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.
base_precision = _resolve_base_precision(cfg, spec, device)
@@ -2628,7 +2694,9 @@ def _train_dit(
# Convert AFTER the LoRA attaches so the adapter modules stay high precision.
if base_precision == "int8":
- _int8_quantize_base(transformer, cfg.resolved_family)
+ _int8_quantize_base(transformer, cfg.resolved_family, device = device)
+ # Re-collect: the move to device must not leave the optimizer holding host tensors.
+ lora_params = [p for p in transformer.parameters() if p.requires_grad]
if base_precision == "fp8" and not _apply_fp8_training(
transformer, on_event, cfg.resolved_family
):
@@ -2856,7 +2924,7 @@ def _train_dit(
stopped = True
break
optimizer.zero_grad(set_to_none = True)
- step_loss = 0.0
+ step_acc = None
for _ in range(cfg.gradient_accumulation_steps):
idxs = index_sampler.next_batch(batch_size)
if canvases is not None:
@@ -2886,7 +2954,10 @@ def _train_dit(
if shift_active:
# timestep = sigma * num_train_timesteps, gathered in fp32 so bf16 rounding never skews it.
timesteps = (
- sigma_table[t_indices].to(device = device, dtype = torch.float32).flatten()
+ sigma_table[t_indices]
+ .to(device = device, non_blocking = True)
+ .to(dtype = torch.float32)
+ .flatten()
* num_train_ts
)
noisy = (1.0 - sigmas) * latents + sigmas * noise
@@ -2917,12 +2988,13 @@ def _train_dit(
w = bell_weights[w_idx].view(-1, *([1] * (per.ndim - 1)))
loss = (per * w).mean()
(loss / cfg.gradient_accumulation_steps).backward()
- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
+ step_acc = accumulate_loss(step_acc, loss, cfg.gradient_accumulation_steps)
grad_norm = None
if cfg.max_grad_norm and cfg.max_grad_norm > 0:
# Pre-clip norm: the UI's health signal.
grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
+ step_loss = loss_value(step_acc)
optimizer.step()
lr_sched.step()
if ema is not None:
diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.py
index a222e395b5e..0ec7179d298 100644
--- a/studio/backend/core/training/diffusion_h3_trainer.py
+++ b/studio/backend/core/training/diffusion_h3_trainer.py
@@ -92,7 +92,12 @@ from core.training.diffusion_train_common import (
resolve_train_steps,
train_recipe_overrides,
)
-from core.training.diffusion_train_extras import LoRAEMA, save_ema_adapter
+from core.training.diffusion_train_extras import (
+ LoRAEMA,
+ accumulate_loss,
+ loss_value,
+ save_ema_adapter,
+)
# adaln_proj inputs have 2-3 rows, so an adapter there learns noise; the projections stay fp32.
# A REGEX, not a list: PEFT suffix-matches lists and would also adapt token_refiner.
@@ -181,10 +186,8 @@ def _load_conditioners(cfg, device):
**auth,
)
_assert_component_grid(pipe)
- # ``load_components`` builds every component on the CPU, so place them explicitly.
+ # Only the conditioner moves here; the VAEs move after it is freed (_train_h3).
pipe.text_encoder.to(device)
- pipe.vae.to(device)
- pipe.audio_vae.to(device)
return pipe
@@ -209,15 +212,50 @@ def _encode_prompt(pipe, caption: str, device) -> Any:
f"conditioner, which needs more than {_H3_TEXT_ENCODER_LAYER} decoder layers; this "
f"one has {num_layers}."
)
+ kwargs = dict(
+ input_ids = input_ids,
+ attention_mask = torch.ones_like(input_ids),
+ mm_token_type_ids = mm_token_type_ids,
+ use_cache = False,
+ )
with torch.no_grad():
- out = encoder.model(
- input_ids = input_ids,
- attention_mask = torch.ones_like(input_ids),
- mm_token_type_ids = mm_token_type_ids,
- use_cache = False,
- output_hidden_states = True,
- )
- return out.hidden_states[_H3_TEXT_ENCODER_LAYER].to("cpu", torch.float32)
+ hidden = _early_exit_hidden_state(encoder, _H3_TEXT_ENCODER_LAYER, kwargs)
+ if hidden is None:
+ out = encoder.model(**kwargs, output_hidden_states = True)
+ hidden = out.hidden_states[_H3_TEXT_ENCODER_LAYER]
+ return hidden.to("cpu", torch.float32)
+
+
+class _HiddenStateReached(Exception):
+ """Raised by the capture hook to stop the encoder once the read layer has run."""
+
+
+def _early_exit_hidden_state(encoder, layer: int, kwargs: dict) -> Any:
+ """hidden_states[layer] is the raw output of decoder layer ``layer - 1`` (index 0 is the embeddings), so capture
+ it there and stop: layers past it and the final norm never run, and the other 64 states are never kept. Text-only
+ input, so no deepstack visual rows are added between layers. None (run the full encoder) on any surprise."""
+ import torch
+
+ text_model = getattr(getattr(encoder, "model", None), "language_model", None)
+ layers = getattr(text_model, "layers", None)
+ if not isinstance(layers, torch.nn.ModuleList) or not 0 < layer < len(layers):
+ return None
+ captured: list = []
+
+ def _capture(_module, _args, output):
+ captured.append(output[0] if isinstance(output, (tuple, list)) else output)
+ raise _HiddenStateReached
+
+ handle = layers[layer - 1].register_forward_hook(_capture)
+ try:
+ encoder.model(**kwargs, output_hidden_states = False)
+ except _HiddenStateReached:
+ pass
+ finally:
+ handle.remove()
+ if len(captured) != 1 or not isinstance(captured[0], torch.Tensor):
+ return None
+ return captured[0]
def _encode_video_stats(vae, frames, device) -> tuple[Any, Any]:
@@ -280,13 +318,15 @@ def _load_transformer(cfg, device, base_precision):
token = cfg.hf_token,
cache_dir = cache_dir,
)
- return MiniMaxH3Transformer3DModel.from_pretrained(
+ transformer = MiniMaxH3Transformer3DModel.from_pretrained(
cfg.base_model,
subfolder = "transformer",
torch_dtype = torch.bfloat16,
token = cfg.hf_token,
cache_dir = cache_dir,
- ).to(device)
+ )
+ # int8 stays on the host: _int8_quantize_base streams it onto the device one Linear at a time.
+ return transformer if base_precision == "int8" else transformer.to(device)
def _patchify(latents, patch: tuple[int, int, int]):
@@ -482,7 +522,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
to_encode = sorted(set(captions))
- # Phase 1: conditioning. The 63 GiB Qwen3-VL conditioner and both VAEs are resident here and nowhere else.
+ # Phase 1: conditioning. The 63 GiB Qwen3-VL conditioner is resident here and nowhere else.
pipe = _load_conditioners(cfg, device)
caption_embeds = {cap: _encode_prompt(pipe, cap, device) for cap in to_encode}
_emit(on_event, "preparing", stage = "encode_prompts", done = len(to_encode), total = len(to_encode))
@@ -494,6 +534,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
torch.cuda.empty_cache()
elif device == "xpu":
torch.xpu.empty_cache()
+ for attr in _H3_VAE_COMPONENTS:
+ module = getattr(pipe, attr, None)
+ if module is not None:
+ module.to(device)
# One canvas from the FIRST clip's aspect ratio; other clips are cover-cropped onto it.
width, height = _dataset_canvas(clip_paths[0], cfg.resolution)
@@ -585,7 +629,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
)
if base_precision == "int8":
# WITH the family: it excludes adaln_proj (runs at M = 1) and supplies the pad list.
- _int8_quantize_base(transformer, cfg.resolved_family)
+ _int8_quantize_base(transformer, cfg.resolved_family, device = device)
+ lora_params = [p for p in transformer.parameters() if p.requires_grad]
ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None
# dynamic=True keeps one graph per block across caption lengths; int8 stays eager like the DiT families.
@@ -614,6 +659,20 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
transformer.train()
patch = tuple(transformer.config.patch_size)
+ from core.inference.host_pinning import host_pinning_allowed
+
+ if device == "cuda" and host_pinning_allowed():
+
+ def _pin_or_keep(t):
+ try:
+ return t.contiguous().pin_memory()
+ except RuntimeError:
+ return t
+
+ cache = [tuple(_pin_or_keep(t) for t in entry) for entry in cache]
+ caption_embeds = {cap: _pin_or_keep(t) for cap, t in caption_embeds.items()}
+ # Packed layout depends only on caption length (fixed canvas).
+ layouts: dict[int, tuple[dict, dict]] = {}
index_sampler = PermutationBatchSampler(len(clip_paths), rng)
autocast = (
torch.autocast(device_type = device, dtype = torch.bfloat16)
@@ -630,17 +689,15 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
compile_watch = _dynamo_compile_counters() if compiled else None
for opt_step in range(cfg.train_steps):
optimizer.zero_grad(set_to_none = True)
- step_loss = 0.0
- step_video = 0.0
- step_audio = 0.0
+ step_acc = video_acc = audio_acc = None
for _ in range(cfg.gradient_accumulation_steps):
index = index_sampler.next_batch(1)[0]
video_a, video_b, audio_latents = cache[index]
- video_a = video_a.to(device)
- video_b = video_b.to(device)
+ video_a = video_a.to(device, non_blocking = True)
+ video_b = video_b.to(device, non_blocking = True)
# A fresh posterior draw per step, exactly like encoding in the loop would give.
clean_video = video_a + video_b * torch.randn_like(video_a)
- clean_audio = audio_latents.to(device)
+ clean_audio = audio_latents.to(device, non_blocking = True)
# ONE base u through BOTH shifts: the (video, audio) sigma pair the sampler visits.
u = rng.random()
@@ -660,14 +717,31 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
(clean_audio - noise_audio).permute(0, 2, 1).reshape(-1, H3_AUDIO_LATENT_CHANNELS)
)
- embeds = caption_embeds[captions[index]].to(device, weight_dtype)
+ embeds = caption_embeds[captions[index]].to(device, non_blocking = True).to(weight_dtype)
num_text_tokens = embeds.shape[1]
- layout = _build_layout(
- num_text_tokens, latent_frames, latent_h, latent_w, num_audio_latents, patch, device
- )
+ if num_text_tokens not in layouts:
+ host_layout = _build_layout(
+ num_text_tokens,
+ latent_frames,
+ latent_h,
+ latent_w,
+ num_audio_latents,
+ patch,
+ "cpu",
+ )
+ layouts[num_text_tokens] = (
+ host_layout,
+ {
+ k: v.to(device) if isinstance(v, torch.Tensor) else v
+ for k, v in host_layout.items()
+ },
+ )
+ host_layout, layout = layouts[num_text_tokens]
timestep, timestep_indices = _row_timesteps(
- layout, num_text_tokens, 1.0 - sigma_video, 1.0 - sigma_audio, device
+ host_layout, num_text_tokens, 1.0 - sigma_video, 1.0 - sigma_audio, "cpu"
)
+ timestep = timestep.to(device, non_blocking = True)
+ timestep_indices = timestep_indices.to(device, non_blocking = True)
with autocast:
pred_video, pred_audio = transformer(
hidden_states = video_rows[None],
@@ -687,13 +761,17 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
# Unweighted: both streams share every adapted matrix.
loss = loss_video + loss_audio
(loss / cfg.gradient_accumulation_steps).backward()
- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
- step_video += float(loss_video.detach()) / cfg.gradient_accumulation_steps
- step_audio += float(loss_audio.detach()) / cfg.gradient_accumulation_steps
+ ga = cfg.gradient_accumulation_steps
+ step_acc = accumulate_loss(step_acc, loss, ga)
+ video_acc = accumulate_loss(video_acc, loss_video, ga)
+ audio_acc = accumulate_loss(audio_acc, loss_audio, ga)
grad_norm = None
if cfg.max_grad_norm and cfg.max_grad_norm > 0:
grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
+ step_loss = loss_value(step_acc)
+ step_video = loss_value(video_acc)
+ step_audio = loss_value(audio_acc)
optimizer.step()
lr_sched.step()
if ema is not None:
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index 883839a0e3b..b2c2e27e85d 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -82,7 +82,12 @@ from core.training.diffusion_samples import (
sample_seed,
SampleRoundStopped,
)
-from core.training.diffusion_train_extras import BucketBatchSampler, cover_resize_dims
+from core.training.diffusion_train_extras import (
+ BucketBatchSampler,
+ accumulate_loss,
+ cover_resize_dims,
+ loss_value,
+)
from core.training.diffusion_checkpoint import (
clear_own_checkpoints,
discard_preexisting_checkpoints,
@@ -96,6 +101,12 @@ from core.training.diffusion_checkpoint import (
)
+def _parts_to_device(parts, device):
+ import torch
+ moved = [p.to(device, non_blocking = True) for p in parts]
+ return moved[0] if len(moved) == 1 else torch.cat(moved)
+
+
def compute_sdxl_add_time_ids(resolution: int) -> tuple[int, int, int, int, int, int]:
"""Square default with no crop offset; the training loop derives per-image ids from the actual crop."""
return (resolution, resolution, 0, 0, resolution, resolution)
@@ -737,7 +748,7 @@ def run_diffusion_lora_training(
stopped = True
break
optimizer.zero_grad(set_to_none = True)
- step_loss = 0.0
+ step_acc = None
for _ in range(cfg.gradient_accumulation_steps):
idx, img_paths, captions = _next_batch()
if latent_cache is not None:
@@ -776,8 +787,10 @@ def run_diffusion_lora_training(
noisy = noise_scheduler.add_noise(latents, noise, timesteps)
if precompute:
- prompt_embeds = torch.cat([caption_embeds[c][0] for c in captions]).to(device)
- pooled = torch.cat([caption_embeds[c][1] for c in captions]).to(device)
+ prompt_embeds = _parts_to_device(
+ [caption_embeds[c][0] for c in captions], device
+ )
+ pooled = _parts_to_device([caption_embeds[c][1] for c in captions], device)
else:
prompt_embeds, pooled = _encode_sdxl_prompts(
captions, tokenizers, text_encoders, device
@@ -808,7 +821,7 @@ def run_diffusion_lora_training(
loss = F.mse_loss(model_pred.float(), target.float(), reduction = "mean")
scaler.scale(loss / cfg.gradient_accumulation_steps).backward()
- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
+ step_acc = accumulate_loss(step_acc, loss, cfg.gradient_accumulation_steps)
micro += 1
# max_grad_norm <= 0 disables clipping; clip_grad_norm_(0.0) would zero all grads.
@@ -819,6 +832,7 @@ def run_diffusion_lora_training(
grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
if not math.isfinite(grad_norm):
grad_norm = None
+ step_loss = loss_value(step_acc)
scale_before = scaler.get_scale() if scaler.is_enabled() else None
scaler.step(optimizer)
scaler.update()
diff --git a/studio/backend/core/training/diffusion_train_extras.py b/studio/backend/core/training/diffusion_train_extras.py
index 6e39749c749..aca88950cd7 100644
--- a/studio/backend/core/training/diffusion_train_extras.py
+++ b/studio/backend/core/training/diffusion_train_extras.py
@@ -15,6 +15,23 @@ from pathlib import Path
from typing import Any, Iterable, Optional
+def accumulate_loss(acc: Any, loss: Any, accumulation_steps: int) -> Any:
+ """``acc + float(loss) / accumulation_steps`` without a per-micro-step host sync: an fp64 device sum is the same
+ IEEE arithmetic as the Python float one. Devices without fp64 (MPS, Arc / integrated XPUs) keep the Python float.
+ Read with loss_value."""
+ detached = loss.detach()
+ if detached.device.type not in ("cpu", "cuda"):
+ from core.inference.video_minimax_h3_pdd import device_has_fp64
+ if not device_has_fp64(detached.device):
+ return (acc if acc is not None else 0.0) + float(detached) / accumulation_steps
+ term = detached.double() / accumulation_steps
+ return term if acc is None else acc.add_(term)
+
+
+def loss_value(acc: Any) -> float:
+ return 0.0 if acc is None else float(acc)
+
+
# Effective decay = min(decay, (1 + n) / (WARMUP_OFFSET + n)).
_EMA_WARMUP_OFFSET = 10.0
@@ -69,17 +86,36 @@ class LoRAEMA:
step = self.updates + 1
return min(self.decay, step / (_EMA_WARMUP_OFFSET + step))
+ def _update_lists(self, model: Any) -> tuple[list, list]:
+ """(shadows, params) for ``model``, cached per model."""
+ import weakref
+
+ cached = getattr(self, "_lists", None)
+ if cached is not None and cached[0]() is model:
+ return cached[1], cached[2]
+ shadows, params = [], []
+ for name, p in model.named_parameters():
+ shadow = self._shadow.get(name)
+ if shadow is None:
+ continue
+ shadows.append(shadow)
+ params.append(p.detach())
+ try:
+ self._lists = (weakref.ref(model), shadows, params)
+ except TypeError: # not weak-referenceable: rebuild each call
+ self._lists = None
+ return shadows, params
+
def update(self, model: Any) -> None:
"""Apply one EMA update from ``model``'s current trainable params."""
import torch
decay = self.effective_decay()
- with torch.no_grad():
- for name, p in model.named_parameters():
- shadow = self._shadow.get(name)
- if shadow is None:
- continue
- shadow.mul_(decay).add_(p.detach(), alpha = 1.0 - decay)
+ shadows, params = self._update_lists(model)
+ if shadows:
+ with torch.no_grad():
+ torch._foreach_mul_(shadows, decay)
+ torch._foreach_add_(shadows, params, alpha = 1.0 - decay)
self.updates += 1
def state_dict(self) -> dict[str, Any]:
diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.py
index 67e0cd78dc9..6ae526512bc 100644
--- a/studio/backend/tests/test_diffusion_h3_trainer.py
+++ b/studio/backend/tests/test_diffusion_h3_trainer.py
@@ -796,7 +796,7 @@ def test_int8_training_applies_h3s_small_m_guards():
assert pad_tokens_for_scheme("int8", "minimax-h3")
src = Path(h3.__file__).read_text()
- assert "_int8_quantize_base(transformer, cfg.resolved_family)" in src
+ assert "_int8_quantize_base(transformer, cfg.resolved_family, device = device)" in src
dit_src = Path(dit.__file__).read_text()
body = dit_src.split("def _int8_quantize_base", 1)[1].split("\ndef ", 1)[0]
# training keeps the baked list: the streamed-inference text-stream scope never reaches it
@@ -1767,7 +1767,9 @@ def test_h3_routes_its_transformer_through_the_shared_regional_compile(
diffusion_h3_trainer, "_load_transformer", lambda cfg, device, precision: _Tiny()
)
monkeypatch.setattr(
- dit, "_int8_quantize_base", lambda transformer, family: order.append("int8")
+ dit,
+ "_int8_quantize_base",
+ lambda transformer, family, device = None: order.append("int8"),
)
monkeypatch.setattr(dit, "_maybe_compile_transformer", _compile)
monkeypatch.setattr(dit, "_make_optimizer", _optimizer)
diff --git a/studio/backend/tests/test_diffusion_train_hot_loop.py b/studio/backend/tests/test_diffusion_train_hot_loop.py
new file mode 100644
index 00000000000..fd2aae5d621
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_train_hot_loop.py
@@ -0,0 +1,417 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Diffusion training hot loop: no per-micro-step host syncs, one decode per cached image, batched EMA, conditioning
+placement and the streamed int8 load. Every change here must leave the numbers bit-identical."""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.training import diffusion_dit_trainer as dit # noqa: E402
+from core.training import diffusion_train_extras as extras # noqa: E402
+
+
+class _ScalarReads(torch.utils._python_dispatch.TorchDispatchMode):
+ """Counts tensor -> Python scalar reads (each one a host sync on an accelerator)."""
+
+ def __init__(self):
+ super().__init__()
+ self.reads = 0
+
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ if func is torch.ops.aten._local_scalar_dense.default:
+ self.reads += 1
+ return func(*args, **(kwargs or {}))
+
+
+def test_loss_accumulation_matches_the_python_float_sum_without_scalar_reads():
+ g = torch.Generator().manual_seed(0)
+ losses = [torch.rand((), generator = g) * 3 for _ in range(8)]
+ expected = 0.0
+ for loss in losses:
+ expected += float(loss) / 8
+ acc = None
+ with _ScalarReads() as mode:
+ for loss in losses:
+ acc = extras.accumulate_loss(acc, loss, 8)
+ assert mode.reads == 0
+ with _ScalarReads() as mode:
+ got = extras.loss_value(acc)
+ assert mode.reads == 1
+ assert got == expected # bitwise: fp64 device math is the Python double math
+ assert extras.loss_value(None) == 0.0
+
+
+def test_loss_accumulation_keeps_the_differentiable_loss_untouched():
+ w = torch.ones(3, requires_grad = True)
+ loss = (w * 2).sum()
+ acc = extras.accumulate_loss(None, loss, 2)
+ assert not acc.requires_grad
+ loss.backward()
+ assert torch.equal(w.grad, torch.full((3,), 2.0))
+
+
+def test_loss_accumulation_keeps_the_python_float_on_a_device_without_fp64(monkeypatch):
+ from core.inference import video_minimax_h3_pdd as pdd
+
+ class _ArcLoss:
+ device = torch.device("xpu")
+
+ def __init__(self, value):
+ self.value = value
+
+ def detach(self):
+ return self
+
+ def __float__(self):
+ return self.value
+
+ def double(self):
+ raise AssertionError("Required aspect fp64 is not supported on the device")
+
+ monkeypatch.setitem(pdd._FP64_BY_DEVICE, ("xpu", None), False)
+ acc = None
+ for v in (0.25, 1.5, 3.0):
+ acc = extras.accumulate_loss(acc, _ArcLoss(v), 4)
+ assert isinstance(acc, float) and extras.loss_value(acc) == 0.25 / 4 + 1.5 / 4 + 3.0 / 4
+
+
+def _ema_reference(shadows, params, decay):
+ for s, p in zip(shadows, params):
+ s.mul_(decay).add_(p, alpha = 1.0 - decay)
+
+
+def test_lora_ema_batched_update_is_bitwise_the_per_tensor_update():
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(torch.nn.Linear(16, 8), torch.nn.Linear(8, 4))
+ model[0].bias.requires_grad_(False)
+ ema = extras.LoRAEMA(model, decay = 0.97)
+ ref = {n: t.clone() for n, t in ema._shadow.items()}
+ for step in range(5):
+ with torch.no_grad():
+ for p in model.parameters():
+ p.add_(torch.randn_like(p) * 0.1)
+ decay = ema.effective_decay()
+ names = [n for n, p in model.named_parameters() if n in ref]
+ _ema_reference(
+ [ref[n] for n in names],
+ [dict(model.named_parameters())[n].detach() for n in names],
+ decay,
+ )
+ ema.update(model)
+ for n in ref:
+ assert torch.equal(ema._shadow[n], ref[n]), (step, n)
+ assert ema.updates == 5
+ assert "0.bias" not in ema._shadow
+
+
+def test_lora_ema_update_walks_the_parameters_once():
+ model = torch.nn.Linear(4, 4)
+ ema = extras.LoRAEMA(model, decay = 0.9)
+ walks = []
+ original = model.named_parameters
+
+ def _counting(*a, **k):
+ walks.append(1)
+ return original(*a, **k)
+
+ model.named_parameters = _counting
+ for _ in range(4):
+ ema.update(model)
+ assert len(walks) == 1
+ # A different model object rebuilds the lists.
+ other = torch.nn.Linear(4, 4)
+ ema.update(other)
+ assert torch.equal(ema._shadow["weight"], ema._shadow["weight"])
+
+
+def test_latent_cache_decodes_each_image_once_with_identical_variants(monkeypatch, tmp_path):
+ from PIL import Image
+ import numpy as np
+
+ path = tmp_path / "a.png"
+ Image.fromarray(np.random.RandomState(0).randint(0, 255, (90, 70, 3), dtype = np.uint8)).save(
+ path
+ )
+ opens = []
+ real_open = dit._open_resized
+
+ def _counting_open(p, res):
+ opens.append(p)
+ return real_open(p, res)
+
+ monkeypatch.setattr(dit, "_open_resized", _counting_open)
+ plan = [[(0.0, 0.0, False), (0.9, 0.9, True), (0.5, 0.1, False), (0.2, 0.7, True)]]
+ got = []
+ spec = types.SimpleNamespace(
+ encode_latent_stats = lambda vae, px: (got.append(px.clone()) or px[:, :1], None)
+ )
+ cfg = types.SimpleNamespace(
+ resolution = 48, center_crop = False, cache_variants = 4, random_flip = True, seed = 0
+ )
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", "1")
+ cache = dit._build_latent_cache(
+ spec,
+ None,
+ [str(path)],
+ cfg,
+ "cpu",
+ torch.float32,
+ None,
+ lambda: False,
+ plan = plan,
+ )
+ assert len(cache[0]) == 4
+ assert len(opens) == 1
+ for px, (u_left, u_top, flip) in zip(got, plan[0]):
+ expected = dit._load_pixel_tensor_planned(str(path), 48, False, u_left, u_top, flip)
+ assert torch.equal(px[0], expected)
+
+
+def _old_qwen_collate(
+ entries,
+ weight_dtype,
+ pad_to = None,
+):
+ import torch.nn.functional as F
+
+ seqs = [e[0].shape[1] for e in entries]
+ target = max(pad_to or 0, max(seqs))
+ pes, masks = [], []
+ for pe, mask in entries:
+ s = pe.shape[1]
+ if mask is None:
+ mask = torch.ones((1, s), dtype = torch.int64)
+ if s < target:
+ pe = F.pad(pe, (0, 0, 0, target - s))
+ mask = F.pad(mask, (0, target - s))
+ pes.append(pe)
+ masks.append(mask)
+ pe_b = torch.cat(pes).to(dtype = weight_dtype)
+ mask_b = torch.cat(masks)
+ if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:
+ mask_b = None
+ return pe_b, mask_b
+
+
[email protected]("pad_to", [None, 9])
[email protected]("n", [1, 3])
+def test_qwen_collate_pads_on_device_identically(n, pad_to):
+ g = torch.Generator().manual_seed(n)
+ entries = []
+ for i in range(n):
+ s = 4 + i
+ mask = None if i % 2 == 0 else torch.ones((1, s), dtype = torch.int64)
+ entries.append((torch.randn(1, s, 6, generator = g), mask))
+ pe, mask = dit._qwen_collate(entries, "cpu", torch.bfloat16, pad_to = pad_to)
+ ref_pe, ref_mask = _old_qwen_collate(entries, torch.bfloat16, pad_to)
+ assert pe.dtype == torch.bfloat16 and torch.equal(pe, ref_pe)
+ assert (mask is None) == (ref_mask is None)
+ if mask is not None:
+ assert torch.equal(mask, ref_mask)
+
+
+def test_fixed_length_collates_match_a_host_cat():
+ a = (torch.randn(1, 5, 4), torch.randn(1, 4), torch.zeros(5, 3))
+ b = (torch.randn(1, 5, 4), torch.randn(1, 4), torch.zeros(5, 3))
+ pe, pooled, ids = dit._flux_collate([a, b], "cpu", torch.bfloat16)
+ assert torch.equal(pe, torch.cat([a[0], b[0]]).to(torch.bfloat16))
+ assert torch.equal(pooled, torch.cat([a[1], b[1]]).to(torch.bfloat16))
+ assert ids.dtype == torch.float32
+ v = [(torch.randn(1, 3, 2), torch.randn(1, 3, 2), torch.ones(1, 3)) for _ in range(2)]
+ video, audio, mask = dit._ltx2_collate(v, "cpu", torch.bfloat16)
+ assert torch.equal(video, torch.cat([e[0] for e in v]).to(torch.bfloat16))
+ assert torch.equal(mask, torch.cat([e[2] for e in v]))
+
+
+def test_caption_embeds_pin_only_on_cuda_and_under_budget(monkeypatch):
+ embeds = {"a": (torch.zeros(1, 2), torch.zeros(1))}
+ assert dit._pin_caption_embeds(embeds, "cpu") is embeds
+ pinned = []
+ monkeypatch.setattr("core.inference.host_pinning.host_pinning_allowed", lambda *a: True)
+ monkeypatch.setattr(torch.Tensor, "pin_memory", lambda self: pinned.append(1) or self)
+ out = dit._pin_caption_embeds(embeds, "cuda")
+ assert len(pinned) == 2 and set(out) == {"a"} and isinstance(out["a"], tuple)
+ monkeypatch.setattr(dit, "_CAPTION_PIN_BUDGET_BYTES", 1)
+ pinned.clear()
+ assert dit._pin_caption_embeds(embeds, "cuda") is embeds and not pinned
+
+
+def test_sampled_timesteps_and_sigmas_are_unchanged(monkeypatch):
+ pytest.importorskip("diffusers")
+ from diffusers import FlowMatchEulerDiscreteScheduler
+
+ sched = FlowMatchEulerDiscreteScheduler(shift = 3.0)
+ torch.manual_seed(5)
+ ts, idx = dit._sample_timesteps(sched, 4, "cpu")
+ torch.manual_seed(5)
+ _ts, _idx = dit._sample_timesteps(sched, 4, "cpu")
+ assert torch.equal(idx, _idx)
+ assert torch.equal(ts, sched.timesteps[idx])
+ sig = dit._gather_sigmas(sched.sigmas, idx, "cpu", torch.bfloat16, 4)
+ assert sig.shape == (4, 1, 1, 1)
+ assert torch.equal(sig.flatten(), sched.sigmas[idx].to(torch.bfloat16))
+
+
+def test_conditioner_load_keeps_the_vae_on_the_host():
+ moves = []
+
+ class _Vae:
+ def to(self, *args, **kwargs):
+ moves.append((args, kwargs))
+ return self
+
+ class _PipeCls:
+ @staticmethod
+ def from_pretrained(*_a, **_k):
+ return types.SimpleNamespace(vae = _Vae())
+
+ cfg = types.SimpleNamespace(base_model = "x", hf_token = None)
+ pipe, vae = dit._load_pipe_without_transformer(_PipeCls, cfg, "cuda")
+ assert moves == [((), {"dtype": torch.float32})]
+
+
+def test_int8_base_streams_onto_the_device(monkeypatch):
+ seen = {}
+
+ def _fake_quantize(
+ model,
+ config,
+ filter_fn = None,
+ device = None,
+ ):
+ seen["device"] = device
+
+ fake = types.ModuleType("torchao.quantization")
+ fake.Int8WeightOnlyConfig = lambda: object()
+ fake.quantize_ = _fake_quantize
+ monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))
+ monkeypatch.setitem(sys.modules, "torchao.quantization", fake)
+ model = torch.nn.Linear(8, 8)
+ dit._int8_quantize_base(model, None, device = "cpu")
+ assert seen["device"] == "cpu"
+ # The legacy call (no device) stays a pure in-place quantize.
+ seen.clear()
+ dit._int8_quantize_base(model, None)
+ assert seen["device"] is None
+
+ loads = []
+
+ class _Model:
+ def to(self, device):
+ loads.append(device)
+ return self
+
+ class _Cls:
+ @staticmethod
+ def from_pretrained(*_a, **_k):
+ return _Model()
+
+ cfg = types.SimpleNamespace(base_model = "x", hf_token = None, resolved_family = None)
+ dit._load_dit_transformer(_Cls, cfg, "cuda", "int8")
+ assert loads == []
+ dit._load_dit_transformer(_Cls, cfg, "cuda", "bf16")
+ assert loads == ["cuda"]
+
+
+def _tiny_qwen3_vl(num_layers):
+ transformers = pytest.importorskip("transformers")
+ if not hasattr(transformers, "Qwen3VLForConditionalGeneration"):
+ pytest.skip("transformers without Qwen3-VL")
+ config = transformers.Qwen3VLConfig(
+ text_config = dict(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = num_layers,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ rope_scaling = {"rope_type": "default", "mrope_section": [1, 1, 2]},
+ ),
+ vision_config = dict(
+ depth = 1,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_heads = 4,
+ out_hidden_size = 32,
+ deepstack_visual_indexes = [0],
+ ),
+ )
+ torch.manual_seed(0)
+ return transformers.Qwen3VLForConditionalGeneration(config).eval()
+
+
[email protected]("layer", [1, 3, 5])
+def test_h3_text_encoder_early_exit_is_bitwise_and_skips_the_tail(layer):
+ from core.training.diffusion_h3_trainer import _early_exit_hidden_state
+
+ model = _tiny_qwen3_vl(6)
+ ids = torch.randint(0, 64, (1, 9), generator = torch.Generator().manual_seed(1))
+ kwargs = dict(input_ids = ids, attention_mask = torch.ones_like(ids), use_cache = False)
+ layers = model.model.language_model.layers
+ ran = []
+ hooks = [
+ m.register_forward_pre_hook(lambda _m, _a, i = i: ran.append(i)) for i, m in enumerate(layers)
+ ]
+ try:
+ with torch.no_grad():
+ full = model.model(**kwargs, output_hidden_states = True).hidden_states[layer]
+ ran.clear()
+ fast = _early_exit_hidden_state(model, layer, kwargs)
+ assert ran == list(range(layer))
+ assert torch.equal(fast, full)
+ # Repeat: a raised early exit must not leave the encoder patched.
+ assert torch.equal(_early_exit_hidden_state(model, layer, kwargs), full)
+ assert len(model.model(**kwargs, output_hidden_states = True).hidden_states) == 7
+ finally:
+ for h in hooks:
+ h.remove()
+ # Out of range (would need the normed last state): the caller runs the full encoder instead.
+ assert _early_exit_hidden_state(model, 6, kwargs) is None
+
+
+def test_h3_vaes_reach_the_device_only_after_the_conditioner_is_freed(monkeypatch):
+ from core.training import diffusion_h3_trainer as h3
+
+ order = []
+
+ class _Placed:
+ def __init__(self, name):
+ self.name = name
+
+ def to(self, device):
+ order.append((self.name, device))
+ return self
+
+ class _Pipe:
+ def __init__(self):
+ self.text_encoder = _Placed("text_encoder")
+ self.vae = _Placed("vae")
+ self.audio_vae = _Placed("audio_vae")
+
+ def load_components(self, **_k):
+ pass
+
+ class _Modular:
+ @staticmethod
+ def from_pretrained(*_a, **_k):
+ return _Pipe()
+
+ monkeypatch.setitem(sys.modules, "diffusers", types.SimpleNamespace(ModularPipeline = _Modular))
+ monkeypatch.setattr(h3, "_assert_component_grid", lambda pipe: None)
+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: "/hub")
+ h3._load_conditioners(types.SimpleNamespace(base_model = "m", hf_token = None), "cuda")
+ assert order == [("text_encoder", "cuda")]
diff --git a/studio/backend/tests/test_integrated_gpu_no_host_pin.py b/studio/backend/tests/test_integrated_gpu_no_host_pin.py
index 44ba29766a7..c31ade49232 100644
--- a/studio/backend/tests/test_integrated_gpu_no_host_pin.py
+++ b/studio/backend/tests/test_integrated_gpu_no_host_pin.py
@@ -699,6 +699,8 @@ def test_only_deviceless_sites_ask_the_predicate_without_a_device():
# device == "cuda" (the current device) is checked first
("core/training/diffusion_dit_trainer.py", "_build_latent_cache"),
("core/training/diffusion_dit_trainer.py", "_load_warm_conditioning"),
+ ("core/training/diffusion_dit_trainer.py", "_pin_caption_embeds"),
+ ("core/training/diffusion_h3_trainer.py", "_train_h3"),
("core/training/diffusion_lora_trainer.py", "_build_sdxl_latent_cache"),
}
bare = set()