Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion training: no per-micro-step host syncs, one decode per cached image, streamed int8 base load (#16763)
Daniel Handa412c072cd9
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 6274b36fd54..c46f60bf3dd 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -97,7 +97,9 @@ from core.training.diffusion_samples import (from core.training.diffusion_train_extras import (BucketBatchSampler,LoRAEMA,+ accumulate_loss,cover_resize_dims,+ loss_value,PersistentConditioningCache,save_ema_adapter,)@@ -183,7 +185,7 @@ class _FamilySpec:def _gather_sigmas(sigma_table, indices, device, dtype, n_dim):"""Gathers by index, so sigma_table must stay aligned with the timesteps the indices were drawn from."""- sigma = sigma_table[indices].to(device = device, dtype = dtype).flatten()+ sigma = sigma_table[indices].to(device = device, non_blocking = True).to(dtype = dtype).flatten()while sigma.ndim < n_dim:sigma = sigma.unsqueeze(-1)return sigma@@ -253,10 +255,19 @@ def _sample_timesteps(scheduler, batch_size, device):)num_train = scheduler.config.num_train_timestepsindices = (u * num_train).long().clamp(0, num_train - 1)- timesteps = scheduler.timesteps.to(device)[indices].to(device)+ # Gather on host then one async copy; device-table indexing with host indices blocks.+ timesteps = scheduler.timesteps[indices].to(device, non_blocking = True)return timesteps, indices+def _empty_device_cache(device) -> None:+ import torch+ if device == "cuda":+ torch.cuda.empty_cache()+ elif device == "xpu":+ torch.xpu.empty_cache()++def _encoders_to_device(pipe, device) -> None:"""A QLoRA load leaves text encoders on CPU, so move them; a 4-bit encoder already placed is skipped."""for attr in ("text_encoder", "text_encoder_2", "text_encoder_3"):@@ -310,7 +321,8 @@ def _load_pipe_without_transformer(pipe_cls, cfg, device):torch_dtype = torch.bfloat16,token = cfg.hf_token,)- pipe.vae.to(device, dtype = torch.float32)+ # Stays on host until the text encoders are freed (never resident beside them).+ pipe.vae.to(dtype = torch.float32)return pipe, pipe.vae@@ -344,12 +356,14 @@ def _load_dit_transformer(transformer_cls, cfg, device, base_precision):# int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear, whose peft-0.18# constructor rejects the torchao-0.16 config API (peft 0.19+ accepts it; the order works for both).- return transformer_cls.from_pretrained(+ transformer = transformer_cls.from_pretrained(cfg.base_model,subfolder = "transformer",torch_dtype = torch.bfloat16,token = cfg.hf_token,- ).to(device)+ )+ # int8 stays on host: _int8_quantize_base streams Linears to device so the bf16 base is never whole in VRAM.+ return transformer if base_precision == "int8" else transformer.to(device)def _load_fp8_prequant_transformer(transformer_cls, cfg, device):@@ -378,8 +392,13 @@ def _load_fp8_prequant_transformer(transformer_cls, cfg, device):return transformer-def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:- """Passing family keeps training and inference on one exclusion list, else the first forward raises."""+def _int8_quantize_base(+ transformer,+ family: Optional[str] = None,+ device = None,+) -> None:+ """Passing family keeps training and inference on one exclusion list, else the first forward raises. With device,+ a host-resident base moves to it one module at a time as it quantizes (peak = int8 base + one bf16 Linear)."""from core.inference.diffusion_transformer_quant import (_quiet_config,apply_small_m_padding,@@ -395,9 +414,15 @@ def _int8_quantize_base(transformer, family: Optional[str] = None) -> None:filter_fn = make_filter_fn(512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family, baked = True)),+ **({} if device is None else {"device": device}),)# After quantize_: the helper reparents the Linears it wraps. Not best-effort: a raise leaves the base quantized but not safely runnable.apply_small_m_padding(transformer, "int8", family)+ if device is not None:+ # quantize_ moves every module it visits; this catches anything outside its walk (root params / buffers).+ transformer.to(device)+ gc.collect()+ _empty_device_cache(device)# Small-M linears (conditioning vectors, LTX-2 one-token audio) stay bf16: the float8 / MX casts cost more than the GEMM.@@ -853,9 +878,9 @@ def _flux_collate(import torch# FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared across prompts.- pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)- pooled = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)- text_ids = entries[0][2].to(device = device, dtype = torch.float32)+ pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)+ pooled = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)+ text_ids = entries[0][2].to(device = device, non_blocking = True).to(dtype = torch.float32)return (pe, pooled, text_ids)@@ -994,17 +1019,21 @@ def _qwen_collate(seqs = [e[0].shape[1] for e in entries]target = max(pad_to or 0, max(seqs))pes, masks = [], []+ # Pad/cat on device: a host pad/cat is pageable and its copy blocks.for pe, mask in entries:s = pe.shape[1]+ pe = pe.to(device, non_blocking = True)if mask is None:- mask = torch.ones((1, s), dtype = torch.int64)+ mask = torch.ones((1, s), dtype = torch.int64, device = pe.device)+ else:+ mask = mask.to(device, non_blocking = True)if s < target:pe = F.pad(pe, (0, 0, 0, target - s))mask = F.pad(mask, (0, target - s))pes.append(pe)masks.append(mask)- pe_b = torch.cat(pes).to(device = device, dtype = weight_dtype)- mask_b = torch.cat(masks).to(device)+ pe_b = (pes[0] if len(pes) == 1 else torch.cat(pes)).to(dtype = weight_dtype)+ mask_b = masks[0] if len(masks) == 1 else torch.cat(masks)# A single unpadded sample keeps the legacy None mask, avoiding a behaviour change for existing single-image runs.if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:mask_b = None@@ -1088,7 +1117,7 @@ def _zimage_collate(weight_dtype,pad_to = None,):- caps = [e[0].to(device = device, dtype = weight_dtype) for e in entries]+ caps = [e[0].to(device = device, non_blocking = True).to(dtype = weight_dtype) for e in entries]return (caps,)@@ -1120,7 +1149,8 @@ def _krea2_load_conditioners(cfg, device, weight_dtype):pipe = load_krea2_pipeline(cfg.base_model, torch.bfloat16, hf_token = cfg.hf_token, with_transformer = False)- pipe.vae.to(device, dtype = torch.float32)+ # Host until the text encoders are freed, as _load_pipe_without_transformer.+ pipe.vae.to(dtype = torch.float32)return pipe, pipe.vae@@ -1159,11 +1189,9 @@ def _krea2_collate(weight_dtype,pad_to = None,):- import torch-# Fixed-length embeds, so collation is a plain concat with the mask riding along; ``pad_to`` is moot.- pe_b = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)- mask_b = torch.cat([e[1] for e in entries]).to(device)+ pe_b = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)+ mask_b = _to_device_cat([e[1] for e in entries], device)return (pe_b, mask_b)@@ -1280,8 +1308,8 @@ def _flux2_collate(import torch# text_ids are PER-SAMPLE (unlike FLUX.1's shared grid).- pe = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)- text_ids = torch.cat([e[1] for e in entries]).to(device = device, dtype = torch.float32)+ pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)+ text_ids = _to_device_cat([e[1] for e in entries], device).to(dtype = torch.float32)return (pe, text_ids)@@ -1425,8 +1453,6 @@ def _ltx2_encode_prompts(pipe, captions, device):def _ltx2_latent_affine(vae, ref):- import torch-mean = vae.latents_mean.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)std = vae.latents_std.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)# scaling_factor is 1.0 on the shipped checkpoint but is applied for fidelity to _normalize_latents.@@ -1460,12 +1486,10 @@ def _ltx2_collate(weight_dtype,pad_to = None,):- import torch-# Connector embeds are fixed at [1, 1024, 3840] via max_sequence_length, so pad_to is moot.- video = torch.cat([e[0] for e in entries]).to(device = device, dtype = weight_dtype)- audio = torch.cat([e[1] for e in entries]).to(device = device, dtype = weight_dtype)- mask = torch.cat([e[2] for e in entries]).to(device)+ video = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)+ audio = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)+ mask = _to_device_cat([e[2] for e in entries], device)return (video, audio, mask)@@ -1966,12 +1990,21 @@ def _load_pixel_tensor(path, resolution, center_crop, random_flip, rng):return _to_unit_tensor(img)-def _load_pixel_tensor_planned(path, resolution, center_crop, u_left, u_top, flip):- """Deterministic pixel-crop variant for the latent cache: crop offsets are unit fractions over the randint range; center_crop matches the uncached path."""+def _load_pixel_tensor_planned(+ path,+ resolution,+ center_crop,+ u_left,+ u_top,+ flip,+ resized = None,+):+ """Deterministic pixel-crop variant for the latent cache: crop offsets are unit fractions over the randint range; center_crop matches the uncached path.+ ``resized`` is a prior ``_open_resized`` result, reused across variants."""from PIL import Imagecw, ch = _canvas(resolution)- img, rw, rh = _open_resized(path, resolution)+ img, rw, rh = resized if resized is not None else _open_resized(path, resolution)if center_crop:left, top = (rw - cw) // 2, (rh - ch) // 2else:@@ -2027,6 +2060,7 @@ def _build_latent_cache(gated = Falsefor i, path in enumerate(image_paths):variants = []+ resized = Nonefor u_left, u_top, flip in plan[i]:canvas = cfg.resolution if canvases is None else canvases[i]key = (@@ -2038,8 +2072,12 @@ def _build_latent_cache(if entry is not None:a, b = entryelse:+ if resized is None:+ resized = _open_resized(path, canvas)px = (- _load_pixel_tensor_planned(path, canvas, cfg.center_crop, u_left, u_top, flip)+ _load_pixel_tensor_planned(+ path, canvas, cfg.center_crop, u_left, u_top, flip, resized = resized+ ).unsqueeze(0).to(device))@@ -2186,6 +2224,34 @@ def _to_device_cat(parts, device):return moved[0] if len(moved) == 1 else torch.cat(moved)+# Pinned (page-locked) host memory is not swappable, so a huge caption set stays pageable; the copies stay async.+_CAPTION_PIN_BUDGET_BYTES = 1024**3+++def _pin_caption_embeds(caption_embeds: dict, device) -> dict:+ import torch++ from core.inference.host_pinning import host_pinning_allowed++ if device != "cuda" or not caption_embeds or not host_pinning_allowed():+ return caption_embeds+ tensors = [t for entry in caption_embeds.values() for t in entry if isinstance(t, torch.Tensor)]+ if any(t.device.type != "cpu" for t in tensors):+ return caption_embeds+ if sum(t.numel() * t.element_size() for t in tensors) > _CAPTION_PIN_BUDGET_BYTES:+ return caption_embeds++ def _pin(t):+ if not isinstance(t, torch.Tensor):+ return t+ try:+ return t.contiguous().pin_memory()+ except RuntimeError:+ return t++ return {cap: type(entry)(_pin(t) for t in entry) for cap, entry in caption_embeds.items()}++def _should_compile(cfg,base_is_bnb,@@ -2544,7 +2610,6 @@ def _train_dit(torch.cuda.empty_cache()elif device == "xpu":torch.xpu.empty_cache()- # Some families keep the VAE on host while text encoders are resident.if vae is not None:vae.to(device)@@ -2596,6 +2661,7 @@ def _train_dit(torch.xpu.empty_cache()# Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.variant_rng = random.Random(cfg.seed + 1)+ caption_embeds = _pin_caption_embeds(caption_embeds, device)# Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.base_precision = _resolve_base_precision(cfg, spec, device)@@ -2628,7 +2694,9 @@ def _train_dit(# Convert AFTER the LoRA attaches so the adapter modules stay high precision.if base_precision == "int8":- _int8_quantize_base(transformer, cfg.resolved_family)+ _int8_quantize_base(transformer, cfg.resolved_family, device = device)+ # Re-collect: the move to device must not leave the optimizer holding host tensors.+ lora_params = [p for p in transformer.parameters() if p.requires_grad]if base_precision == "fp8" and not _apply_fp8_training(transformer, on_event, cfg.resolved_family):@@ -2856,7 +2924,7 @@ def _train_dit(stopped = Truebreakoptimizer.zero_grad(set_to_none = True)- step_loss = 0.0+ step_acc = Nonefor _ in range(cfg.gradient_accumulation_steps):idxs = index_sampler.next_batch(batch_size)if canvases is not None:@@ -2886,7 +2954,10 @@ def _train_dit(if shift_active:# timestep = sigma * num_train_timesteps, gathered in fp32 so bf16 rounding never skews it.timesteps = (- sigma_table[t_indices].to(device = device, dtype = torch.float32).flatten()+ sigma_table[t_indices]+ .to(device = device, non_blocking = True)+ .to(dtype = torch.float32)+ .flatten()* num_train_ts)noisy = (1.0 - sigmas) * latents + sigmas * noise@@ -2917,12 +2988,13 @@ def _train_dit(w = bell_weights[w_idx].view(-1, *([1] * (per.ndim - 1)))loss = (per * w).mean()(loss / cfg.gradient_accumulation_steps).backward()- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps+ step_acc = accumulate_loss(step_acc, loss, cfg.gradient_accumulation_steps)grad_norm = Noneif cfg.max_grad_norm and cfg.max_grad_norm > 0:# Pre-clip norm: the UI's health signal.grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))+ step_loss = loss_value(step_acc)optimizer.step()lr_sched.step()if ema is not None:diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.pyindex a222e395b5e..0ec7179d298 100644--- a/studio/backend/core/training/diffusion_h3_trainer.py+++ b/studio/backend/core/training/diffusion_h3_trainer.py@@ -92,7 +92,12 @@ from core.training.diffusion_train_common import (resolve_train_steps,train_recipe_overrides,)-from core.training.diffusion_train_extras import LoRAEMA, save_ema_adapter+from core.training.diffusion_train_extras import (+ LoRAEMA,+ accumulate_loss,+ loss_value,+ save_ema_adapter,+)# adaln_proj inputs have 2-3 rows, so an adapter there learns noise; the projections stay fp32.# A REGEX, not a list: PEFT suffix-matches lists and would also adapt token_refiner.@@ -181,10 +186,8 @@ def _load_conditioners(cfg, device):**auth,)_assert_component_grid(pipe)- # ``load_components`` builds every component on the CPU, so place them explicitly.+ # Only the conditioner moves here; the VAEs move after it is freed (_train_h3).pipe.text_encoder.to(device)- pipe.vae.to(device)- pipe.audio_vae.to(device)return pipe@@ -209,15 +212,50 @@ def _encode_prompt(pipe, caption: str, device) -> Any:f"conditioner, which needs more than {_H3_TEXT_ENCODER_LAYER} decoder layers; this "f"one has {num_layers}.")+ kwargs = dict(+ input_ids = input_ids,+ attention_mask = torch.ones_like(input_ids),+ mm_token_type_ids = mm_token_type_ids,+ use_cache = False,+ )with torch.no_grad():- out = encoder.model(- input_ids = input_ids,- attention_mask = torch.ones_like(input_ids),- mm_token_type_ids = mm_token_type_ids,- use_cache = False,- output_hidden_states = True,- )- return out.hidden_states[_H3_TEXT_ENCODER_LAYER].to("cpu", torch.float32)+ hidden = _early_exit_hidden_state(encoder, _H3_TEXT_ENCODER_LAYER, kwargs)+ if hidden is None:+ out = encoder.model(**kwargs, output_hidden_states = True)+ hidden = out.hidden_states[_H3_TEXT_ENCODER_LAYER]+ return hidden.to("cpu", torch.float32)+++class _HiddenStateReached(Exception):+ """Raised by the capture hook to stop the encoder once the read layer has run."""+++def _early_exit_hidden_state(encoder, layer: int, kwargs: dict) -> Any:+ """hidden_states[layer] is the raw output of decoder layer ``layer - 1`` (index 0 is the embeddings), so capture+ it there and stop: layers past it and the final norm never run, and the other 64 states are never kept. Text-only+ input, so no deepstack visual rows are added between layers. None (run the full encoder) on any surprise."""+ import torch++ text_model = getattr(getattr(encoder, "model", None), "language_model", None)+ layers = getattr(text_model, "layers", None)+ if not isinstance(layers, torch.nn.ModuleList) or not 0 < layer < len(layers):+ return None+ captured: list = []++ def _capture(_module, _args, output):+ captured.append(output[0] if isinstance(output, (tuple, list)) else output)+ raise _HiddenStateReached++ handle = layers[layer - 1].register_forward_hook(_capture)+ try:+ encoder.model(**kwargs, output_hidden_states = False)+ except _HiddenStateReached:+ pass+ finally:+ handle.remove()+ if len(captured) != 1 or not isinstance(captured[0], torch.Tensor):+ return None+ return captured[0]def _encode_video_stats(vae, frames, device) -> tuple[Any, Any]:@@ -280,13 +318,15 @@ def _load_transformer(cfg, device, base_precision):token = cfg.hf_token,cache_dir = cache_dir,)- return MiniMaxH3Transformer3DModel.from_pretrained(+ transformer = MiniMaxH3Transformer3DModel.from_pretrained(cfg.base_model,subfolder = "transformer",torch_dtype = torch.bfloat16,token = cfg.hf_token,cache_dir = cache_dir,- ).to(device)+ )+ # int8 stays on the host: _int8_quantize_base streams it onto the device one Linear at a time.+ return transformer if base_precision == "int8" else transformer.to(device)def _patchify(latents, patch: tuple[int, int, int]):@@ -482,7 +522,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savto_encode = sorted(set(captions))- # Phase 1: conditioning. The 63 GiB Qwen3-VL conditioner and both VAEs are resident here and nowhere else.+ # Phase 1: conditioning. The 63 GiB Qwen3-VL conditioner is resident here and nowhere else.pipe = _load_conditioners(cfg, device)caption_embeds = {cap: _encode_prompt(pipe, cap, device) for cap in to_encode}_emit(on_event, "preparing", stage = "encode_prompts", done = len(to_encode), total = len(to_encode))@@ -494,6 +534,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savtorch.cuda.empty_cache()elif device == "xpu":torch.xpu.empty_cache()+ for attr in _H3_VAE_COMPONENTS:+ module = getattr(pipe, attr, None)+ if module is not None:+ module.to(device)# One canvas from the FIRST clip's aspect ratio; other clips are cover-cropped onto it.width, height = _dataset_canvas(clip_paths[0], cfg.resolution)@@ -585,7 +629,8 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav)if base_precision == "int8":# WITH the family: it excludes adaln_proj (runs at M = 1) and supplies the pad list.- _int8_quantize_base(transformer, cfg.resolved_family)+ _int8_quantize_base(transformer, cfg.resolved_family, device = device)+ lora_params = [p for p in transformer.parameters() if p.requires_grad]ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None# dynamic=True keeps one graph per block across caption lengths; int8 stays eager like the DiT families.@@ -614,6 +659,20 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savtransformer.train()patch = tuple(transformer.config.patch_size)+ from core.inference.host_pinning import host_pinning_allowed++ if device == "cuda" and host_pinning_allowed():++ def _pin_or_keep(t):+ try:+ return t.contiguous().pin_memory()+ except RuntimeError:+ return t++ cache = [tuple(_pin_or_keep(t) for t in entry) for entry in cache]+ caption_embeds = {cap: _pin_or_keep(t) for cap, t in caption_embeds.items()}+ # Packed layout depends only on caption length (fixed canvas).+ layouts: dict[int, tuple[dict, dict]] = {}index_sampler = PermutationBatchSampler(len(clip_paths), rng)autocast = (torch.autocast(device_type = device, dtype = torch.bfloat16)@@ -630,17 +689,15 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savcompile_watch = _dynamo_compile_counters() if compiled else Nonefor opt_step in range(cfg.train_steps):optimizer.zero_grad(set_to_none = True)- step_loss = 0.0- step_video = 0.0- step_audio = 0.0+ step_acc = video_acc = audio_acc = Nonefor _ in range(cfg.gradient_accumulation_steps):index = index_sampler.next_batch(1)[0]video_a, video_b, audio_latents = cache[index]- video_a = video_a.to(device)- video_b = video_b.to(device)+ video_a = video_a.to(device, non_blocking = True)+ video_b = video_b.to(device, non_blocking = True)# A fresh posterior draw per step, exactly like encoding in the loop would give.clean_video = video_a + video_b * torch.randn_like(video_a)- clean_audio = audio_latents.to(device)+ clean_audio = audio_latents.to(device, non_blocking = True)# ONE base u through BOTH shifts: the (video, audio) sigma pair the sampler visits.u = rng.random()@@ -660,14 +717,31 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav(clean_audio - noise_audio).permute(0, 2, 1).reshape(-1, H3_AUDIO_LATENT_CHANNELS))- embeds = caption_embeds[captions[index]].to(device, weight_dtype)+ embeds = caption_embeds[captions[index]].to(device, non_blocking = True).to(weight_dtype)num_text_tokens = embeds.shape[1]- layout = _build_layout(- num_text_tokens, latent_frames, latent_h, latent_w, num_audio_latents, patch, device- )+ if num_text_tokens not in layouts:+ host_layout = _build_layout(+ num_text_tokens,+ latent_frames,+ latent_h,+ latent_w,+ num_audio_latents,+ patch,+ "cpu",+ )+ layouts[num_text_tokens] = (+ host_layout,+ {+ k: v.to(device) if isinstance(v, torch.Tensor) else v+ for k, v in host_layout.items()+ },+ )+ host_layout, layout = layouts[num_text_tokens]timestep, timestep_indices = _row_timesteps(- layout, num_text_tokens, 1.0 - sigma_video, 1.0 - sigma_audio, device+ host_layout, num_text_tokens, 1.0 - sigma_video, 1.0 - sigma_audio, "cpu")+ timestep = timestep.to(device, non_blocking = True)+ timestep_indices = timestep_indices.to(device, non_blocking = True)with autocast:pred_video, pred_audio = transformer(hidden_states = video_rows[None],@@ -687,13 +761,17 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav# Unweighted: both streams share every adapted matrix.loss = loss_video + loss_audio(loss / cfg.gradient_accumulation_steps).backward()- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps- step_video += float(loss_video.detach()) / cfg.gradient_accumulation_steps- step_audio += float(loss_audio.detach()) / cfg.gradient_accumulation_steps+ ga = cfg.gradient_accumulation_steps+ step_acc = accumulate_loss(step_acc, loss, ga)+ video_acc = accumulate_loss(video_acc, loss_video, ga)+ audio_acc = accumulate_loss(audio_acc, loss_audio, ga)grad_norm = Noneif cfg.max_grad_norm and cfg.max_grad_norm > 0:grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))+ step_loss = loss_value(step_acc)+ step_video = loss_value(video_acc)+ step_audio = loss_value(audio_acc)optimizer.step()lr_sched.step()if ema is not None:diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.pyindex 883839a0e3b..b2c2e27e85d 100644--- a/studio/backend/core/training/diffusion_lora_trainer.py+++ b/studio/backend/core/training/diffusion_lora_trainer.py@@ -82,7 +82,12 @@ from core.training.diffusion_samples import (sample_seed,SampleRoundStopped,)-from core.training.diffusion_train_extras import BucketBatchSampler, cover_resize_dims+from core.training.diffusion_train_extras import (+ BucketBatchSampler,+ accumulate_loss,+ cover_resize_dims,+ loss_value,+)from core.training.diffusion_checkpoint import (clear_own_checkpoints,discard_preexisting_checkpoints,@@ -96,6 +101,12 @@ from core.training.diffusion_checkpoint import ()+def _parts_to_device(parts, device):+ import torch+ moved = [p.to(device, non_blocking = True) for p in parts]+ return moved[0] if len(moved) == 1 else torch.cat(moved)++def compute_sdxl_add_time_ids(resolution: int) -> tuple[int, int, int, int, int, int]:"""Square default with no crop offset; the training loop derives per-image ids from the actual crop."""return (resolution, resolution, 0, 0, resolution, resolution)@@ -737,7 +748,7 @@ def run_diffusion_lora_training(stopped = Truebreakoptimizer.zero_grad(set_to_none = True)- step_loss = 0.0+ step_acc = Nonefor _ in range(cfg.gradient_accumulation_steps):idx, img_paths, captions = _next_batch()if latent_cache is not None:@@ -776,8 +787,10 @@ def run_diffusion_lora_training(noisy = noise_scheduler.add_noise(latents, noise, timesteps)if precompute:- prompt_embeds = torch.cat([caption_embeds[c][0] for c in captions]).to(device)- pooled = torch.cat([caption_embeds[c][1] for c in captions]).to(device)+ prompt_embeds = _parts_to_device(+ [caption_embeds[c][0] for c in captions], device+ )+ pooled = _parts_to_device([caption_embeds[c][1] for c in captions], device)else:prompt_embeds, pooled = _encode_sdxl_prompts(captions, tokenizers, text_encoders, device@@ -808,7 +821,7 @@ def run_diffusion_lora_training(loss = F.mse_loss(model_pred.float(), target.float(), reduction = "mean")scaler.scale(loss / cfg.gradient_accumulation_steps).backward()- step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps+ step_acc = accumulate_loss(step_acc, loss, cfg.gradient_accumulation_steps)micro += 1# max_grad_norm <= 0 disables clipping; clip_grad_norm_(0.0) would zero all grads.@@ -819,6 +832,7 @@ def run_diffusion_lora_training(grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))if not math.isfinite(grad_norm):grad_norm = None+ step_loss = loss_value(step_acc)scale_before = scaler.get_scale() if scaler.is_enabled() else Nonescaler.step(optimizer)scaler.update()diff --git a/studio/backend/core/training/diffusion_train_extras.py b/studio/backend/core/training/diffusion_train_extras.pyindex 6e39749c749..aca88950cd7 100644--- a/studio/backend/core/training/diffusion_train_extras.py+++ b/studio/backend/core/training/diffusion_train_extras.py@@ -15,6 +15,23 @@ from pathlib import Pathfrom typing import Any, Iterable, Optional+def accumulate_loss(acc: Any, loss: Any, accumulation_steps: int) -> Any:+ """``acc + float(loss) / accumulation_steps`` without a per-micro-step host sync: an fp64 device sum is the same+ IEEE arithmetic as the Python float one. Devices without fp64 (MPS, Arc / integrated XPUs) keep the Python float.+ Read with loss_value."""+ detached = loss.detach()+ if detached.device.type not in ("cpu", "cuda"):+ from core.inference.video_minimax_h3_pdd import device_has_fp64+ if not device_has_fp64(detached.device):+ return (acc if acc is not None else 0.0) + float(detached) / accumulation_steps+ term = detached.double() / accumulation_steps+ return term if acc is None else acc.add_(term)+++def loss_value(acc: Any) -> float:+ return 0.0 if acc is None else float(acc)++# Effective decay = min(decay, (1 + n) / (WARMUP_OFFSET + n))._EMA_WARMUP_OFFSET = 10.0@@ -69,17 +86,36 @@ class LoRAEMA:step = self.updates + 1return min(self.decay, step / (_EMA_WARMUP_OFFSET + step))+ def _update_lists(self, model: Any) -> tuple[list, list]:+ """(shadows, params) for ``model``, cached per model."""+ import weakref++ cached = getattr(self, "_lists", None)+ if cached is not None and cached[0]() is model:+ return cached[1], cached[2]+ shadows, params = [], []+ for name, p in model.named_parameters():+ shadow = self._shadow.get(name)+ if shadow is None:+ continue+ shadows.append(shadow)+ params.append(p.detach())+ try:+ self._lists = (weakref.ref(model), shadows, params)+ except TypeError: # not weak-referenceable: rebuild each call+ self._lists = None+ return shadows, params+def update(self, model: Any) -> None:"""Apply one EMA update from ``model``'s current trainable params."""import torchdecay = self.effective_decay()- with torch.no_grad():- for name, p in model.named_parameters():- shadow = self._shadow.get(name)- if shadow is None:- continue- shadow.mul_(decay).add_(p.detach(), alpha = 1.0 - decay)+ shadows, params = self._update_lists(model)+ if shadows:+ with torch.no_grad():+ torch._foreach_mul_(shadows, decay)+ torch._foreach_add_(shadows, params, alpha = 1.0 - decay)self.updates += 1def state_dict(self) -> dict[str, Any]:diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.pyindex 67e0cd78dc9..6ae526512bc 100644--- a/studio/backend/tests/test_diffusion_h3_trainer.py+++ b/studio/backend/tests/test_diffusion_h3_trainer.py@@ -796,7 +796,7 @@ def test_int8_training_applies_h3s_small_m_guards():assert pad_tokens_for_scheme("int8", "minimax-h3")src = Path(h3.__file__).read_text()- assert "_int8_quantize_base(transformer, cfg.resolved_family)" in src+ assert "_int8_quantize_base(transformer, cfg.resolved_family, device = device)" in srcdit_src = Path(dit.__file__).read_text()body = dit_src.split("def _int8_quantize_base", 1)[1].split("\ndef ", 1)[0]# training keeps the baked list: the streamed-inference text-stream scope never reaches it@@ -1767,7 +1767,9 @@ def test_h3_routes_its_transformer_through_the_shared_regional_compile(diffusion_h3_trainer, "_load_transformer", lambda cfg, device, precision: _Tiny())monkeypatch.setattr(- dit, "_int8_quantize_base", lambda transformer, family: order.append("int8")+ dit,+ "_int8_quantize_base",+ lambda transformer, family, device = None: order.append("int8"),)monkeypatch.setattr(dit, "_maybe_compile_transformer", _compile)monkeypatch.setattr(dit, "_make_optimizer", _optimizer)diff --git a/studio/backend/tests/test_diffusion_train_hot_loop.py b/studio/backend/tests/test_diffusion_train_hot_loop.pynew file mode 100644index 00000000000..fd2aae5d621--- /dev/null+++ b/studio/backend/tests/test_diffusion_train_hot_loop.py@@ -0,0 +1,417 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Diffusion training hot loop: no per-micro-step host syncs, one decode per cached image, batched EMA, conditioning+placement and the streamed int8 load. Every change here must leave the numbers bit-identical."""++from __future__ import annotations++import sys+import types++import pytest++torch = pytest.importorskip("torch")++from core.training import diffusion_dit_trainer as dit # noqa: E402+from core.training import diffusion_train_extras as extras # noqa: E402+++class _ScalarReads(torch.utils._python_dispatch.TorchDispatchMode):+ """Counts tensor -> Python scalar reads (each one a host sync on an accelerator)."""++ def __init__(self):+ super().__init__()+ self.reads = 0++ def __torch_dispatch__(+ self,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ if func is torch.ops.aten._local_scalar_dense.default:+ self.reads += 1+ return func(*args, **(kwargs or {}))+++def test_loss_accumulation_matches_the_python_float_sum_without_scalar_reads():+ g = torch.Generator().manual_seed(0)+ losses = [torch.rand((), generator = g) * 3 for _ in range(8)]+ expected = 0.0+ for loss in losses:+ expected += float(loss) / 8+ acc = None+ with _ScalarReads() as mode:+ for loss in losses:+ acc = extras.accumulate_loss(acc, loss, 8)+ assert mode.reads == 0+ with _ScalarReads() as mode:+ got = extras.loss_value(acc)+ assert mode.reads == 1+ assert got == expected # bitwise: fp64 device math is the Python double math+ assert extras.loss_value(None) == 0.0+++def test_loss_accumulation_keeps_the_differentiable_loss_untouched():+ w = torch.ones(3, requires_grad = True)+ loss = (w * 2).sum()+ acc = extras.accumulate_loss(None, loss, 2)+ assert not acc.requires_grad+ loss.backward()+ assert torch.equal(w.grad, torch.full((3,), 2.0))+++def test_loss_accumulation_keeps_the_python_float_on_a_device_without_fp64(monkeypatch):+ from core.inference import video_minimax_h3_pdd as pdd++ class _ArcLoss:+ device = torch.device("xpu")++ def __init__(self, value):+ self.value = value++ def detach(self):+ return self++ def __float__(self):+ return self.value++ def double(self):+ raise AssertionError("Required aspect fp64 is not supported on the device")++ monkeypatch.setitem(pdd._FP64_BY_DEVICE, ("xpu", None), False)+ acc = None+ for v in (0.25, 1.5, 3.0):+ acc = extras.accumulate_loss(acc, _ArcLoss(v), 4)+ assert isinstance(acc, float) and extras.loss_value(acc) == 0.25 / 4 + 1.5 / 4 + 3.0 / 4+++def _ema_reference(shadows, params, decay):+ for s, p in zip(shadows, params):+ s.mul_(decay).add_(p, alpha = 1.0 - decay)+++def test_lora_ema_batched_update_is_bitwise_the_per_tensor_update():+ torch.manual_seed(0)+ model = torch.nn.Sequential(torch.nn.Linear(16, 8), torch.nn.Linear(8, 4))+ model[0].bias.requires_grad_(False)+ ema = extras.LoRAEMA(model, decay = 0.97)+ ref = {n: t.clone() for n, t in ema._shadow.items()}+ for step in range(5):+ with torch.no_grad():+ for p in model.parameters():+ p.add_(torch.randn_like(p) * 0.1)+ decay = ema.effective_decay()+ names = [n for n, p in model.named_parameters() if n in ref]+ _ema_reference(+ [ref[n] for n in names],+ [dict(model.named_parameters())[n].detach() for n in names],+ decay,+ )+ ema.update(model)+ for n in ref:+ assert torch.equal(ema._shadow[n], ref[n]), (step, n)+ assert ema.updates == 5+ assert "0.bias" not in ema._shadow+++def test_lora_ema_update_walks_the_parameters_once():+ model = torch.nn.Linear(4, 4)+ ema = extras.LoRAEMA(model, decay = 0.9)+ walks = []+ original = model.named_parameters++ def _counting(*a, **k):+ walks.append(1)+ return original(*a, **k)++ model.named_parameters = _counting+ for _ in range(4):+ ema.update(model)+ assert len(walks) == 1+ # A different model object rebuilds the lists.+ other = torch.nn.Linear(4, 4)+ ema.update(other)+ assert torch.equal(ema._shadow["weight"], ema._shadow["weight"])+++def test_latent_cache_decodes_each_image_once_with_identical_variants(monkeypatch, tmp_path):+ from PIL import Image+ import numpy as np++ path = tmp_path / "a.png"+ Image.fromarray(np.random.RandomState(0).randint(0, 255, (90, 70, 3), dtype = np.uint8)).save(+ path+ )+ opens = []+ real_open = dit._open_resized++ def _counting_open(p, res):+ opens.append(p)+ return real_open(p, res)++ monkeypatch.setattr(dit, "_open_resized", _counting_open)+ plan = [[(0.0, 0.0, False), (0.9, 0.9, True), (0.5, 0.1, False), (0.2, 0.7, True)]]+ got = []+ spec = types.SimpleNamespace(+ encode_latent_stats = lambda vae, px: (got.append(px.clone()) or px[:, :1], None)+ )+ cfg = types.SimpleNamespace(+ resolution = 48, center_crop = False, cache_variants = 4, random_flip = True, seed = 0+ )+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", "1")+ cache = dit._build_latent_cache(+ spec,+ None,+ [str(path)],+ cfg,+ "cpu",+ torch.float32,+ None,+ lambda: False,+ plan = plan,+ )+ assert len(cache[0]) == 4+ assert len(opens) == 1+ for px, (u_left, u_top, flip) in zip(got, plan[0]):+ expected = dit._load_pixel_tensor_planned(str(path), 48, False, u_left, u_top, flip)+ assert torch.equal(px[0], expected)+++def _old_qwen_collate(+ entries,+ weight_dtype,+ pad_to = None,+):+ import torch.nn.functional as F++ seqs = [e[0].shape[1] for e in entries]+ target = max(pad_to or 0, max(seqs))+ pes, masks = [], []+ for pe, mask in entries:+ s = pe.shape[1]+ if mask is None:+ mask = torch.ones((1, s), dtype = torch.int64)+ if s < target:+ pe = F.pad(pe, (0, 0, 0, target - s))+ mask = F.pad(mask, (0, target - s))+ pes.append(pe)+ masks.append(mask)+ pe_b = torch.cat(pes).to(dtype = weight_dtype)+ mask_b = torch.cat(masks)+ if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:+ mask_b = None+ return pe_b, mask_b++[email protected]("pad_to", [None, 9])[email protected]("n", [1, 3])+def test_qwen_collate_pads_on_device_identically(n, pad_to):+ g = torch.Generator().manual_seed(n)+ entries = []+ for i in range(n):+ s = 4 + i+ mask = None if i % 2 == 0 else torch.ones((1, s), dtype = torch.int64)+ entries.append((torch.randn(1, s, 6, generator = g), mask))+ pe, mask = dit._qwen_collate(entries, "cpu", torch.bfloat16, pad_to = pad_to)+ ref_pe, ref_mask = _old_qwen_collate(entries, torch.bfloat16, pad_to)+ assert pe.dtype == torch.bfloat16 and torch.equal(pe, ref_pe)+ assert (mask is None) == (ref_mask is None)+ if mask is not None:+ assert torch.equal(mask, ref_mask)+++def test_fixed_length_collates_match_a_host_cat():+ a = (torch.randn(1, 5, 4), torch.randn(1, 4), torch.zeros(5, 3))+ b = (torch.randn(1, 5, 4), torch.randn(1, 4), torch.zeros(5, 3))+ pe, pooled, ids = dit._flux_collate([a, b], "cpu", torch.bfloat16)+ assert torch.equal(pe, torch.cat([a[0], b[0]]).to(torch.bfloat16))+ assert torch.equal(pooled, torch.cat([a[1], b[1]]).to(torch.bfloat16))+ assert ids.dtype == torch.float32+ v = [(torch.randn(1, 3, 2), torch.randn(1, 3, 2), torch.ones(1, 3)) for _ in range(2)]+ video, audio, mask = dit._ltx2_collate(v, "cpu", torch.bfloat16)+ assert torch.equal(video, torch.cat([e[0] for e in v]).to(torch.bfloat16))+ assert torch.equal(mask, torch.cat([e[2] for e in v]))+++def test_caption_embeds_pin_only_on_cuda_and_under_budget(monkeypatch):+ embeds = {"a": (torch.zeros(1, 2), torch.zeros(1))}+ assert dit._pin_caption_embeds(embeds, "cpu") is embeds+ pinned = []+ monkeypatch.setattr("core.inference.host_pinning.host_pinning_allowed", lambda *a: True)+ monkeypatch.setattr(torch.Tensor, "pin_memory", lambda self: pinned.append(1) or self)+ out = dit._pin_caption_embeds(embeds, "cuda")+ assert len(pinned) == 2 and set(out) == {"a"} and isinstance(out["a"], tuple)+ monkeypatch.setattr(dit, "_CAPTION_PIN_BUDGET_BYTES", 1)+ pinned.clear()+ assert dit._pin_caption_embeds(embeds, "cuda") is embeds and not pinned+++def test_sampled_timesteps_and_sigmas_are_unchanged(monkeypatch):+ pytest.importorskip("diffusers")+ from diffusers import FlowMatchEulerDiscreteScheduler++ sched = FlowMatchEulerDiscreteScheduler(shift = 3.0)+ torch.manual_seed(5)+ ts, idx = dit._sample_timesteps(sched, 4, "cpu")+ torch.manual_seed(5)+ _ts, _idx = dit._sample_timesteps(sched, 4, "cpu")+ assert torch.equal(idx, _idx)+ assert torch.equal(ts, sched.timesteps[idx])+ sig = dit._gather_sigmas(sched.sigmas, idx, "cpu", torch.bfloat16, 4)+ assert sig.shape == (4, 1, 1, 1)+ assert torch.equal(sig.flatten(), sched.sigmas[idx].to(torch.bfloat16))+++def test_conditioner_load_keeps_the_vae_on_the_host():+ moves = []++ class _Vae:+ def to(self, *args, **kwargs):+ moves.append((args, kwargs))+ return self++ class _PipeCls:+ @staticmethod+ def from_pretrained(*_a, **_k):+ return types.SimpleNamespace(vae = _Vae())++ cfg = types.SimpleNamespace(base_model = "x", hf_token = None)+ pipe, vae = dit._load_pipe_without_transformer(_PipeCls, cfg, "cuda")+ assert moves == [((), {"dtype": torch.float32})]+++def test_int8_base_streams_onto_the_device(monkeypatch):+ seen = {}++ def _fake_quantize(+ model,+ config,+ filter_fn = None,+ device = None,+ ):+ seen["device"] = device++ fake = types.ModuleType("torchao.quantization")+ fake.Int8WeightOnlyConfig = lambda: object()+ fake.quantize_ = _fake_quantize+ monkeypatch.setitem(sys.modules, "torchao", types.ModuleType("torchao"))+ monkeypatch.setitem(sys.modules, "torchao.quantization", fake)+ model = torch.nn.Linear(8, 8)+ dit._int8_quantize_base(model, None, device = "cpu")+ assert seen["device"] == "cpu"+ # The legacy call (no device) stays a pure in-place quantize.+ seen.clear()+ dit._int8_quantize_base(model, None)+ assert seen["device"] is None++ loads = []++ class _Model:+ def to(self, device):+ loads.append(device)+ return self++ class _Cls:+ @staticmethod+ def from_pretrained(*_a, **_k):+ return _Model()++ cfg = types.SimpleNamespace(base_model = "x", hf_token = None, resolved_family = None)+ dit._load_dit_transformer(_Cls, cfg, "cuda", "int8")+ assert loads == []+ dit._load_dit_transformer(_Cls, cfg, "cuda", "bf16")+ assert loads == ["cuda"]+++def _tiny_qwen3_vl(num_layers):+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "Qwen3VLForConditionalGeneration"):+ pytest.skip("transformers without Qwen3-VL")+ config = transformers.Qwen3VLConfig(+ text_config = dict(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = num_layers,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 8,+ rope_scaling = {"rope_type": "default", "mrope_section": [1, 1, 2]},+ ),+ vision_config = dict(+ depth = 1,+ hidden_size = 32,+ intermediate_size = 64,+ num_heads = 4,+ out_hidden_size = 32,+ deepstack_visual_indexes = [0],+ ),+ )+ torch.manual_seed(0)+ return transformers.Qwen3VLForConditionalGeneration(config).eval()++[email protected]("layer", [1, 3, 5])+def test_h3_text_encoder_early_exit_is_bitwise_and_skips_the_tail(layer):+ from core.training.diffusion_h3_trainer import _early_exit_hidden_state++ model = _tiny_qwen3_vl(6)+ ids = torch.randint(0, 64, (1, 9), generator = torch.Generator().manual_seed(1))+ kwargs = dict(input_ids = ids, attention_mask = torch.ones_like(ids), use_cache = False)+ layers = model.model.language_model.layers+ ran = []+ hooks = [+ m.register_forward_pre_hook(lambda _m, _a, i = i: ran.append(i)) for i, m in enumerate(layers)+ ]+ try:+ with torch.no_grad():+ full = model.model(**kwargs, output_hidden_states = True).hidden_states[layer]+ ran.clear()+ fast = _early_exit_hidden_state(model, layer, kwargs)+ assert ran == list(range(layer))+ assert torch.equal(fast, full)+ # Repeat: a raised early exit must not leave the encoder patched.+ assert torch.equal(_early_exit_hidden_state(model, layer, kwargs), full)+ assert len(model.model(**kwargs, output_hidden_states = True).hidden_states) == 7+ finally:+ for h in hooks:+ h.remove()+ # Out of range (would need the normed last state): the caller runs the full encoder instead.+ assert _early_exit_hidden_state(model, 6, kwargs) is None+++def test_h3_vaes_reach_the_device_only_after_the_conditioner_is_freed(monkeypatch):+ from core.training import diffusion_h3_trainer as h3++ order = []++ class _Placed:+ def __init__(self, name):+ self.name = name++ def to(self, device):+ order.append((self.name, device))+ return self++ class _Pipe:+ def __init__(self):+ self.text_encoder = _Placed("text_encoder")+ self.vae = _Placed("vae")+ self.audio_vae = _Placed("audio_vae")++ def load_components(self, **_k):+ pass++ class _Modular:+ @staticmethod+ def from_pretrained(*_a, **_k):+ return _Pipe()++ monkeypatch.setitem(sys.modules, "diffusers", types.SimpleNamespace(ModularPipeline = _Modular))+ monkeypatch.setattr(h3, "_assert_component_grid", lambda pipe: None)+ monkeypatch.setattr("core.inference.diffusion.hub_cache_dir", lambda: "/hub")+ h3._load_conditioners(types.SimpleNamespace(base_model = "m", hf_token = None), "cuda")+ assert order == [("text_encoder", "cuda")]diff --git a/studio/backend/tests/test_integrated_gpu_no_host_pin.py b/studio/backend/tests/test_integrated_gpu_no_host_pin.pyindex 44ba29766a7..c31ade49232 100644--- a/studio/backend/tests/test_integrated_gpu_no_host_pin.py+++ b/studio/backend/tests/test_integrated_gpu_no_host_pin.py@@ -699,6 +699,8 @@ def test_only_deviceless_sites_ask_the_predicate_without_a_device():# device == "cuda" (the current device) is checked first("core/training/diffusion_dit_trainer.py", "_build_latent_cache"),("core/training/diffusion_dit_trainer.py", "_load_warm_conditioning"),+ ("core/training/diffusion_dit_trainer.py", "_pin_caption_embeds"),+ ("core/training/diffusion_h3_trainer.py", "_train_h3"),("core/training/diffusion_lora_trainer.py", "_build_sdxl_latent_cache"),}bare = set()