Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep INT8 and compile on conventional video models when they have to offload (#12289)
Daniel Hane8b2bb2f959b
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex ef573ad7983..db864631521 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -210,8 +210,9 @@ def _pin_host_weights(module: Any,host: dict,logger: Any = None,+ buffer_host: Optional[dict] = None,) -> int:- """Pack kept host weights into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""+ """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""import torchmode = _pin_mode()@@ -226,6 +227,16 @@ def _pin_host_weights(and p.data.is_contiguous()and not p.data.is_pinned()]+ buffers = [+ (name, b)+ for name, b in _named_buffers(module)+ if (buffer_host or {}).get(name) is b+ and b.device.type == "cpu"+ and _keepable_buffer(b)+ and b.is_contiguous()+ and not b.is_pinned()+ ]+ params = params + [("\0" + name, b) for name, b in buffers]sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]if not sizes or sum(sizes) == 0:return 0@@ -288,11 +299,40 @@ def _pin_host_weights()return 0for name, p, view in placed:+ if name.startswith("\0"):+ name = name[1:]+ _set_buffer(module, name, view)+ buffer_host[name] = view+ continuep.data = viewhost[name] = viewreturn need+def _named_buffers(module: Any) -> list:+ named = getattr(module, "named_buffers", None)+ return list(named()) if callable(named) else []+++def _keepable_buffer(buffer: Any) -> bool:+ import torch+ return type(buffer) is torch.Tensor+++def _buffer_version(buffer: Any) -> Optional[int]:+ """``_version``, or None for an inference tensor (moved under inference_mode), which has no version counter."""+ try:+ return None if buffer.is_inference() else int(buffer._version)+ except Exception: # noqa: BLE001 - untracked: the stock copy handles it+ return None+++def _set_buffer(module: Any, name: str, tensor: Any) -> None:+ prefix, _, leaf = name.rpartition(".")+ owner = module.get_submodule(prefix) if prefix else module+ owner._buffers[leaf] = tensor++def _wrap_cpu_offload_hook(hook: Any,module: Any,@@ -305,20 +345,45 @@ def _wrap_cpu_offload_hook(module.__dict__[_KEEP_ATTR] = statestate.setdefault("owner", {})host, owner, version = state["host"], state["owner"], state["version"]+ # Plain buffers too (torchao-free int8): the stock offload re-copied them every call, 6.3 s vs a 1.2 s denoise on Wan2.2-5B.+ # Kept while the device copy is the same tensor at the same version.+ buffer_host = state.setdefault("buffer_host", {})+ buffer_version = state.setdefault("buffer_version", {})def _capture(mod: Any) -> None:host.clear()owner.clear()+ buffer_host.clear()for name, p in mod.named_parameters():if p.device.type == "cpu" and _keepable(p):host[name] = p.dataowner[name] = p+ for name, b in _named_buffers(mod):+ if b.device.type == "cpu" and _keepable_buffer(b):+ buffer_host[name] = b_capture(module)version.clear()+ buffer_version.clear()init_hook, pre_forward = hook.init_hook, hook.pre_forwarddef _init_hook(mod: Any) -> Any:+ for name, b in _named_buffers(mod):+ kept = buffer_host.get(name)+ seen = buffer_version.get(name)+ if (+ kept is None+ or b.device.type == "cpu"+ or seen is None+ or seen[0] is not b+ or seen[1] != _buffer_version(b)+ or seen[2] != b.data_ptr()+ ):+ continue+ try:+ _set_buffer(mod, name, kept)+ except Exception: # noqa: BLE001 - the stock copy below handles it+ passfor name, p in mod.named_parameters():kept = host.get(name)seen = version.get(name)@@ -339,11 +404,12 @@ def _wrap_cpu_offload_hook(out = init_hook(mod)_capture(mod)version.clear()+ buffer_version.clear()return outdef _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:# `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.- onload = not version and bool(host)+ onload = not version and not buffer_version and bool(host or buffer_host)if onload:for name, p in mod.named_parameters():kept = host.get(name)@@ -354,11 +420,15 @@ def _wrap_cpu_offload_hook():host.pop(name, None)owner.pop(name, None)+ current = dict(_named_buffers(mod))+ for name in list(buffer_host):+ if current.get(name) is not buffer_host[name]:+ buffer_host.pop(name, None)if onload and not state.get("pin_tried"):# Once per module, on its first onload, so loading pays nothing.state["pin_tried"] = Truetry:- pinned = _pin_host_weights(mod, host, logger)+ pinned = _pin_host_weights(mod, host, logger, buffer_host = buffer_host)except Exception: # noqa: BLE001 - pinning is an optimisation, never a failurepinned = 0if pinned and logger is not None:@@ -372,6 +442,13 @@ def _wrap_cpu_offload_hook(for name, p in mod.named_parameters():if name in host and owner.get(name) is p and p.device.type != "cpu":version[name] = (p, p._version, p.data_ptr())+ for name, b in _named_buffers(mod):+ if (+ name in buffer_host+ and b.device.type != "cpu"+ and _buffer_version(b) is not None+ ):+ buffer_version[name] = (b, _buffer_version(b), b.data_ptr())return outtry:@@ -1608,6 +1685,259 @@ def _streamed_pin_plan(return pin_transformer, pin_encoders+BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_BACKGROUND_PIN"+_BG_PIN_ATTR = "_unsloth_background_pin"+_PENDING_PINS_ATTR = "_unsloth_background_pins"+_BACKGROUND_PIN_REQUEST_ATTR = "_unsloth_background_pin_requested"+++def request_background_pins(pipe: Any) -> None:+ """Ask the next group offload apply on ``pipe`` to pin off the load path (see _GroupPinner)."""+ try:+ setattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, True)+ except Exception: # noqa: BLE001 - a pipe refusing attributes pins eagerly, as before+ pass+++def _background_pin_enabled() -> bool:+ return (os.environ.get(BACKGROUND_PIN_ENV) or "").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _offload_groups(module: Any) -> list:+ """The diffusers offload groups hooked under ``module``, in registration (block) order."""+ try:+ from diffusers.hooks import group_offloading as go+ name = getattr(go, "_GROUP_OFFLOADING", "group_offloading")+ except Exception: # noqa: BLE001+ return []+ groups: list = []+ seen: set = set()+ for sub in module.modules():+ registry = getattr(sub, "_diffusers_hook", None)+ get_hook = getattr(registry, "get_hook", None)+ group = getattr(get_hook(name), "group", None) if callable(get_hook) else None+ if group is not None and id(group) not in seen:+ seen.add(id(group))+ groups.append(group)+ return groups+++class _GroupPinner:+ """Pins a streamed module's offload groups on a worker thread, first block first.++ Pinning at load reads every streamed byte from disk before the load returns: 125-139 s of LTX-2.3's 37 GB DiT,+ 20-44 s of Wan2.2-5B's text encoder on Colab. Off the load path the read overlaps the prompt encode and the+ first compile. A group's ``onload_`` waits for that group only, and the swap happens before the wait releases,+ so no group is ever onloaded while its host copy is being replaced."""++ def __init__(+ self,+ module: Any,+ groups: list,+ device: Any,+ logger: Any = None,+ ):+ import threading++ self.module = module+ self.label = type(module).__name__+ self.groups = groups+ self.device = device+ self.logger = logger+ self.pinned = 0+ self.waited_s = 0.0+ self._done = {id(g): threading.Event() for g in groups}+ self._stop = threading.Event()+ self._lock = threading.Lock()+ self._thread = threading.Thread(target = self._run, name = "unsloth-offload-pin", daemon = True)+ self._started = False+ for group in groups:+ setattr(group, _BG_PIN_ATTR, self)++ def start(self) -> None:+ with self._lock:+ if not self._started:+ self._started = True+ self._thread.start()++ def wait(self, group: Any) -> None:+ import threading++ event = self._done.get(id(group))+ if event is None or threading.current_thread() is self._thread:+ return+ self.start()+ if not event.is_set():+ import time++ began = time.perf_counter()+ event.wait()+ self.waited_s += time.perf_counter() - began++ def join(self, timeout: Optional[float] = None) -> bool:+ """Wait for the worker; True once it has exited (or when called from the worker itself)."""+ import threading++ if threading.current_thread() is self._thread:+ return True+ self._thread.join(timeout)+ return not self._thread.is_alive()++ def stop(self, timeout: Optional[float] = None) -> None:+ self._stop.set()+ if self._started:+ self._thread.join(timeout)++ def _unpinned(self, group: Any) -> list:+ import torch+ return [+ (tensor, src)+ for tensor, src in list(group.cpu_param_dict.items())+ if type(src) is torch.Tensor+ and src.device.type == "cpu"+ and src.numel() > 0+ and not src.is_pinned()+ ]++ def _run(self) -> None:+ import time++ import torch++ start = time.perf_counter()+ failed = None+ try:+ if (+ getattr(self.device, "type", None) == "cuda"+ and getattr(self.device, "index", None) is not None+ ):+ torch.cuda.set_device(self.device)+ for group in self.groups:+ if self._stop.is_set():+ break+ # One pin per tensor, exactly what the eager apply makes: chunk views streamed ~10% slower on LTX-2.3.+ placed = [(tensor, src, src.pin_memory()) for tensor, src in self._unpinned(group)]+ cpu = group.cpu_param_dict+ for tensor, src, pinned in placed:+ if cpu.get(tensor) is src:+ cpu[tensor] = pinned+ if tensor.device.type == "cpu" and tensor.data_ptr() == src.data_ptr():+ tensor.data = pinned+ self.pinned += src.nbytes+ self._done[id(group)].set()+ except Exception as exc: # noqa: BLE001 - diffusers pins what is left on each onload+ failed = exc+ finally:+ for event in self._done.values():+ event.set()+ if self.logger is not None:+ try:+ if failed is not None:+ self.logger.warning(+ "diffusion.memory: background pinning of %s stopped (%s); the rest pins on each onload",+ self.label,+ failed,+ )+ self.logger.info(+ "diffusion.memory: pinned %.1f GiB of %s host weights in the background in %.1f s "+ "(onloads waited %.1f s)",+ self.pinned / 2**30,+ self.label,+ time.perf_counter() - start,+ self.waited_s,+ )+ except Exception: # noqa: BLE001+ pass+++def install_group_pin_wait() -> bool:+ """Make a diffusers offload group wait for its background pin before it onloads."""+ try:+ import torch+ from diffusers.hooks import group_offloading as go+ except Exception: # noqa: BLE001+ return False+ group_cls = getattr(go, "ModuleGroup", None)+ original = getattr(group_cls, "onload_", None)+ if original is None or getattr(original, "_unsloth_pin_wait", False):+ return False++ @functools.wraps(original)+ def onload_(self, *args: Any, **kwargs: Any) -> Any:+ pinner = getattr(self, _BG_PIN_ATTR, None)+ if pinner is not None:+ pinner.wait(self)+ return original(self, *args, **kwargs)++ disable = getattr(getattr(torch, "compiler", None), "disable", None)+ wrapped = disable(onload_) if callable(disable) else onload_+ wrapped._unsloth_pin_wait = True+ group_cls.onload_ = wrapped+ return True+++def start_background_pins(pipe: Any) -> int:+ """Start the pinners the load deferred; returns how many."""+ pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())+ for pinner in pinners:+ pinner.start()+ return len(pinners)+++def finish_background_pins(pipe: Any, cancel: Any = None) -> float:+ """Block until every deferred pinner on ``pipe`` is done; returns the seconds waited.++ A render that overlaps the pinner ran its warm renders 2.5 to 3 s slower on an A100 (LTX-2.3, n=15 per arm);+ letting the pinner finish first matched the eager pin. So the pin overlaps the idle time after the load, not+ the first render."""+ import time++ start = time.perf_counter()+ for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):+ pinner.start()+ # Polled so a cancelled request leaves now; the pin itself keeps running for the next render.+ while not pinner.join(0.25):+ if cancel is not None and cancel.is_set():+ return time.perf_counter() - start+ return time.perf_counter() - start+++def stop_background_pins(pipe: Any, timeout: Optional[float] = 30.0) -> None:+ for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):+ try:+ pinner.stop(timeout)+ except Exception: # noqa: BLE001 - teardown is best effort+ pass+++def _drop_deferred_pinning(pipe: Any, module: Any) -> None:+ pending = getattr(pipe, _PENDING_PINS_ATTR, None)+ if pending:+ pending[:] = [pinner for pinner in pending if pinner.module is not module]+++def _defer_pinning(pipe: Any, module: Any, device: Any, logger: Any) -> bool:+ groups = _offload_groups(module)+ if not groups:+ return False+ pinner = _GroupPinner(module, groups, device, logger)+ pending = getattr(pipe, _PENDING_PINS_ATTR, None)+ if pending is None:+ pending = []+ try:+ setattr(pipe, _PENDING_PINS_ATTR, pending)+ except Exception: # noqa: BLE001 - nowhere to park it: pin now, like the eager path+ pinner.start()+ return True+ pending.append(pinner)+ return True++def install_group_offload_buffer_restore() -> bool:"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""try:@@ -1650,6 +1980,7 @@ def _apply_group_offload(*,stream_text_encoders: bool = False,stream_transformer: bool = True,+ background_pin: Optional[bool] = None,) -> bool:"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping thesmaller components resident. Returns False (caller falls back to whole-module) on any failure.@@ -1723,6 +2054,18 @@ def _apply_group_offload(logger,)gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]+ # ``background_pin``: a module the plan pins is applied unpinned and handed to a _GroupPinner, which the caller+ # starts with start_background_pins once the load has committed.+ if background_pin is None:+ background_pin = bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))+ defer = (+ background_pin+ and use_stream+ and "low_cpu_mem_usage" in _params+ and _background_pin_enabled()+ )+ if defer:+ install_group_pin_wait()# Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()# OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group# hooks exist.@@ -1732,7 +2075,11 @@ def _apply_group_offload(if isinstance(comp, torch.nn.Module):comp.to(onload)for module in streamed.values():- apply_group_offloading(module, **gkwargs)+ if defer and not gkwargs.get("low_cpu_mem_usage", False):+ apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})+ _defer_pinning(pipe, module, onload, logger)+ else:+ apply_group_offloading(module, **gkwargs)installed += 1# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no@@ -1752,7 +2099,11 @@ def _apply_group_offload(transformer_demoted = Falsefor name, module in streamed_encoders.items():try:- apply_group_offloading(module, **ekwargs)+ if defer and not ekwargs.get("low_cpu_mem_usage", False):+ apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})+ _defer_pinning(pipe, module, onload, logger)+ else:+ apply_group_offloading(module, **ekwargs)installed += 1_pin_vision_embedding_device(module)except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load@@ -1786,6 +2137,10 @@ def _apply_group_offload(name,exc,)+ # a leaf-level apply can raise after hooking part of the encoder; resident means no hooks at all, or+ # the applied VRAM floor reads the whole encoder as streamed while its unhooked layers stay on the card+ _remove_group_offload_hooks(module)+ _drop_deferred_pinning(pipe, module)module.to(onload)return Trueexcept Exception as exc: # noqa: BLE001 - fall back to whole-module offloaddiff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 447354c3b3c..0bd4e6d9367 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -87,8 +87,14 @@ from .diffusion_device import (resolve_selected_cuda_ordinal,)from .diffusion_memory import (+ DEFAULT_BASE_OVERHEAD_MIB,+ DEFAULT_GROUP_BLOCKS,+ MEMORY_MODE_AUTO,MEMORY_MODE_FAST,+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,OFFLOAD_NONE,+ OFFLOAD_STREAMING,apply_memory_plan,estimate_gguf_resident_mib,estimate_safetensors_dense_mib,@@ -96,12 +102,18 @@ from .diffusion_memory import (file_size_mib,normalize_memory_mode,plan_diffusion_memory,+ plan_keeps_transformer_resident,raise_on_unified_memory_shortfall,reclaim_host_memory,reclaim_offload_host_memory,+ refine_memory_plan_for_components,+ finish_background_pins,release_pinned_host_memory,+ request_background_pins,safetensors_prefix_mib,settled_snapshot_device_memory,+ start_background_pins,+ stop_background_pins,)from .diffusion_torchao_patches import install_torchao_int_mm_patchfrom .media_decode_phase import decode_phase as _decode_phase@@ -340,7 +352,7 @@ def _memory_mode_resolved(memory_mode: Any, plan: Any, offload_policy: str) -> treturn (memory_mode,plan.requested_mode,- f"planned '{plan.offload_policy}' offload from the family size table",+ f"planned '{plan.offload_policy}' offload ({_video_plan_label(plan)}) from the family size table",)@@ -994,6 +1006,10 @@ class _VideoLoadState:denoiser_pinned: bool = False# MiniMax-H3: streamed denoiser, so the VRAM floor is the largest single phase, not a sum.denoiser_streamed: bool = False+ # MiB the offload tier holds on the device at once; None when nothing is offloaded.+ vram_floor_mib: Optional[int] = None+ # MiB still on the device during the VAE decode (the streamed DiT has offloaded by then).+ vram_decode_floor_mib: Optional[int] = None# MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.denoiser_host_copy: bool = Falseresolved: Optional[dict] = None@@ -1515,7 +1531,7 @@ def _video_seed_stays_resident(base_repo: Optional[str],reclaimable_mib: int = 0,) -> bool:- """True when an artifact-sized plan keeps the denoiser resident; torchao rejects offload."""+ """True when an artifact-sized plan keeps the denoiser resident (text encoders may stream)."""components = getattr(fam, "bf16_components_gb", None)if not components:return True@@ -1543,10 +1559,9 @@ def _video_seed_stays_resident()vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scalemib_per_gb = 1000.0**3 / (1024.0 * 1024.0)- model_dense_mib = int(- (denoiser_gb + components[1] * te_scale * dtype_scale + components[2] * vae_scale)- * mib_per_gb- )+ text_encoder_gb = components[1] * te_scale * dtype_scale+ companions_gb = text_encoder_gb + components[2] * vae_scale+ model_dense_mib = int((denoiser_gb + companions_gb) * mib_per_gb)device_memory = settled_snapshot_device_memory(target)if reclaimable_mib > 0 and device_memory.free_mib is not None:free = device_memory.free_mib + int(reclaimable_mib)@@ -1562,10 +1577,442 @@ def _video_seed_stays_resident(height = fam.resolution_presets[0][1],num_frames = fam.default_num_frames,),- companion_dense_mib = None,+ companion_dense_mib = int(companions_gb * mib_per_gb),+ text_encoder_dense_mib = int(text_encoder_gb * mib_per_gb),requested_mode = normalize_memory_mode(memory_mode),)- return planned.offload_policy == "none"+ return plan_keeps_transformer_resident(planned)+++def _render_under_no_grad(state: Any) -> bool:+ """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode."""+ if not getattr(state, "transformer_quant", None):+ return False+ if getattr(state, "denoiser_streamed", False):+ return True+ family = getattr(state, "family", None)+ return not getattr(family, "modular_workflow", None) and (+ getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE+ )+++# Staging headroom: base overhead plus one linear's bf16, rotated and int8 copies, as a multiple of the largest.+_STAGE_LINEAR_WORKSPACE = 3+++def _stage_units(module: Any) -> list:+ import torch++ units: list = []+ for child in module.children():+ if isinstance(child, torch.nn.ModuleList):+ units.extend(child)+ else:+ units.append(child)+ return units+++def _module_dense_bytes(module: Any) -> int:+ from itertools import chain+ return sum(+ int(t.numel()) * int(t.element_size()) for t in chain(module.parameters(), module.buffers())+ )+++def _stage_denoiser_for_quant(+ transformer: Any,+ target: Any,+ *,+ logger: Any = None,+) -> list:+ """Stage a CPU DiT onto ``target``'s CUDA device block by block as far as it fits; quantisers work per linear,+ so a partial fit gives the same result. Returns the moved modules; never raises."""+ try:+ import torch++ if (+ transformer is None+ or getattr(target, "device", None) != "cuda"+ or not torch.cuda.is_available()+ ):+ return []+ first = next(transformer.parameters(), None)+ if first is None or first.device.type != "cpu":+ return []+ device = torch.device(getattr(target, "torch_device", None) or "cuda")+ from utils.hardware import trusted_mem_get_info++ free_bytes, _ = trusted_mem_get_info(device, module = torch.cuda)+ largest_linear = max(+ (+ _module_dense_bytes(m)+ for m in transformer.modules()+ if isinstance(m, torch.nn.Linear)+ ),+ default = 0,+ )+ budget = (+ int(free_bytes)+ - DEFAULT_BASE_OVERHEAD_MIB * 1024 * 1024+ - _STAGE_LINEAR_WORKSPACE * largest_linear+ )+ staged: list = []+ moved_bytes = 0+ try:+ for unit in _stage_units(transformer):+ size = _module_dense_bytes(unit)+ if size == 0 or moved_bytes + size > budget:+ continue+ unit.to(device)+ staged.append(unit)+ moved_bytes += size+ except Exception:+ # The DiT started on the host: put every moved unit, and a half-moved one, back before the CPU fallback.+ transformer.to("cpu")+ clear_gpu_cache()+ raise+ if staged and logger is not None:+ logger.info(+ "video.transformer_quant: quantising on %s with %.1f of %.1f GB of the DiT staged there",+ device,+ moved_bytes / 1e9,+ _module_dense_bytes(transformer) / 1e9,+ )+ return staged+ except Exception as exc: # noqa: BLE001 -- quantising on the host is slower, never wrong+ if logger is not None:+ logger.warning("video.transformer_quant: staging on the card failed (%s)", exc)+ return []+++def _unstage_modules(modules: list) -> None:+ for module in modules:+ module.to("cpu")+ clear_gpu_cache()+++def _auto_offload_scheme(requested: Optional[str]) -> Optional[str]:+ """auto takes int8 for an offloaded DiT: its torchao-free build rides every offload hook."""+ scheme = normalize_transformer_quant(requested)+ return TQ_INT8 if scheme == TQ_AUTO else scheme+++# Denoise-side base of estimate_video_runtime_mib; the rest scales with the decoded clip, held only by the VAE.+_VIDEO_DENOISE_ACTIVATION_MIB = 4096+++def _video_prefer_whole_module(+ plan: Any,+ *,+ denoiser_mib: Optional[int],+ text_encoder_mib: Optional[int],+ vae_mib: Optional[int],+ runtime_mib: int,+) -> Any:+ """Swap a streamed-DiT plan for whole-module offload when every component fits whole, one at a time.++ Wan2.2-5B at 16 GiB: 0.14-0.28 s per forward whole-module vs 0.28-0.33 s int8 streamed, so streaming only wins+ very short calls. ``denoiser_mib`` is one expert of a dual-DiT family. An explicit memory mode is left as asked."""+ if (+ getattr(plan, "offload_policy", None) != OFFLOAD_GROUP+ or not bool(getattr(plan, "stream_transformer", True))+ or getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO+ or denoiser_mib is None+ or text_encoder_mib is None+ or vae_mib is None+ ):+ return plan+ budget = (getattr(plan, "estimates", None) or {}).get("safe_device_budget_mib")+ if budget is None:+ return plan+ phases = (+ int(text_encoder_mib) + DEFAULT_BASE_OVERHEAD_MIB,+ int(denoiser_mib)+ + min(int(runtime_mib), _VIDEO_DENOISE_ACTIVATION_MIB)+ + DEFAULT_BASE_OVERHEAD_MIB,+ # the decode holds only the decoded-clip share; the denoise base is already in the DiT phase+ int(vae_mib)+ + max(0, int(runtime_mib) - _VIDEO_DENOISE_ACTIVATION_MIB)+ + DEFAULT_BASE_OVERHEAD_MIB,+ )+ if max(phases) > int(budget):+ return plan+ return dataclasses.replace(+ plan,+ estimates = {**(plan.estimates or {}), "whole_module_fits_mib": max(phases)},+ offload_policy = OFFLOAD_MODEL,+ stream_text_encoders = False,+ stream_transformer = True,+ vae_tiling = True,+ vae_slicing = True,+ reasons = tuple(plan.reasons)+ + (+ f"every component fits the card whole (largest phase {max(phases)} MiB of {int(budget)}): "+ "whole-module offload pages the DiT once per call instead of streaming it every step",+ ),+ )+++def _video_plan_rank(plan: Any) -> int:+ """Video speed rank, fastest first (0..6); see the tier comments in the tests."""+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)+ if policy == OFFLOAD_NONE:+ return 0+ if policy == OFFLOAD_GROUP:+ if not bool(getattr(plan, "stream_transformer", True)):+ return 1+ return 4 if bool(getattr(plan, "stream_text_encoders", False)) else 3+ if policy == OFFLOAD_MODEL:+ fits = (getattr(plan, "estimates", None) or {}).get("whole_module_fits_mib") is not None+ return 2 if fits else 5+ return 6+++def _video_plan_label(plan: Any) -> str:+ return (+ "every component resident",+ "denoiser resident, text encoders streamed",+ "whole-module CPU offload, each component fits whole",+ "denoiser streamed block by block, companions resident",+ "denoiser and text encoders streamed",+ "whole-module CPU offload",+ "denoiser blocks and text-encoder layers streamed",+ )[_video_plan_rank(plan)]+++_VIDEO_DENOISER_ATTRS = ("transformer", "transformer_2", "unconditional_transformer")+++def _video_streamed_peak_bytes(module: Any, offload_type: str, *, prefetch: bool, size: Any) -> int:+ """Bytes a group-offload hook holds on the device at once for ``module`` (diffusers' own grouping)."""+ from itertools import chain++ import torch++ try:+ from diffusers.hooks._common import _GO_LC_SUPPORTED_PYTORCH_LAYERS as leaf_types+ except Exception: # noqa: BLE001 -- same layer set diffusers streams leaf by leaf+ nn = torch.nn+ leaf_types = (+ nn.Conv1d,+ nn.Conv2d,+ nn.Conv3d,+ nn.ConvTranspose1d,+ nn.ConvTranspose2d,+ nn.ConvTranspose3d,+ nn.Linear,+ nn.Embedding,+ )++ counted: set[int] = set()++ def _own(mod: Any, recurse: bool) -> int:+ # a tied weight (shared embedding) moves once, so count it for its first group only+ total = 0+ for t in chain(mod.parameters(recurse = recurse), mod.buffers(recurse = recurse)):+ if id(t) not in counted:+ counted.add(id(t))+ total += size(t)+ return total++ if offload_type == "block_level":+ windows: list[int] = []+ grouped: set[int] = set()+ for child in module.children():+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):+ blocks = list(child)+ for i in range(0, len(blocks), DEFAULT_GROUP_BLOCKS):+ windows.append(sum(_own(b, True) for b in blocks[i : i + DEFAULT_GROUP_BLOCKS]))+ grouped |= {id(t) for t in chain(child.parameters(), child.buffers())}+ # the unmatched group (embeddings, heads, loose params) stays onloaded for the whole forward+ unmatched = sum(+ size(t)+ for t in {id(t): t for t in chain(module.parameters(), module.buffers())}.values()+ if id(t) not in grouped and id(t) not in counted+ )+ units, fixed = windows, unmatched+ else:+ # (size, enclosing ancestors' loose bytes, ids of this group and its ancestors) per onload group+ groups: list[tuple[int, int, frozenset]] = []+ visited: set[int] = set()++ def _walk(mod: Any, enclosing: int, chain: frozenset) -> None:+ if id(mod) in visited:+ return+ visited.add(id(mod))+ if isinstance(mod, leaf_types):+ groups.append((_own(mod, True), enclosing, chain | {id(mod)}))+ elif next(mod.children(), None) is not None:+ # a parent's loose params onload in its pre-forward and stay until its post-forward, under every+ # descendant; a sibling branch has offloaded its own by then, so only the enclosing chain stacks+ loose = _own(mod, False)+ here = chain | {id(mod)}+ groups.append((loose, enclosing, here))+ for child in mod.children():+ _walk(child, enclosing + loose, here)+ else:+ # loose params of a childless non-leaf (a norm) form their own group+ groups.append((_own(mod, False), enclosing, chain | {id(mod)}))++ _walk(module, 0, frozenset())+ ranked = sorted(range(len(groups)), key = lambda i: groups[i][0], reverse = True)+ peak = 0+ for size, enclosing, here in groups:+ # a copy stream prefetches the next group while this one computes: any group not already held+ other = 0+ if prefetch:+ for i in ranked:+ if not (groups[i][2] <= here):+ other = groups[i][0]+ break+ peak = max(peak, enclosing + size + other)+ return peak+ units = sorted(units, reverse = True)+ # a copy stream prefetches the next group while the current one computes+ return fixed + sum(units[: 2 if prefetch else 1])+++def _video_group_hooked(module: Any) -> bool:+ """Whether group offloading actually hooked ``module`` (leaf level hooks its leaves). Unreadable counts as hooked."""+ try:+ for sub in module.modules():+ hooks = getattr(getattr(sub, "_diffusers_hook", None), "hooks", None) or {}+ if any("group_offloading" in str(key) for key in hooks):+ return True+ except Exception: # noqa: BLE001 -- trust the plan+ return True+ return False+++def _video_offload_vram_floor_mib(+ pipe: Any,+ plan: Any,+ *,+ applied: bool = False,+ phase: str = "peak",+) -> Optional[int]:+ """MiB of weights co-resident on the device under ``plan``'s tier, from the loaded modules; None if not offloaded.+ ``applied`` reads which modules the hooks actually stream: an encoder that refused leaf offload stays resident.+ ``phase="decode"`` counts only what stays on the device during the VAE decode, after the DiT has offloaded."""+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)+ if policy == OFFLOAD_NONE:+ return None+ try:+ from itertools import chain++ import torch++ from .diffusion_prequant import tensor_payload_bytes++ components = getattr(pipe, "components", None)+ if not isinstance(components, dict):+ return None+ sizes: dict[str, int] = {}+ modules: dict[str, Any] = {}+ seen: set[int] = set()+ for name, module in components.items():+ if not isinstance(module, torch.nn.Module):+ continue+ total = 0+ for tensor in chain(module.parameters(), module.buffers()):+ if id(tensor) in seen or getattr(tensor, "is_meta", False):+ continue+ seen.add(id(tensor))+ total += tensor_payload_bytes(tensor)+ sizes[str(name)] = total+ modules[str(name)] = module+ if not sizes:+ return None+ denoisers = {n for n in sizes if n in _VIDEO_DENOISER_ATTRS}+ encoders = {n for n in sizes if n.startswith("text_encoder")}+ if policy == OFFLOAD_MODEL:+ if phase == "decode":+ # whole-module offload onloads one component at a time; the decode runs the decoders alone+ decoders = [+ size for name, size in sizes.items() if name not in denoisers | encoders+ ]+ return int(max(decoders, default = 0) // (1024 * 1024))+ return int(max(sizes.values()) // (1024 * 1024))+ if policy == OFFLOAD_GROUP:+ streamed = (+ set() if not bool(getattr(plan, "stream_transformer", True)) else set(denoisers)+ )+ if bool(getattr(plan, "stream_text_encoders", False)):+ streamed |= encoders+ elif policy == OFFLOAD_STREAMING:+ streamed = denoisers | encoders+ else:+ return None+ if applied:+ # From the hooks alone: a refusing encoder stays resident, and the fallback may hook a DiT the plan kept.+ streamed = {name for name in denoisers | encoders if _video_group_hooked(modules[name])}+ held = sum(size for name, size in sizes.items() if name not in streamed)+ if phase == "decode":+ # streamed groups have all offloaded by the decode; only the residents remain+ return int(held // (1024 * 1024))+ backend = getattr(getattr(plan, "device_memory", None), "backend", None)+ prefetch = backend not in ("mps", "cpu")++ def _size(tensor: Any) -> int:+ return 0 if getattr(tensor, "is_meta", False) else tensor_payload_bytes(tensor)++ # streamed modules run one after another, so only the largest onloaded unit joins the residents+ held += max(+ (+ _video_streamed_peak_bytes(+ modules[name],+ "block_level" if name in denoisers else "leaf_level",+ prefetch = prefetch,+ size = _size,+ )+ for name in streamed+ ),+ default = 0,+ )+ return int(held // (1024 * 1024))+ except Exception: # noqa: BLE001 -- an unreadable pipeline keeps today's unchecked behaviour+ return None+++def video_offload_shortfall_message(+ *,+ family: str,+ floor_mib: int,+ available_mib: int,+ placement: str,+ width: Optional[int] = None,+ height: Optional[int] = None,+ frames: Optional[int] = None,+ decode_floor_mib: Optional[int] = None,+) -> Optional[str]:+ """Refusal when co-resident weights plus the activation headroom exceed the device, else None.+ Denoise and VAE decode are separate phases: the decoded-clip share is held only beside the weights still on the+ device during the decode (``decode_floor_mib``, the full floor when unknown), never beside the streamed DiT."""+ headroom = DEFAULT_BASE_OVERHEAD_MIB+ required = int(floor_mib) + headroom+ if width and height and frames:+ # decoded-clip share of the runtime estimate; its fixed 4 GiB denoise base over-refuses measured cells+ clip_mib = max(+ 0,+ estimate_video_runtime_mib(width = width, height = height, num_frames = frames)+ - _VIDEO_DENOISE_ACTIVATION_MIB,+ )+ decode_floor = int(+ floor_mib if decode_floor_mib is None else min(decode_floor_mib, floor_mib)+ )+ if decode_floor + headroom + clip_mib > required:+ floor_mib, headroom = decode_floor, headroom + clip_mib+ required = decode_floor + headroom+ if int(available_mib) >= required:+ return None+ shape = f" for {width}x{height} at {frames} frames" if width and height and frames else ""+ return (+ f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "+ f"placement ({placement}: {floor_mib / 1024:.1f} GiB of weights on the GPU at once plus "+ f"{headroom / 1024:.1f} GiB for activations); {available_mib / 1024:.1f} GiB is "+ "available. Free VRAM held by other processes, set the text encoder or DiT precision lower, "+ "or pick a smaller model."+ )def _progress(phase: Optional[str], **extra: Any) -> dict[str, Any]:@@ -4884,7 +5331,6 @@ class VideoBackend:vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scalescaled_te_gb = text_encoder_gb * dtype_scalescaled_vae_gb = vae_gb * vae_scale- companions_gb = text_encoder_gb + vae_gb if components is not None else 0.0scaled_companions_gb = scaled_te_gb + scaled_vae_gb if components is not None else 0.0if log and scale != 1.0 and components is not None:logger.info(@@ -4895,6 +5341,11 @@ class VideoBackend:text_encoder_gb,components[1],)+ # Without the text-encoder split the planner can only pick all-resident or whole-module offload.+ companion_mib = (+ int(scaled_companions_gb * mib_per_gb) if components is not None else None+ )+ text_encoder_mib = int(scaled_te_gb * mib_per_gb) if components is not None else Noneif kind == "pipeline":transformer_gb = (components[0] * dtype_scale if denoiser_gb is None else float(denoiser_gb)@@ -4904,62 +5355,106 @@ class VideoBackend:if components is not Noneelse None)- companion_mib = Noneelse:- companion_mib = (- int(scaled_companions_gb * mib_per_gb) if components is not None else None- )# Budget ALL weights: companions stay resident, so budgeting the transformer alone lets auto pick# OFFLOAD_NONE and OOM.model_dense_mib = (transformer_mib + (companion_mib or 0) if transformer_mib is not None else None)- planned = plan_diffusion_memory(- target = target,- device_memory = device_memory,- model_dense_mib = model_dense_mib,- runtime_headroom_mib = runtime_mib,- companion_dense_mib = companion_mib,- requested_mode = normalize_memory_mode(memory_mode),+ experts = 2 if getattr(fam, "is_moe", False) else 1+ vae_mib = int(scaled_vae_gb * mib_per_gb) if components is not None else None+ dit_mib = (+ (int(transformer_gb * mib_per_gb) if components is not None else None)+ if kind == "pipeline"+ else transformer_mib+ )+ planned = _video_prefer_whole_module(+ plan_diffusion_memory(+ target = target,+ device_memory = device_memory,+ model_dense_mib = model_dense_mib,+ runtime_headroom_mib = runtime_mib,+ companion_dense_mib = companion_mib,+ text_encoder_dense_mib = text_encoder_mib,+ requested_mode = normalize_memory_mode(memory_mode),+ ),+ denoiser_mib = None if dit_mib is None else dit_mib // experts,+ text_encoder_mib = text_encoder_mib,+ vae_mib = vae_mib,+ runtime_mib = runtime_mib,)- # Parity with the image dense-quant path: the bf16-table plan can force offload a quantised DiT would not- # need, so re-plan with the scheme factor and keep resident if it fits.+ # Re-plan with the scheme factor: the bf16-table plan can offload a DiT the quantised one would not.dense_plan = plannedreplanned_for_quant = False+ requested_scheme = normalize_transformer_quant(transformer_quant)+ # auto keeps a resident bf16 DiT: quantising costs accuracy for little speed.+ worth_replanning = (+ not plan_keeps_transformer_resident(planned)+ if requested_scheme == TQ_AUTO+ else planned.offload_policy != OFFLOAD_NONE+ )# Not for a unified-memory shortfall: runtime quant still peaks at the dense bf16 DiT.if (kind == "pipeline"and denoiser_gb is None- and planned.offload_policy != "none"- and normalize_transformer_quant(transformer_quant) is not None+ and worth_replanning+ and requested_scheme is not Noneand (dense_transformer_supported(target)or native_quant_scheme(target, transformer_quant, family = fam.name) is not None+ or native_quant_scheme(+ target,+ _auto_offload_scheme(transformer_quant),+ family = fam.name,+ offload = True,+ )+ is not None)and components is not None):- scheme_preview = native_quant_scheme(- target, transformer_quant, family = fam.name- ) or select_transformer_quant_scheme(target, transformer_quant, family = fam.name)+ scheme_preview = (+ native_quant_scheme(target, transformer_quant, family = fam.name)+ or (+ select_transformer_quant_scheme(target, transformer_quant, family = fam.name)+ if dense_transformer_supported(target)+ else None+ )+ or native_quant_scheme(+ target,+ _auto_offload_scheme(transformer_quant),+ family = fam.name,+ offload = True,+ )+ )factor = _QUANT_STEADY_FACTOR.get(scheme_preview) if scheme_preview else Noneif factor is not None:- quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)- replanned = plan_diffusion_memory(- target = target,- device_memory = device_memory,- model_dense_mib = quant_mib,- runtime_headroom_mib = runtime_mib,- companion_dense_mib = None,- requested_mode = normalize_memory_mode(memory_mode),+ # companions priced exactly as the split below (dtype-scaled), or the planner's DiT share+ # (quant total minus the scaled text encoder) drifts under an fp16 to fp32 promotion+ quant_mib = int((components[0] * factor + scaled_companions_gb) * mib_per_gb)+ replanned = _video_prefer_whole_module(+ plan_diffusion_memory(+ target = target,+ device_memory = device_memory,+ model_dense_mib = quant_mib,+ runtime_headroom_mib = runtime_mib,+ companion_dense_mib = companion_mib,+ text_encoder_dense_mib = text_encoder_mib,+ requested_mode = normalize_memory_mode(memory_mode),+ ),+ denoiser_mib = int(components[0] * factor * mib_per_gb) // experts,+ text_encoder_mib = text_encoder_mib,+ vae_mib = vae_mib,+ runtime_mib = runtime_mib,)- if replanned.offload_policy == "none":+ if _video_plan_rank(replanned) < _video_plan_rank(planned):if log:logger.info(- "video.transformer_quant: %s fits resident (%d MiB steady); "- "dropping the bf16 plan's '%s' offload",+ "video.transformer_quant: %s (%d MiB steady) places the DiT as '%s' "+ "instead of the bf16 plan's '%s'",scheme_preview,quant_mib,- planned.offload_policy,+ _video_plan_label(replanned),+ _video_plan_label(planned),)planned = replannedreplanned_for_quant = True@@ -4991,8 +5486,13 @@ class VideoBackend:plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = True, denoiser_gb = denoiser_seed_gb)- if denoiser_seed_scheme is not None and plan.offload_policy != "none":- # Offload hooks use Module.to(), which torchao tensors reject: re-plan at bf16.+ if denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):+ logger.info(+ "video.denoiser_prequant: the %s denoiser would have to be offloaded ('%s') on this "+ "card, so the hosted checkpoint is not seeded",+ denoiser_seed_scheme,+ _video_plan_label(plan),+ )denoiser_seed_scheme = Nonedenoiser_seed_gb = Noneplan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)@@ -5052,7 +5552,7 @@ class VideoBackend:plan, bf16_plan, quant_replanned = _plan_for_te_scale(settled_te_scale, log = False, denoiser_gb = denoiser_seed_gb)- if denoiser_seed_scheme is not None and plan.offload_policy != "none":+ if denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):denoiser_seed_scheme = Nonedenoiser_seed_gb = Noneplan, bf16_plan, quant_replanned = _plan_for_te_scale(settled_te_scale, log = False)@@ -5142,7 +5642,7 @@ class VideoBackend:ltx23_override = Noneltx23_scheme = normalize_transformer_quant(transformer_quant)seeded = None- if ltx23_prequant_pick and plan.offload_policy == "none":+ if ltx23_prequant_pick and plan_keeps_transformer_resident(plan):from .video_ltx2 import load_ltx23_prequant_transformerseeded = load_ltx23_prequant_transformer(fam,@@ -5164,9 +5664,10 @@ class VideoBackend:elif ltx23_prequant_pick:# Decline before the dense assembly loads the 44 GB bf16 DiT only to refuse it.ltx23_reason = (- f"this GPU's '{plan.offload_policy}' memory plan offloads the DiT even at fp8 size, and "- "torchao quantised tensors cannot be moved by the offload hooks"- if plan.offload_policy != "none"+ f"this GPU's memory plan ({_video_plan_label(plan)}) offloads the DiT even at fp8 "+ "size, and the hosted fp8 DiT is kept resident only (the text encoders may stream "+ "around it)"+ if not plan_keeps_transformer_resident(plan)else "the hosted fp8 DiT (unsloth/LTX-2.3-FP8) could not be loaded"+ (" from the local cache" if local_files_only else ""))@@ -5245,10 +5746,16 @@ class VideoBackend:transformer_quant_decline: Optional[str] = Nonetransformer_quant_decline_status = RESOLVED_FELL_BACKtransformer_quant_source: Optional[str] = None- video_offload = plan.offload_policy != "none"+ # Text encoders streaming around a resident DiT do not move its weights, so torchao stays valid.+ video_offload = not plan_keeps_transformer_resident(plan)native_scheme = (native_quant_scheme(- target, transformer_quant_pinned, family = fam.name, offload = video_offload+ target,+ _auto_offload_scheme(transformer_quant)+ if video_offload and transformer_quant_pinned is None+ else transformer_quant_pinned,+ family = fam.name,+ offload = video_offload,)# A seeded denoiser is a torchao checkpoint; the torchao-free path must not claim it.if kind == "pipeline" and not denoiser_injected@@ -5268,7 +5775,7 @@ class VideoBackend:and normalize_transformer_quant(transformer_quant) == TQ_AUTOand not quant_replannedand not denoiser_injected- and plan.offload_policy == "none"+ and plan_keeps_transformer_resident(plan)# An unmeasured budget also plans "none" without proving a fit.and plan.estimates.get("safe_device_budget_mib") is not Noneand plan.estimates.get("resident_required_mib") is not None@@ -5311,22 +5818,21 @@ class VideoBackend:kind == "pipeline"and normalize_transformer_quant(transformer_quant) is not Noneand dense_transformer_supported(target)- and plan.offload_policy != "none"+ and video_offloadand native_scheme is None):- # Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant- # (dense-under-offload beats a crash); forceable via a resident mode.+ # torchao subclasses do not survive offload hooks under inference_mode; int8 has a torchao-free build, others skip.logger.info(- "video.transformer_quant: skipped (offload policy '%s' moves the "- "DiT via Module.to(), unsupported for torchao quantized tensors); "- "pin a resident memory mode to combine quant with this model",- plan.offload_policy,+ "video.transformer_quant: skipped (the memory plan offloads the DiT: %s; only "+ "int8 runs torchao-free there)",+ _video_plan_label(plan),)quant_skipped_for_offload = Truetransformer_quant_decline = (- f"the memory plan picked '{plan.offload_policy}' offload, which moves the DiT via "- "Module.to(); torchao quantized tensors reject that. Pin a resident memory mode "- "(Fast) to combine quant with this model"+ f"this GPU's memory plan moves the DiT ({_video_plan_label(plan)}), and "+ f"'{normalize_transformer_quant(transformer_quant)}' is built as torchao weights here, "+ "which Studio keeps on a resident DiT only; int8 runs torchao-free under offload. Pick "+ "int8, or a resident memory mode (Fast) on a card that holds the quantised DiT")elif (kind == "pipeline"@@ -5351,16 +5857,24 @@ class VideoBackend:):engaged = []for view in views:- # Pass each expert view so both DiTs quantise with the same scheme. The family name drives- # _FAMILY_SCHEME_DENY.- scheme = quantize_transformer(- view,- target,- mode = transformer_quant,- family = fam.name,- logger = logger,- **native_kwargs,+ # Quantise on the card: on the host int8 packing took 59 s (Wan2.2-5B) to 154 s (LTX-2) of the load.+ staged = _stage_denoiser_for_quant(+ getattr(view, "transformer", None), target, logger = logger)+ scheme = None+ try:+ scheme = quantize_transformer(+ view,+ target,+ mode = native_scheme or transformer_quant,+ family = fam.name,+ logger = logger,+ **native_kwargs,+ )+ finally:+ # A declined quant can roll back to the bf16 offload plan, which must not start with dense blocks staged.+ if staged and (video_offload or scheme is None):+ _unstage_modules(staged)if scheme is not None:engaged.append(scheme)# All experts or none: the first is mutated in place, so a second-expert failure cannot fall back to dense.@@ -5615,6 +6129,26 @@ class VideoBackend:del pipeclear_gpu_cache()raise RuntimeError("Video load was cancelled or superseded.")+ # A component larger than the card OOMs at its first whole-module onload; fall back to leaf streaming.+ refined = refine_memory_plan_for_components(pipe, plan)+ if refined is not plan:+ logger.info("video.memory: %s", refined.reasons[-1])+ plan = refined+ vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)+ free_mib = getattr(getattr(plan, "device_memory", None), "free_mib", None)+ if vram_floor_mib is not None and free_mib is not None:+ shortfall = video_offload_shortfall_message(+ family = fam.name,+ floor_mib = vram_floor_mib,+ available_mib = int(free_mib),+ placement = _video_plan_label(plan),+ )+ if shortfall is not None:+ del pipe+ clear_gpu_cache()+ raise RuntimeError(shortfall)+ # the streamed modules pin after the load returns, overlapping the first encode and compile+ request_background_pins(pipe)offload_policy, vae_tiling = apply_memory_plan(pipe,plan,@@ -5624,6 +6158,28 @@ class VideoBackend:placement_device = target.torch_device,logger = logger,)+ if (+ offload_policy == plan.offload_policy+ and vram_floor_mib is not None+ and free_mib is not None+ ):+ # An encoder that refused leaf offload is kept resident under the same policy: re-check what landed.+ applied_floor = _video_offload_vram_floor_mib(pipe, plan, applied = True)+ raised = applied_floor is not None and applied_floor > vram_floor_mib+ if applied_floor is not None:+ # record what landed even when lower (the fallback may stream a DiT the plan kept resident)+ vram_floor_mib = applied_floor+ if raised:+ shortfall = video_offload_shortfall_message(+ family = fam.name,+ floor_mib = applied_floor,+ available_mib = int(free_mib),+ placement = _video_plan_label(plan),+ )+ if shortfall is not None:+ del pipe+ clear_gpu_cache()+ raise RuntimeError(shortfall)# A dual-DiT MoE needs no extra per-expert pass: apply_memory_plan covers every DiT; a second pass would# duplicate-hook.if not vae_tiling:@@ -5746,6 +6302,21 @@ class VideoBackend:cache_threshold = transformer_cache_threshold,transformer_quant = transformer_quant_engaged,text_encoder_quant = text_encoder_quant_engaged,+ vram_floor_mib = (+ vram_floor_mib+ if offload_policy == plan.offload_policy+ else _video_offload_vram_floor_mib(+ pipe, dataclasses.replace(plan, offload_policy = offload_policy)+ )+ ),+ vram_decode_floor_mib = _video_offload_vram_floor_mib(+ pipe,+ plan+ if offload_policy == plan.offload_policy+ else dataclasses.replace(plan, offload_policy = offload_policy),+ applied = True,+ phase = "decode",+ ),resolved = resolved,)self._precommit_globals = None@@ -5758,6 +6329,7 @@ class VideoBackend:effective_speed,transformer_quant_engaged or "off",)+ start_background_pins(pipe)return self.status()@staticmethod@@ -7135,6 +7707,15 @@ class VideoBackend:# is pinned the un-indexed state.device below -- the H3 memory probe and every torch.Generator --# resolves to its own default card while the pipeline sits on the selected one.self._state_device_target(state)+ # A render racing the post-load pinner ran every later render slower; the wait is usually over.+ waited = finish_background_pins(getattr(state, "pipe", None), cancel)+ if cancel.is_set():+ raise RuntimeError(VIDEO_CANCELLED_MSG)+ if waited >= 1.0:+ logger.info(+ "video.generate: waited %.1f s for the host weights to finish pinning",+ waited,+ )fam = state.familyif _resolved_inputs is None:first_pil, last_pil, width, height, conditioning = self._resolve_keyframes(@@ -7281,6 +7862,41 @@ class VideoBackend:f"system RAM at this VRAM tier; {host_capacity_gb:.1f} GB is ""available. Load the GGUF artifact instead.")+ elif (+ state.engine == "diffusers"+ and not fam.modular_workflow+ and getattr(state, "vram_floor_mib", None) is not None+ and str(state.device).startswith("cuda")+ and hasattr(torch, "cuda")+ and hasattr(torch.cuda, "mem_get_info")+ ):+ # Otherwise a card that cannot hold the tier's weights is admitted and dies at the first onload.+ from utils.hardware import trusted_mem_get_info++ ordinal = getattr(state, "placed_ordinal", None)+ device_obj = (+ torch.device("cuda", int(ordinal))+ if ordinal is not None+ else torch.device(state.device)+ )+ free_bytes, _ = trusted_mem_get_info(device_obj, module = torch.cuda)+ reserved_bytes = (+ torch.cuda.memory_reserved(device_obj)+ if hasattr(torch.cuda, "memory_reserved")+ else 0+ )+ shortfall = video_offload_shortfall_message(+ family = fam.name,+ floor_mib = int(state.vram_floor_mib),+ available_mib = int((free_bytes + reserved_bytes) // (1024 * 1024)),+ placement = f"'{state.offload_policy}' offload",+ width = width,+ height = height,+ frames = frames,+ decode_floor_mib = getattr(state, "vram_decode_floor_mib", None),+ )+ if shortfall is not None:+ raise RuntimeError(shortfall)# MPS seeds from the CPU generator too: diffusers reproduces a Metal seed only that way, and the# pipelines move the noise to the device themselves.@@ -7559,11 +8175,13 @@ class VideoBackend:)elif state.transformer_cache:self._reset_step_cache(pipe)- # torchao's aliasing check fails a streamed weight move under inference_mode.+ # torchao weights fail a hooked weight move under inference_mode (see _render_under_no_grad).grad_ctx = (torch.no_grad()if state.transformer_quant and getattr(state, "denoiser_streamed", False)else torch.inference_mode()+ if not _render_under_no_grad(state)+ else torch.no_grad())try:with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:@@ -8163,6 +8781,8 @@ class VideoBackend:from . import diffusion_prompt_cachediffusion_prompt_cache.release(getattr(state, "pipe", None))+ # A pinner still copying would hold its chunks past release_pinned_host_memory().+ stop_background_pins(getattr(state, "pipe", None))# Before clear_gpu_cache(), or the graph pool stays reserved.diffusion_cuda_graph.uninstall_all(getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.pyindex 269af3d55c6..03d8eb01808 100644--- a/studio/backend/tests/test_diffusion_memory.py+++ b/studio/backend/tests/test_diffusion_memory.py@@ -2340,6 +2340,127 @@ def test_the_resident_transformer_tier_pins_its_encoders_when_ram_allows(monkeypassert seen["text_encoder_2"]["low_cpu_mem_usage"] is False+def _background_pin_calls(monkeypatch, sizes, budget, **call_kw):+ """(low_cpu_mem_usage per module, modules handed to a deferred pinner) for one apply."""+ import core.inference.diffusion_memory as mem++ usage: dict = {}+ deferred: list = []+ pipe, *_ = _stream_te_pipe(monkeypatch)++ def _apply(+ module,+ onload_device = None,+ offload_device = None,+ offload_type = None,+ num_blocks_per_group = None,+ non_blocking = False,+ use_stream = False,+ record_stream = False,+ low_cpu_mem_usage = False,+ **_,+ ):+ usage[module.name] = low_cpu_mem_usage++ _swap_group_offloading(monkeypatch, _apply)+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)+ monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))+ monkeypatch.setattr(+ mem,+ "_defer_pinning",+ lambda pipe, module, device, logger: deferred.append(module.name) or True,+ )+ assert mem._apply_group_offload(pipe, "cuda", logger = None, **call_kw) is True+ return usage, deferred+++_BG_SIZES = {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500}+++def test_a_video_load_defers_every_planned_pin_off_the_load_path(monkeypatch):+ # Pinning at load read every streamed byte from disk first: 125-139 s of LTX-2.3's DiT on Colab. The apply goes+ # unpinned and a pinner takes the same modules, so what gets pinned is unchanged, only when.+ usage, deferred = _background_pin_calls(+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True+ )+ assert all(v is True for v in usage.values()), usage+ assert deferred == ["transformer", "text_encoder", "text_encoder_2"]+++def test_a_module_the_ram_gate_left_unpinned_gets_no_pinner(monkeypatch):+ usage, deferred = _background_pin_calls(+ monkeypatch, _BG_SIZES, 10_000, stream_text_encoders = True, background_pin = True+ )+ assert usage == {"transformer": True, "text_encoder": True, "text_encoder_2": True}, usage+ assert deferred == ["transformer"]+++def test_a_resident_dit_tier_defers_only_its_encoders(monkeypatch):+ usage, deferred = _background_pin_calls(+ monkeypatch,+ _BG_SIZES,+ 10_000,+ stream_text_encoders = True,+ stream_transformer = False,+ background_pin = True,+ )+ assert "transformer" not in usage+ assert deferred == ["text_encoder", "text_encoder_2"]+++def test_without_background_pin_the_apply_still_pins_eagerly(monkeypatch):+ usage, deferred = _background_pin_calls(+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True+ )+ assert all(v is False for v in usage.values()), usage+ assert deferred == []+++def test_the_background_pin_env_restores_eager_pinning(monkeypatch):+ import core.inference.diffusion_memory as mem++ monkeypatch.setenv(mem.BACKGROUND_PIN_ENV, "0")+ usage, deferred = _background_pin_calls(+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True+ )+ assert all(v is False for v in usage.values()), usage+ assert deferred == []+++def test_a_pipe_request_turns_background_pinning_on(monkeypatch):+ # video asks through the pipe, so apply_memory_plan (and every stub of it) keeps its signature+ import core.inference.diffusion_memory as mem++ usage, deferred = _background_pin_calls(+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True+ )+ assert deferred == []+ import sys++ pipe, *_ = _stream_te_pipe(monkeypatch)+ seen: dict = {}++ def _apply(+ module,+ low_cpu_mem_usage = False,+ use_stream = False,+ **_,+ ):+ seen[module.name] = low_cpu_mem_usage++ _swap_group_offloading(monkeypatch, _apply)+ monkeypatch.setattr(+ mem,+ "_defer_pinning",+ lambda pipe, module, device, logger: deferred.append(module.name) or True,+ )+ mem.request_background_pins(pipe)+ assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True+ assert "transformer" in deferred, deferred++@pytest.mark.parametrize("stream_transformer", [True, False])def test_windows_and_wsl_never_pin_the_streamed_tiers(monkeypatch, stream_transformer):import core.inference.diffusion_memory as mem@@ -2390,6 +2511,30 @@ def test_a_first_encoder_refusal_is_unhooked_before_whole_module_offload(monkeypassert unhooked == ["text_encoder"]+def test_an_encoder_kept_resident_after_a_partial_leaf_apply_is_unhooked(monkeypatch):+ """A refusal that raised after hooking part of the encoder must not leave those hooks on the resident encoder."""+ import core.inference.diffusion_memory as mem++ def _apply(module, **kw):+ if getattr(module, "name", "") == "text_encoder":+ raise RuntimeError("leaf offload failed part way")++ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)+ _swap_group_offloading(monkeypatch, _apply)+ unhooked: list[str] = []+ monkeypatch.setattr(+ mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False+ )++ assert (+ mem._apply_group_offload(+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = True+ )+ is True+ )+ assert unhooked == ["text_encoder"]++def test_pinned_memory_cap_detection(monkeypatch, tmp_path):import builtinsdiff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.pyindex af236c07451..7bef7fecddd 100644--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py@@ -400,3 +400,75 @@ def test_the_ram_gate_is_sized_from_the_container(monkeypatch.setattr(torch, "empty", empty)assert dm._pin_host_weights(module, {"weight": data}) == 0assert bool(allocated) is pinned+++class _BufferLinear(torch.nn.Module):+ """A weight held as plain buffers, as the torchao-free int8 linear stores it."""++ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.register_buffer("weight_q", torch.randint(-127, 127, (8, 8), dtype = torch.int8))+ self.register_buffer("weight_scale", torch.rand(8))+ self.bias = torch.nn.Parameter(torch.zeros(8))++ def forward(self, x):+ return x @ (self.weight_q.float() * self.weight_scale[:, None]).t() + self.bias+++def _buffer_pipe(device):+ pipe = _pipe(device, "transformer")+ pipe.components["transformer"] = pipe.transformer = _BufferLinear()+ return pipe+++def _buffer_ptrs(module):+ return [b.data_ptr() for b in module.buffers()]+++@cuda[email protected]("pin", ["0", "1"])+def test_buffer_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)+ x = torch.randn(4, 8)+ stock = _buffer_pipe("cuda")+ stock.enable_model_cpu_offload()+ ref = _render(stock, x)+ pipe = _buffer_pipe("cuda")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ assert torch.equal(_render(pipe, x), ref)+ host = _buffer_ptrs(pipe.transformer)+ for _ in range(3):+ assert torch.equal(_render(pipe, x), ref)+ assert _buffer_ptrs(pipe.transformer) == host+ assert all(b.is_pinned() == (pin == "1") for b in pipe.transformer.buffers())+++@cuda+def test_a_buffer_written_on_the_device_is_copied_back():+ pipe = _buffer_pipe("cuda")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ before = pipe.transformer.weight_scale.clone()+ pipe.transformer(torch.ones(1, 8))+ pipe.transformer.weight_scale.add_(1)+ pipe.transformer._hf_hook.init_hook(pipe.transformer)+ assert pipe.transformer.weight_scale.device.type == "cpu"+ assert torch.equal(pipe.transformer.weight_scale, before + 1)+++@cuda+def test_buffers_onloaded_under_inference_mode_do_not_break_the_hook():+ """A buffer moved under inference_mode is an inference tensor with no version counter; it takes the stock copy."""+ x = torch.randn(4, 8)+ stock = _buffer_pipe("cuda")+ stock.enable_model_cpu_offload()+ with torch.inference_mode():+ ref = _render(stock, x)+ pipe = _buffer_pipe("cuda")+ pipe.enable_model_cpu_offload()+ dm.keep_cpu_weights_on_offload(pipe)+ for _ in range(3):+ with torch.inference_mode():+ assert torch.equal(_render(pipe, x), ref)diff --git a/studio/backend/tests/test_group_offload_background_pin.py b/studio/backend/tests/test_group_offload_background_pin.pynew file mode 100644index 00000000000..d14a3ae49ac--- /dev/null+++ b/studio/backend/tests/test_group_offload_background_pin.py@@ -0,0 +1,124 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Background pinning of diffusers stream group offload, on a real card."""++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("diffusers")++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")+++class _Pipe:+ pass+++def _net():+ torch.manual_seed(0)++ class Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList(+ torch.nn.Sequential(+ torch.nn.Linear(256, 384), torch.nn.GELU(), torch.nn.Linear(384, 256)+ )+ for _ in range(6)+ )+ self.register_buffer("gain", torch.full((256,), 0.5))++ def forward(self, x):+ for block in self.blocks:+ x = x + block(x)+ return x * self.gain++ return Net().to(torch.bfloat16)+++def _offload(net, pinned: bool):+ from diffusers.hooks import apply_group_offloading+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = True,+ low_cpu_mem_usage = not pinned,+ )+ return net+++def _reference(x):+ with torch.no_grad():+ return _offload(_net(), pinned = True)(x)+++def _groups_pinned(net):+ import core.inference.diffusion_memory as mem+ groups = mem._offload_groups(net)+ return groups, all(t.is_pinned() for g in groups for t in g.cpu_param_dict.values())+++def test_a_deferred_pin_renders_identically_and_ends_with_every_group_pinned():+ import core.inference.diffusion_memory as mem++ mem.install_group_pin_wait()+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")+ ref = _reference(x)+ pipe, net = _Pipe(), _offload(_net(), pinned = False)+ before = {id(p): p.detach().clone() for p in net.parameters()}+ assert mem._defer_pinning(pipe, net, torch.device("cuda"), None)+ groups, pinned = _groups_pinned(net)+ assert groups and not pinned+ assert mem.start_background_pins(pipe) == 1+ with torch.no_grad():+ out = net(x)+ again = net(x)+ torch.cuda.synchronize()+ mem.stop_background_pins(pipe)+ assert torch.equal(out, ref) and torch.equal(again, ref)+ groups, pinned = _groups_pinned(net)+ assert pinned+ # the host copies were replaced, not duplicated: every offloaded parameter now IS its pinned chunk view+ for g in groups:+ for t, host in g.cpu_param_dict.items():+ if isinstance(t, torch.nn.Parameter):+ assert t.device.type == "cpu" and t.data_ptr() == host.data_ptr()+ for p in net.parameters():+ assert torch.equal(p.detach(), before[id(p)])+++def test_an_onload_before_the_pinner_starts_waits_for_its_group():+ import core.inference.diffusion_memory as mem++ mem.install_group_pin_wait()+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")+ ref = _reference(x)+ pipe, net = _Pipe(), _offload(_net(), pinned = False)+ mem._defer_pinning(pipe, net, torch.device("cuda"), None)+ # never started by the caller: the first onload starts it and waits for its own group+ with torch.no_grad():+ out = net(x)+ torch.cuda.synchronize()+ mem.stop_background_pins(pipe)+ assert torch.equal(out, ref)+ assert _groups_pinned(net)[1]+++def test_a_stopped_pinner_leaves_the_rest_to_diffusers_and_never_blocks():+ import core.inference.diffusion_memory as mem++ mem.install_group_pin_wait()+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")+ ref = _reference(x)+ pipe, net = _Pipe(), _offload(_net(), pinned = False)+ mem._defer_pinning(pipe, net, torch.device("cuda"), None)+ mem.stop_background_pins(pipe)+ with torch.no_grad():+ out = net(x)+ torch.cuda.synchronize()+ assert torch.equal(out, ref)+ assert not _groups_pinned(net)[1]diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.pyindex 1977c9f6567..8d25d7380c1 100644--- a/studio/backend/tests/test_video_backend.py+++ b/studio/backend/tests/test_video_backend.py@@ -8143,8 +8143,8 @@ def test_pipeline_plan_budgets_a_pre_cast_text_encoder_at_its_real_size(fake_runassert calls[0]["model_dense_mib"] == int((transformer_gb + text_encoder_gb * scale + vae_gb) * _MIB_PER_GB)- # The pipeline kind budgets one total, so companion_dense_mib stays None as before.- assert calls[0]["companion_dense_mib"] is None+ assert calls[0]["companion_dense_mib"] == int((text_encoder_gb * scale + vae_gb) * _MIB_PER_GB)+ assert calls[0]["text_encoder_dense_mib"] == int(text_encoder_gb * scale * _MIB_PER_GB)def test_plan_returns_to_bf16_when_the_pre_cast_encoder_does_not_inject(fake_runtime, monkeypatch):@@ -11185,10 +11185,21 @@ def test_a_resident_video_int8_on_nvidia_keeps_torchao(fake_runtime, monkeypatchbackend.unload()-def test_video_auto_under_offload_on_nvidia_is_still_skipped(fake_runtime, monkeypatch):+def test_video_auto_under_offload_on_nvidia_takes_the_torchao_free_int8(fake_runtime, monkeypatch):calls = _stub_nvidia_video_offload(monkeypatch)backend = VideoBackend()status = backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")+ assert [(call["mode"], call["offload"]) for call in calls] == [("int8", True)]+ assert status["transformer_quant"] == "int8"+ backend.unload()+++def test_video_auto_under_offload_on_nvidia_keeps_bf16_under_speed_off(fake_runtime, monkeypatch):+ calls = _stub_nvidia_video_offload(monkeypatch)+ backend = VideoBackend()+ status = backend.load_pipeline(+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", speed_mode = "off"+ )assert calls == []assert status["transformer_quant"] is Nonebackend.unload()diff --git a/studio/backend/tests/test_video_quantised_offload.py b/studio/backend/tests/test_video_quantised_offload.pynew file mode 100644index 00000000000..1f85c26c1ed--- /dev/null+++ b/studio/backend/tests/test_video_quantised_offload.py@@ -0,0 +1,887 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Conventional video families (Wan2.2, HunyuanVideo-1.5, LTX-2) when the card has to offload."""++import dataclasses+import sys+import types++import pytest++from .test_video_backend import ( # noqa: F401 -- fixtures+ _assume_the_restricted_load_is_available,+ _FakePipe,+ _load_gguf,+ fake_runtime,+)+from core.inference.video import VideoBackend++GIB = 1024+TIERS = (8, 12, 16, 24, 32, 48, 80)+CAPS = {"sm_120": (12, 0), "sm_89": (8, 9)}+SUPPORTED = {(12, 0): {"int8", "fp8", "mxfp8", "nvfp4"}, (8, 9): {"int8", "fp8"}}++FAMILIES = {+ "wan2.2-ti2v-5b": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",+ "wan2.2-t2v-a14b": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",+ "hv15-480p": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",+ "hv15-720p": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v",+ "ltx-2": "Lightricks/LTX-2",+}++# Tiers: none, dit (TE streamed), stream (DiT streamed), both, model (whole-module). Precision: bf16, ao (torchao), w8 (torchao-free int8).+_BF, _AO, _W8 = "bf16", "ao", "w8"+EXPECTED = {+ "wan2.2-ti2v-5b": {+ 8: ("model", _W8),+ 12: ("model", _W8),+ 16: ("model", _W8),+ 24: ("dit", _AO),+ 32: ("dit", _BF),+ 48: ("none", _BF),+ 80: ("none", _BF),+ },+ "wan2.2-t2v-a14b": {+ 8: ("model", _W8),+ 12: ("both", _W8),+ 16: ("both", _W8),+ 24: ("model", _W8),+ 32: ("model", _W8),+ 48: ("dit", _AO),+ 80: ("dit", _BF),+ },+ "hv15-480p": {+ 8: ("model", _W8),+ 12: ("model", _W8),+ 16: ("both", _W8),+ 24: ("dit", _AO),+ 32: ("dit", _BF),+ 48: ("dit", _BF),+ 80: ("none", _BF),+ },+ "ltx-2": {+ 8: ("model", _W8),+ 12: ("model", _W8),+ 16: ("both", _W8),+ 24: ("both", _W8),+ 32: ("model", _W8),+ 48: ("dit", _AO),+ 80: ("dit", _BF),+ },+}+EXPECTED["hv15-720p"] = EXPECTED["hv15-480p"]+++def _tier_code(plan) -> str:+ if plan.offload_policy == "group":+ if not plan.stream_transformer:+ return "dit"+ return "both" if plan.stream_text_encoders else "stream"+ return plan.offload_policy+++def _spoof(+ monkeypatch,+ *,+ tier_gib: int,+ cap = (12, 0),+):+ import core.inference.video as V+ from core.inference import diffusion_transformer_quant as tq+ from core.inference.diffusion_device import DiffusionDeviceTarget+ from core.inference.diffusion_memory import DeviceMemory++ torch = sys.modules["torch"]+ target = DiffusionDeviceTarget(+ device = "cuda",+ dtype = torch.bfloat16,+ backend = "cuda",+ vendor = "nvidia",+ supports_model_cpu_offload = True,+ supports_default_torch_compile = True,+ supports_pinned_transfer = True,+ )+ total = tier_gib * GIB+ mem = DeviceMemory(+ backend = "cuda",+ device = "cuda",+ memory_kind = "discrete_vram",+ free_mib = total - 600,+ total_mib = total,+ )+ monkeypatch.setattr(V, "settled_snapshot_device_memory", lambda *a, **k: mem)+ monkeypatch.setattr(VideoBackend, "_device_target", lambda self, ordinal = None: target)+ monkeypatch.setattr(V, "resolve_diffusion_device_target", lambda *a, **k: target)+ monkeypatch.setattr(V, "dense_transformer_supported", lambda t: True)+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda t: True)+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)+ monkeypatch.setattr(+ tq, "_scheme_supported", lambda s, d, unproven_ok = False: s in SUPPORTED[cap]+ )+ monkeypatch.setattr(tq, "_is_consumer_gpu", lambda d: tier_gib <= 32)+ monkeypatch.setattr(tq, "native_quant_host", lambda t: False)+ monkeypatch.setattr(tq, "native_offload_host", lambda t: True)+ monkeypatch.setattr(V, "stored_denoiser_precision", lambda *a, **k: None)+ spy = types.SimpleNamespace(plans = [], quant = [], speed = [], seeded = [])++ import core.inference.video_denoiser_prequant as vdp++ def _seed(fam, base, *, scheme, **kw):+ spy.seeded.append(scheme)+ return {"transformer": object()}++ monkeypatch.setattr(vdp, "denoiser_prequant_pipe_kwargs", _seed)++ def _quantize(+ view,+ tgt,+ *,+ mode,+ family = None,+ logger = None,+ **kw,+ ):+ spy.quant.append({"mode": mode, **kw})+ if kw.get("offload"):+ return "int8"+ return tq.select_transformer_quant_scheme(tgt, mode, family = family)++ monkeypatch.setattr(V, "quantize_transformer", _quantize)+ monkeypatch.setattr(V, "native_quant_reason", lambda m, s: f"native {s}")++ def _speed(view, tgt, *, is_gguf, family, speed_mode, cache_active, offload_active, **kw):+ compiled = (+ speed_mode in ("default", "max") and not is_gguf and family.supports_torch_compile+ )+ spy.speed.append({"compiled": compiled, "offload_active": offload_active})+ return {"compiled": compiled}++ monkeypatch.setattr(V, "apply_speed_optims", _speed)++ def _apply(+ pipe,+ plan,+ *,+ device = None,+ placement_device = None,+ logger = None,+ ):+ spy.plans.append(plan)+ return plan.offload_policy, plan.vae_tiling++ monkeypatch.setattr(V, "apply_memory_plan", _apply)+ return spy+++def _precision_code(spy, status) -> str:+ engaged = status.get("transformer_quant")+ if not engaged or engaged == "off":+ return _BF+ assert engaged == "int8", engaged+ return _W8 if spy.quant and spy.quant[0].get("offload") else _AO++[email protected]("cap_name", list(CAPS))[email protected]("tier_gib", TIERS)[email protected]("family", list(FAMILIES))+def test_auto_planner_table(fake_runtime, monkeypatch, family, tier_gib, cap_name):+ """Every conventional family at every card size: tier, precision, compile on, admitted."""+ spy = _spoof(monkeypatch, tier_gib = tier_gib, cap = CAPS[cap_name])+ status = VideoBackend().load_pipeline(FAMILIES[family])+ tier, precision = EXPECTED[family][tier_gib]+ assert _tier_code(spy.plans[-1]) == tier+ assert _precision_code(spy, status) == precision+ assert spy.speed and all(call["compiled"] for call in spy.speed)+ assert status["loaded"] is True+ if precision == _W8:+ # the torchao-free build is asked for by name: auto alone would walk the torchao ladder+ assert spy.quant[0]["mode"] == "int8"++[email protected]("family", ["wan2.2-t2v-a14b", "ltx-2"])+def test_auto_keeps_bf16_where_the_dit_stays_resident_at_80(fake_runtime, monkeypatch, family):+ """80 GiB: bf16 DiT stays resident with the text encoder streamed, so auto keeps bf16."""+ spy = _spoof(monkeypatch, tier_gib = 80)+ status = VideoBackend().load_pipeline(FAMILIES[family])+ assert _tier_code(spy.plans[-1]) == "dit"+ assert status["transformer_quant"] in (None, "off")+ assert not spy.quant+++def test_explicit_fp8_resident_dit_beside_streamed_text_encoder(fake_runtime, monkeypatch):+ """A torchao fp8 DiT on 24 GiB engages once the text encoder streams."""+ spy = _spoof(monkeypatch, tier_gib = 24)+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "fp8")+ assert _tier_code(spy.plans[-1]) == "dit"+ assert status["transformer_quant"] == "fp8"+ assert spy.quant and not spy.quant[0].get("offload")+++def test_explicit_fp8_refused_with_reason_when_the_dit_must_move(fake_runtime, monkeypatch):+ _spoof(monkeypatch, tier_gib = 16)+ with pytest.raises(RuntimeError, match = r"moves the DiT .*int8 runs torchao-free"):+ VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "fp8")+++def test_explicit_int8_under_streamed_dit_is_torchao_free(fake_runtime, monkeypatch):+ spy = _spoof(monkeypatch, tier_gib = 24)+ status = VideoBackend().load_pipeline(FAMILIES["ltx-2"], transformer_quant = "int8")+ assert _tier_code(spy.plans[-1]) == "both"+ assert status["transformer_quant"] == "int8"+ assert spy.quant[0] == {"mode": "int8", "offload": True, "act_int8": True}+++def test_speed_off_keeps_bf16_under_offload(fake_runtime, monkeypatch):+ """speed_mode='off' is the bit-exact contract: no auto quant even where the DiT has to move."""+ spy = _spoof(monkeypatch, tier_gib = 16)+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], speed_mode = "off")+ assert status["transformer_quant"] in (None, "off")+ assert not spy.quant+++def test_hosted_nvfp4_seeds_beside_streamed_text_encoder(fake_runtime, monkeypatch):+ """Wan2.2-5B NVFP4 on 32 GiB: the split keeps the DiT resident and streams the 11.4 GB encoder."""+ monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")+ spy = _spoof(monkeypatch, tier_gib = 32)+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "nvfp4")+ assert spy.seeded == ["nvfp4"]+ assert status["transformer_quant"] == "nvfp4"+ assert _tier_code(spy.plans[-1]) == "dit"+++def test_hosted_nvfp4_refused_where_the_dit_must_stream(fake_runtime, monkeypatch):+ monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")+ spy = _spoof(monkeypatch, tier_gib = 16)+ with pytest.raises(RuntimeError, match = "moves the DiT"):+ VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "nvfp4")+ assert not spy.seeded+++def test_seed_stays_resident_counts_the_streamed_text_encoder(monkeypatch):+ torch = pytest.importorskip("torch")+ import core.inference.video as V+ from core.inference.diffusion_memory import DeviceMemory+ from core.inference.video_families import detect_video_family++ fam = detect_video_family(FAMILIES["wan2.2-ti2v-5b"])+ target = types.SimpleNamespace(+ device = "cuda", dtype = torch.bfloat16, supports_model_cpu_offload = True+ )++ def _at(gib):+ mem = DeviceMemory(+ backend = "cuda",+ device = "cuda",+ memory_kind = "discrete_vram",+ free_mib = gib * GIB - 600,+ total_mib = gib * GIB,+ )+ monkeypatch.setattr(V, "settled_snapshot_device_memory", lambda *a, **k: mem)+ return V._video_seed_stays_resident(+ fam,+ target = target,+ scheme = "nvfp4",+ memory_mode = None,+ text_encoder_quant = None,+ base_repo = FAMILIES["wan2.2-ti2v-5b"],+ )++ assert _at(32) is True+ assert _at(12) is False+++def test_offload_tiers_rank_fastest_first():+ from core.inference.video import _video_plan_label, _video_plan_rank++ def _plan(+ policy,+ te = False,+ dit = True,+ ):+ return types.SimpleNamespace(+ offload_policy = policy, stream_text_encoders = te, stream_transformer = dit+ )++ whole = _plan("model")+ whole.estimates = {"whole_module_fits_mib": 20000}+ order = [+ _plan("none"),+ _plan("group", te = True, dit = False),+ whole,+ _plan("group"),+ _plan("group", te = True),+ _plan("model"),+ _plan("streaming"),+ ]+ assert [_video_plan_rank(p) for p in order] == [0, 1, 2, 3, 4, 5, 6]+ assert _video_plan_label(order[1]) == "denoiser resident, text encoders streamed"+++ "state, expected",+ [+ # a torchao DiT under whole-module offload: the hook swap fails under inference_mode+ (dict(transformer_quant = "int8", offload_policy = "model"), True),+ (dict(transformer_quant = "fp8", offload_policy = "group"), True),+ (dict(transformer_quant = "int8", offload_policy = "none"), False),+ (dict(transformer_quant = None, offload_policy = "model"), False),+ # MiniMax-H3: only its streamed denoiser switches; the pinned one never moves+ (dict(transformer_quant = "int8", offload_policy = "model", modular = True), False),+ (+ dict(+ transformer_quant = "int8",+ offload_policy = "model",+ modular = True,+ denoiser_streamed = True,+ ),+ True,+ ),+ ],+)+def test_render_grad_mode(state, expected):+ from core.inference.video import _render_under_no_grad++ modular = state.pop("modular", False)+ ns = types.SimpleNamespace(+ family = types.SimpleNamespace(modular_workflow = "t2v" if modular else None),+ denoiser_streamed = state.pop("denoiser_streamed", False),+ **state,+ )+ assert _render_under_no_grad(ns) is expected+++def test_vram_floor_counts_what_each_tier_holds_at_once():+ torch = pytest.importorskip("torch")+ from core.inference.video import _video_offload_vram_floor_mib++ def _module(mib):+ module = torch.nn.Module()+ module.register_buffer("w", torch.zeros(mib * 1024 * 1024 // 2, dtype = torch.bfloat16))+ return module++ pipe = types.SimpleNamespace(+ components = {+ "transformer": _module(10),+ "transformer_2": _module(10),+ "text_encoder": _module(12),+ "vae": _module(3),+ "scheduler": object(),+ }+ )++ def _plan(+ policy,+ te = False,+ dit = True,+ ):+ return types.SimpleNamespace(+ offload_policy = policy, stream_text_encoders = te, stream_transformer = dit+ )++ assert _video_offload_vram_floor_mib(pipe, _plan("none")) is None+ assert _video_offload_vram_floor_mib(pipe, _plan("model")) == 12+ # a flat module is one group, so its whole payload is onloaded during its forward+ assert _video_offload_vram_floor_mib(pipe, _plan("group", te = True, dit = False)) == 35+ assert _video_offload_vram_floor_mib(pipe, _plan("group")) == 25+ assert _video_offload_vram_floor_mib(pipe, _plan("group", te = True)) == 15+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming")) == 15+++def test_vram_floor_adds_the_largest_onloaded_block_or_leaf():+ """Streamed tiers hold the unmatched group plus the current and prefetched block (or leaf) on the device."""+ torch = pytest.importorskip("torch")+ from core.inference.video import _video_offload_vram_floor_mib++ def _linear(mib):+ return torch.nn.Linear(1024, mib * 512, bias = False).to(torch.bfloat16)++ dit = torch.nn.Module()+ dit.patch_embedding = _linear(1)+ dit.blocks = torch.nn.ModuleList([_linear(2) for _ in range(4)])+ encoder = torch.nn.Module()+ encoder.shared = torch.nn.Embedding(1024, 3 * 512).to(torch.bfloat16)+ encoder.layers = torch.nn.ModuleList([_linear(1) for _ in range(4)])+ vae = _linear(3)+ pipe = types.SimpleNamespace(+ components = {"transformer": dit, "text_encoder": encoder, "vae": vae}+ )++ def _plan(+ policy,+ backend,+ te = False,+ dit = True,+ ):+ return types.SimpleNamespace(+ offload_policy = policy,+ stream_text_encoders = te,+ stream_transformer = dit,+ device_memory = types.SimpleNamespace(backend = backend),+ )++ # resident TE 7 + VAE 3, plus DiT embed 1 + current and prefetched 2 MiB blocks+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda")) == 15+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "mps")) == 13+ # resident DiT 9 + VAE 3, plus the 3 MiB embedding and the next 1 MiB leaf+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda", te = True, dit = False)) == 16+ # streamed modules run in turn: VAE 3 plus the larger of DiT 5 and TE 4+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cuda")) == 8+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cpu")) == 6+ # the VAE decode runs after the streamed DiT has offloaded: only the residents (or the decoder alone) remain+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda"), phase = "decode") == 10+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda", te = True), phase = "decode") == 3+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cuda"), phase = "decode") == 3+ assert _video_offload_vram_floor_mib(pipe, _plan("model", "cuda"), phase = "decode") == 3+++def test_shortfall_message_only_for_what_cannot_run():+ from core.inference.video import video_offload_shortfall_message++ assert (+ video_offload_shortfall_message(+ family = "wan2.2-t2v-a14b", floor_mib = 20000, available_mib = 24000, placement = "x"+ )+ is None+ )+ message = video_offload_shortfall_message(+ family = "wan2.2-t2v-a14b",+ floor_mib = 27300,+ available_mib = 24000,+ placement = "whole-module CPU offload",+ width = 1280,+ height = 720,+ frames = 81,+ )+ assert "wan2.2-t2v-a14b needs about" in message and "1280x720 at 81 frames" in message+++def test_generate_requirement_grows_with_the_requested_clip():+ import re++ from core.inference.video import video_offload_shortfall_message++ def _required(width, height, frames):+ message = video_offload_shortfall_message(+ family = "wan2.2-ti2v-5b",+ floor_mib = 10000,+ available_mib = 0,+ placement = "x",+ width = width,+ height = height,+ frames = frames,+ )+ return float(re.search(r"needs about ([0-9.]+) GiB", message).group(1))++ base = _required(640, 480, 49)+ assert _required(1280, 480, 49) > base+ assert _required(640, 960, 49) > base+ assert _required(640, 480, 161) > base+ # a small clip still fits where a long 720p one does not+ kwargs = dict(family = "f", floor_mib = 10000, available_mib = 12500, placement = "x")+ assert video_offload_shortfall_message(**kwargs, width = 256, height = 256, frames = 9) is None+ assert video_offload_shortfall_message(**kwargs, width = 1280, height = 720, frames = 121)+++def test_load_refuses_a_tier_whose_weights_cannot_fit(fake_runtime, monkeypatch):+ """A tier whose co-resident weights exceed the card is refused before placement, not at the first onload."""+ import core.inference.video as V++ spy = _spoof(monkeypatch, tier_gib = 24)+ monkeypatch.setattr(V, "_video_offload_vram_floor_mib", lambda pipe, plan: 30000)+ with pytest.raises(RuntimeError, match = "needs about"):+ VideoBackend().load_pipeline(FAMILIES["wan2.2-t2v-a14b"])+ assert not spy.plans+++def test_generate_refuses_below_the_offload_floor(fake_runtime, monkeypatch, tmp_path):+ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ torch = sys.modules["torch"]+ monkeypatch.setattr(+ torch,+ "device",+ lambda kind, index = None: types.SimpleNamespace(type = kind, index = index),+ raising = False,+ )+ free = {"bytes": 40 * 1024**3}+ monkeypatch.setattr(+ torch,+ "cuda",+ types.SimpleNamespace(+ is_available = lambda: False,+ mem_get_info = lambda device = None: (free["bytes"], 80 * 1024**3),+ memory_reserved = lambda device = None: 0,+ ),+ )+ backend._state = dataclasses.replace(+ backend._state, device = "cuda", offload_policy = "model", vram_floor_mib = 27300+ )+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)+ free["bytes"] = 20 * 1024**3+ with pytest.raises(RuntimeError, match = "needs about"):+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)+ # same free VRAM: a small clip runs, a long 720p clip is refused before the render+ free["bytes"] = 31 * 1024**3+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)+ with pytest.raises(RuntimeError, match = "at 121 frames"):+ backend.generate(prompt = "a sloth", width = 1280, height = 720, num_frames = 121, fps = 8)+++def test_quantise_stages_what_fits_on_the_card_and_unstages_it(monkeypatch):+ """The DiT is quantised on the card block by block and sent back to the host when the plan moves it."""+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ import core.inference.video as V++ dit = torch.nn.Module()+ dit.blocks = torch.nn.ModuleList(+ [torch.nn.Linear(1024, 1024, bias = False).to(torch.bfloat16) for _ in range(4)]+ )+ dit.proj_out = torch.nn.Linear(1024, 64, bias = False).to(torch.bfloat16)+ block_bytes = 1024 * 1024 * 2+ target = types.SimpleNamespace(device = "cuda", torch_device = "cuda")+ # room for the overhead, the linear workspace and exactly two blocks+ free = (+ V.DEFAULT_BASE_OVERHEAD_MIB * 1024 * 1024+ + V._STAGE_LINEAR_WORKSPACE * block_bytes+ + 2 * block_bytes+ + 1+ )+ monkeypatch.setattr(+ "utils.hardware.trusted_mem_get_info", lambda d, module = None: (free, 10 * free)+ )+ staged = V._stage_denoiser_for_quant(dit, target)+ on_card = [m for m in staged if next(m.parameters()).is_cuda]+ assert len(on_card) == len(staged) >= 2+ assert all(next(b.parameters()).is_cuda for b in dit.blocks[:2])+ assert not next(dit.blocks[3].parameters()).is_cuda+ V._unstage_modules(staged)+ assert all(not p.is_cuda for p in dit.parameters())+ # nothing staged off CUDA, or for a DiT already on the card+ assert V._stage_denoiser_for_quant(dit, types.SimpleNamespace(device = "mps")) == []+++def test_leaf_streaming_peak_keeps_an_enclosing_parent_group_onloaded():+ """A parent's own params stay onloaded across its descendant leaves, so they add to the two-leaf window."""+ torch = pytest.importorskip("torch")+ from core.inference.video import _video_streamed_peak_bytes++ mib = 1 << 20++ def _linear(n):+ return torch.nn.Linear(1024, n * 512, bias = False).to(torch.bfloat16)++ root = torch.nn.Module()+ root.block = torch.nn.Module()+ root.block.table = torch.nn.Parameter(torch.zeros(mib // 2, dtype = torch.bfloat16))+ root.block.a = _linear(2)+ root.block.b = _linear(2)+ root.head = _linear(2)+ root.norm = torch.nn.LayerNorm(mib // 4, dtype = torch.bfloat16)++ def size(t):+ return t.numel() * t.element_size()++ # inside block: its 1 MiB table plus leaf a and the prefetched leaf b+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 5 * mib+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = False, size = size) == 3 * mib+++def test_decoded_clip_share_sits_beside_the_decode_floor_not_the_streamed_dit():+ """Denoise and VAE decode are separate phases: the clip share never stacks on the streamed DiT's weights."""+ import inspect+ import re++ import core.inference.video as V++ kwargs = dict(family = "f", floor_mib = 20000, placement = "x", width = 1280, height = 720, frames = 121)+ # denoise needs 20000 + 2048; decode needs 3000 + 2048 + the clip share, so 23000 MiB is enough+ assert (+ V.video_offload_shortfall_message(**kwargs, available_mib = 23000, decode_floor_mib = 3000)+ is None+ )+ # unknown decode floor keeps the conservative sum+ assert V.video_offload_shortfall_message(**kwargs, available_mib = 23000)+ # a huge clip makes the decode phase the binding one, even with a small decode floor+ big = dict(kwargs, floor_mib = 5000, width = 1920, height = 1080, frames = 241)+ message = V.video_offload_shortfall_message(**big, available_mib = 12000, decode_floor_mib = 4000)+ assert message and float(re.search(r"needs about ([0-9.]+) GiB", message).group(1)) > 20+ # the generate guard hands the decode floor recorded at load to the check+ src = inspect.getsource(V)+ assert 'decode_floor_mib = getattr(state, "vram_decode_floor_mib", None)' in src+ assert 'phase = "decode"' in src+++def test_leaf_streaming_peak_does_not_stack_sibling_branch_groups():+ """A sibling branch has offloaded its own params before the next runs: only the enclosing chain stacks."""+ torch = pytest.importorskip("torch")+ from core.inference.video import _video_streamed_peak_bytes++ mib = 1 << 20++ def _branch():+ branch = torch.nn.Module()+ branch.table = torch.nn.Parameter(torch.zeros(mib // 2, dtype = torch.bfloat16))+ branch.leaf = torch.nn.Linear(1024, 2 * 512, bias = False).to(torch.bfloat16)+ return branch++ root = torch.nn.Module()+ root.first, root.second = _branch(), _branch()++ def size(t):+ return t.numel() * t.element_size()++ # one 1 MiB enclosing table plus the current and prefetched 2 MiB leaves; not both tables+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 5 * mib+++def test_applied_floor_is_recorded_even_when_lower_than_planned():+ """The fallback can stream a DiT the plan kept resident: the lower applied floor must replace the planned one."""+ import inspect++ import core.inference.video as V++ src = inspect.getsource(V)+ assert "raised = applied_floor is not None and applied_floor > vram_floor_mib" in src+ assert (+ "if applied_floor is not None:\n # record what landed even when lower"+ in src+ )+++def test_whole_module_vae_phase_counts_only_the_decoded_clip_share():+ """The VAE phase holds the decoded clip, not the denoise-side base the DiT phase already carries."""+ import dataclasses as dc++ import core.inference.video as V+ from core.inference.diffusion_memory import MEMORY_MODE_AUTO, OFFLOAD_GROUP, OFFLOAD_MODEL++ @dc.dataclass+ class _Plan:+ offload_policy: str = OFFLOAD_GROUP+ stream_transformer: bool = True+ stream_text_encoders: bool = False+ requested_mode: str = MEMORY_MODE_AUTO+ vae_tiling: bool = False+ vae_slicing: bool = False+ reasons: tuple = ()+ estimates: dict = dc.field(default_factory = lambda: {"safe_device_budget_mib": 17000})++ runtime = V.estimate_video_runtime_mib(width = 1280, height = 704, num_frames = 121)+ # DiT phase 10000 + 4096 + 2048 = 16144; VAE phase 8000 + clip share + 2048 ~ 13.8 GiB, not 8000 + runtime + 2048+ out = V._video_prefer_whole_module(+ _Plan(), denoiser_mib = 10000, text_encoder_mib = 6000, vae_mib = 8000, runtime_mib = runtime+ )+ assert out.offload_policy == OFFLOAD_MODEL+ assert out.estimates["whole_module_fits_mib"] == 10000 + V._VIDEO_DENOISE_ACTIVATION_MIB + 2048+++def test_quantised_replan_prices_companions_like_the_split_it_passes():+ """The replan hands the planner the dtype-scaled text encoder, so its total must use the same scaled companions."""+ import inspect++ import core.inference.video as V++ src = inspect.getsource(V)+ assert "quant_mib = int((components[0] * factor + scaled_companions_gb) * mib_per_gb)" in src+ assert "quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)" not in src+++def test_leaf_streaming_peak_counts_a_large_parent_group_once():+ """A parent's loose params are held once under its leaves; they are not also a second prefetch candidate."""+ torch = pytest.importorskip("torch")+ from core.inference.video import _video_streamed_peak_bytes++ mib = 1 << 20+ root = torch.nn.Module()+ root.parent = torch.nn.Module()+ root.parent.table = torch.nn.Parameter(torch.zeros(2 * mib, dtype = torch.bfloat16))+ root.parent.a = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)+ root.parent.b = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)++ def size(t):+ return t.numel() * t.element_size()++ # the 4 MiB table plus leaf a and the prefetched leaf b, not 4 + 4 + 1+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 6 * mib+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = False, size = size) == 5 * mib+++def test_applied_floor_counts_an_encoder_that_refused_leaf_offload():+ """_apply_group_offload keeps a refusing encoder resident under the same policy; the floor must follow the hooks."""+ torch = pytest.importorskip("torch")+ import inspect++ import core.inference.video as V++ def _module(mib):+ module = torch.nn.Module()+ module.register_buffer("w", torch.zeros(mib * 1024 * 1024 // 2, dtype = torch.bfloat16))+ return module++ dit, encoder = _module(10), _module(12)+ dit._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})+ pipe = types.SimpleNamespace(+ components = {"transformer": dit, "text_encoder": encoder, "vae": _module(3)}+ )+ plan = types.SimpleNamespace(+ offload_policy = "group",+ stream_text_encoders = True,+ stream_transformer = True,+ device_memory = types.SimpleNamespace(backend = "cuda"),+ )+ # as planned: VAE 3 plus the larger streamed unit (TE 12)+ assert V._video_offload_vram_floor_mib(pipe, plan) == 15+ # as applied: the unhooked encoder is resident, so VAE 3 + TE 12 plus the streamed DiT 10+ assert V._video_offload_vram_floor_mib(pipe, plan, applied = True) == 25+ assert "_video_offload_vram_floor_mib(pipe, plan, applied = True)" in inspect.getsource(V)+++def test_failed_staging_leaves_nothing_on_the_card(monkeypatch):+ torch = pytest.importorskip("torch")+ if not torch.cuda.is_available():+ pytest.skip("needs a CUDA device")+ import core.inference.video as V++ class _Refuses(torch.nn.Linear):+ def to(self, *args, **kwargs):+ raise RuntimeError("CUDA out of memory")++ dit = torch.nn.Module()+ dit.blocks = torch.nn.ModuleList(+ [torch.nn.Linear(256, 256, bias = False).to(torch.bfloat16) for _ in range(2)]+ + [_Refuses(256, 256, bias = False)]+ )+ monkeypatch.setattr(+ "utils.hardware.trusted_mem_get_info", lambda d, module = None: (80 * 1024**3, 80 * 1024**3)+ )+ target = types.SimpleNamespace(device = "cuda", torch_device = "cuda")+ assert V._stage_denoiser_for_quant(dit, target) == []+ assert all(not p.is_cuda for p in dit.parameters())+++def test_applied_floor_counts_a_dit_the_fallback_hooked():+ """A refusing encoder can make _apply_group_offload hook the DiT the plan kept resident; count it as streamed."""+ torch = pytest.importorskip("torch")+ import core.inference.video as V++ def _linear(mib):+ return torch.nn.Linear(1024, mib * 512, bias = False).to(torch.bfloat16)++ dit = torch.nn.Module()+ dit.patch_embedding = _linear(1)+ dit.blocks = torch.nn.ModuleList([_linear(2) for _ in range(4)])+ dit.blocks[0]._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})+ encoder = torch.nn.Module()+ encoder.register_buffer("w", torch.zeros(12 * 1024 * 1024 // 2, dtype = torch.bfloat16))+ vae = torch.nn.Module()+ vae.register_buffer("w", torch.zeros(3 * 1024 * 1024 // 2, dtype = torch.bfloat16))+ pipe = types.SimpleNamespace(+ components = {"transformer": dit, "text_encoder": encoder, "vae": vae}+ )+ plan = types.SimpleNamespace(+ offload_policy = "group",+ stream_text_encoders = True,+ stream_transformer = False,+ device_memory = types.SimpleNamespace(backend = "cuda"),+ )+ # resident TE 12 + VAE 3, plus the streamed DiT's embed 1 and current and prefetched 2 MiB blocks+ assert V._video_offload_vram_floor_mib(pipe, plan, applied = True) == 20+++def test_a_declined_quant_unstages_before_any_bf16_rollback():+ import inspect++ import core.inference.video as V+ assert "if staged and (video_offload or scheme is None):" in inspect.getsource(V)+++def test_a_render_starts_only_after_the_background_pins_finish(fake_runtime, monkeypatch, tmp_path):+ """A render overlapping the post-load pinner ran every later render 2.5-3 s slower (LTX-2.3, A100): wait first."""+ import core.inference.diffusion_memory as mem++ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ pipe = backend._state.pipe+ order: list = []++ class _Pinner:+ module = None++ def start(self):+ order.append("start")++ def join(self, timeout = None):+ order.append("pinned")+ return True++ def stop(self, timeout = None):+ pass++ setattr(pipe, mem._PENDING_PINS_ATTR, [_Pinner()])+ monkeypatch.setattr(+ pipe, "scheduler", types.SimpleNamespace(step = lambda *a, **k: None), raising = False+ )+ original_call = type(pipe).__call__++ def _call(self, *args, **kwargs):+ order.append("render")+ return original_call(self, *args, **kwargs)++ monkeypatch.setattr(type(pipe), "__call__", _call)+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)+ assert order[:3] == ["start", "pinned", "render"], order+++def test_a_cancel_during_the_background_pin_wait_ends_the_render_promptly(+ fake_runtime, monkeypatch, tmp_path+):+ """The wait held _generate_lock for the whole pin (minutes on LTX-2.3), so Cancel and unload waited too."""+ import re+ import threading+ import time++ import core.inference.diffusion_memory as mem+ from core.inference.video import VIDEO_CANCELLED_MSG++ backend = VideoBackend()+ _load_gguf(backend, tmp_path)+ pipe = backend._state.pipe+ rendered: list = []++ class _SlowPinner:+ module = None++ def start(self):+ pass++ def join(self, timeout = None):+ # A pin that takes 3 s; an unbounded join waits it out, like the real worker.+ time.sleep(3.0 if timeout is None else timeout)+ return timeout is None++ def stop(self, timeout = None):+ pass++ setattr(pipe, mem._PENDING_PINS_ATTR, [_SlowPinner()])+ monkeypatch.setattr(+ pipe, "scheduler", types.SimpleNamespace(step = lambda *a, **k: None), raising = False+ )+ original_call = type(pipe).__call__++ def _call(self, *args, **kwargs):+ rendered.append(1)+ return original_call(self, *args, **kwargs)++ monkeypatch.setattr(type(pipe), "__call__", _call)+ cancel = threading.Event()+ threading.Timer(0.5, cancel.set).start()+ began = time.perf_counter()+ with pytest.raises(RuntimeError, match = re.escape(VIDEO_CANCELLED_MSG)):+ backend.generate(+ prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8, cancel_event = cancel+ )+ assert time.perf_counter() - began < 10+ assert not rendered