Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep INT8 and compile on conventional video models when they have to offload (#12289)

Daniel Hane8b2bb2f959b
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index ef573ad7983..db864631521 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -210,8 +210,9 @@ def _pin_host_weights(
module: Any,
host: dict,
logger: Any = None,
+ buffer_host: Optional[dict] = None,
) -> int:
- """Pack kept host weights into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""
+ """Pack kept host weights and buffers into page-locked chunks; returns bytes pinned. Contiguous only, so channels_last survives."""
import torch
mode = _pin_mode()
@@ -226,6 +227,16 @@ def _pin_host_weights(
and p.data.is_contiguous()
and not p.data.is_pinned()
]
+ buffers = [
+ (name, b)
+ for name, b in _named_buffers(module)
+ if (buffer_host or {}).get(name) is b
+ and b.device.type == "cpu"
+ and _keepable_buffer(b)
+ and b.is_contiguous()
+ and not b.is_pinned()
+ ]
+ params = params + [("\0" + name, b) for name, b in buffers]
sizes = [-(-p.data.nbytes // _PIN_ALIGN) * _PIN_ALIGN for _, p in params]
if not sizes or sum(sizes) == 0:
return 0
@@ -288,11 +299,40 @@ def _pin_host_weights(
)
return 0
for name, p, view in placed:
+ if name.startswith("\0"):
+ name = name[1:]
+ _set_buffer(module, name, view)
+ buffer_host[name] = view
+ continue
p.data = view
host[name] = view
return need
+def _named_buffers(module: Any) -> list:
+ named = getattr(module, "named_buffers", None)
+ return list(named()) if callable(named) else []
+
+
+def _keepable_buffer(buffer: Any) -> bool:
+ import torch
+ return type(buffer) is torch.Tensor
+
+
+def _buffer_version(buffer: Any) -> Optional[int]:
+ """``_version``, or None for an inference tensor (moved under inference_mode), which has no version counter."""
+ try:
+ return None if buffer.is_inference() else int(buffer._version)
+ except Exception: # noqa: BLE001 - untracked: the stock copy handles it
+ return None
+
+
+def _set_buffer(module: Any, name: str, tensor: Any) -> None:
+ prefix, _, leaf = name.rpartition(".")
+ owner = module.get_submodule(prefix) if prefix else module
+ owner._buffers[leaf] = tensor
+
+
def _wrap_cpu_offload_hook(
hook: Any,
module: Any,
@@ -305,20 +345,45 @@ def _wrap_cpu_offload_hook(
module.__dict__[_KEEP_ATTR] = state
state.setdefault("owner", {})
host, owner, version = state["host"], state["owner"], state["version"]
+ # Plain buffers too (torchao-free int8): the stock offload re-copied them every call, 6.3 s vs a 1.2 s denoise on Wan2.2-5B.
+ # Kept while the device copy is the same tensor at the same version.
+ buffer_host = state.setdefault("buffer_host", {})
+ buffer_version = state.setdefault("buffer_version", {})
def _capture(mod: Any) -> None:
host.clear()
owner.clear()
+ buffer_host.clear()
for name, p in mod.named_parameters():
if p.device.type == "cpu" and _keepable(p):
host[name] = p.data
owner[name] = p
+ for name, b in _named_buffers(mod):
+ if b.device.type == "cpu" and _keepable_buffer(b):
+ buffer_host[name] = b
_capture(module)
version.clear()
+ buffer_version.clear()
init_hook, pre_forward = hook.init_hook, hook.pre_forward
def _init_hook(mod: Any) -> Any:
+ for name, b in _named_buffers(mod):
+ kept = buffer_host.get(name)
+ seen = buffer_version.get(name)
+ if (
+ kept is None
+ or b.device.type == "cpu"
+ or seen is None
+ or seen[0] is not b
+ or seen[1] != _buffer_version(b)
+ or seen[2] != b.data_ptr()
+ ):
+ continue
+ try:
+ _set_buffer(mod, name, kept)
+ except Exception: # noqa: BLE001 - the stock copy below handles it
+ pass
for name, p in mod.named_parameters():
kept = host.get(name)
seen = version.get(name)
@@ -339,11 +404,12 @@ def _wrap_cpu_offload_hook(
out = init_hook(mod)
_capture(mod)
version.clear()
+ buffer_version.clear()
return out
def _pre_forward(mod: Any, *args: Any, **kwargs: Any) -> Any:
# `host` gate: an all-subclass module (GGUF, torchao) never fills `version`, so would rescan every forward.
- onload = not version and bool(host)
+ onload = not version and not buffer_version and bool(host or buffer_host)
if onload:
for name, p in mod.named_parameters():
kept = host.get(name)
@@ -354,11 +420,15 @@ def _wrap_cpu_offload_hook(
):
host.pop(name, None)
owner.pop(name, None)
+ current = dict(_named_buffers(mod))
+ for name in list(buffer_host):
+ if current.get(name) is not buffer_host[name]:
+ buffer_host.pop(name, None)
if onload and not state.get("pin_tried"):
# Once per module, on its first onload, so loading pays nothing.
state["pin_tried"] = True
try:
- pinned = _pin_host_weights(mod, host, logger)
+ pinned = _pin_host_weights(mod, host, logger, buffer_host = buffer_host)
except Exception: # noqa: BLE001 - pinning is an optimisation, never a failure
pinned = 0
if pinned and logger is not None:
@@ -372,6 +442,13 @@ def _wrap_cpu_offload_hook(
for name, p in mod.named_parameters():
if name in host and owner.get(name) is p and p.device.type != "cpu":
version[name] = (p, p._version, p.data_ptr())
+ for name, b in _named_buffers(mod):
+ if (
+ name in buffer_host
+ and b.device.type != "cpu"
+ and _buffer_version(b) is not None
+ ):
+ buffer_version[name] = (b, _buffer_version(b), b.data_ptr())
return out
try:
@@ -1608,6 +1685,259 @@ def _streamed_pin_plan(
return pin_transformer, pin_encoders
+BACKGROUND_PIN_ENV = "UNSLOTH_DIFFUSION_BACKGROUND_PIN"
+_BG_PIN_ATTR = "_unsloth_background_pin"
+_PENDING_PINS_ATTR = "_unsloth_background_pins"
+_BACKGROUND_PIN_REQUEST_ATTR = "_unsloth_background_pin_requested"
+
+
+def request_background_pins(pipe: Any) -> None:
+ """Ask the next group offload apply on ``pipe`` to pin off the load path (see _GroupPinner)."""
+ try:
+ setattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, True)
+ except Exception: # noqa: BLE001 - a pipe refusing attributes pins eagerly, as before
+ pass
+
+
+def _background_pin_enabled() -> bool:
+ return (os.environ.get(BACKGROUND_PIN_ENV) or "").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _offload_groups(module: Any) -> list:
+ """The diffusers offload groups hooked under ``module``, in registration (block) order."""
+ try:
+ from diffusers.hooks import group_offloading as go
+ name = getattr(go, "_GROUP_OFFLOADING", "group_offloading")
+ except Exception: # noqa: BLE001
+ return []
+ groups: list = []
+ seen: set = set()
+ for sub in module.modules():
+ registry = getattr(sub, "_diffusers_hook", None)
+ get_hook = getattr(registry, "get_hook", None)
+ group = getattr(get_hook(name), "group", None) if callable(get_hook) else None
+ if group is not None and id(group) not in seen:
+ seen.add(id(group))
+ groups.append(group)
+ return groups
+
+
+class _GroupPinner:
+ """Pins a streamed module's offload groups on a worker thread, first block first.
+
+ Pinning at load reads every streamed byte from disk before the load returns: 125-139 s of LTX-2.3's 37 GB DiT,
+ 20-44 s of Wan2.2-5B's text encoder on Colab. Off the load path the read overlaps the prompt encode and the
+ first compile. A group's ``onload_`` waits for that group only, and the swap happens before the wait releases,
+ so no group is ever onloaded while its host copy is being replaced."""
+
+ def __init__(
+ self,
+ module: Any,
+ groups: list,
+ device: Any,
+ logger: Any = None,
+ ):
+ import threading
+
+ self.module = module
+ self.label = type(module).__name__
+ self.groups = groups
+ self.device = device
+ self.logger = logger
+ self.pinned = 0
+ self.waited_s = 0.0
+ self._done = {id(g): threading.Event() for g in groups}
+ self._stop = threading.Event()
+ self._lock = threading.Lock()
+ self._thread = threading.Thread(target = self._run, name = "unsloth-offload-pin", daemon = True)
+ self._started = False
+ for group in groups:
+ setattr(group, _BG_PIN_ATTR, self)
+
+ def start(self) -> None:
+ with self._lock:
+ if not self._started:
+ self._started = True
+ self._thread.start()
+
+ def wait(self, group: Any) -> None:
+ import threading
+
+ event = self._done.get(id(group))
+ if event is None or threading.current_thread() is self._thread:
+ return
+ self.start()
+ if not event.is_set():
+ import time
+
+ began = time.perf_counter()
+ event.wait()
+ self.waited_s += time.perf_counter() - began
+
+ def join(self, timeout: Optional[float] = None) -> bool:
+ """Wait for the worker; True once it has exited (or when called from the worker itself)."""
+ import threading
+
+ if threading.current_thread() is self._thread:
+ return True
+ self._thread.join(timeout)
+ return not self._thread.is_alive()
+
+ def stop(self, timeout: Optional[float] = None) -> None:
+ self._stop.set()
+ if self._started:
+ self._thread.join(timeout)
+
+ def _unpinned(self, group: Any) -> list:
+ import torch
+ return [
+ (tensor, src)
+ for tensor, src in list(group.cpu_param_dict.items())
+ if type(src) is torch.Tensor
+ and src.device.type == "cpu"
+ and src.numel() > 0
+ and not src.is_pinned()
+ ]
+
+ def _run(self) -> None:
+ import time
+
+ import torch
+
+ start = time.perf_counter()
+ failed = None
+ try:
+ if (
+ getattr(self.device, "type", None) == "cuda"
+ and getattr(self.device, "index", None) is not None
+ ):
+ torch.cuda.set_device(self.device)
+ for group in self.groups:
+ if self._stop.is_set():
+ break
+ # One pin per tensor, exactly what the eager apply makes: chunk views streamed ~10% slower on LTX-2.3.
+ placed = [(tensor, src, src.pin_memory()) for tensor, src in self._unpinned(group)]
+ cpu = group.cpu_param_dict
+ for tensor, src, pinned in placed:
+ if cpu.get(tensor) is src:
+ cpu[tensor] = pinned
+ if tensor.device.type == "cpu" and tensor.data_ptr() == src.data_ptr():
+ tensor.data = pinned
+ self.pinned += src.nbytes
+ self._done[id(group)].set()
+ except Exception as exc: # noqa: BLE001 - diffusers pins what is left on each onload
+ failed = exc
+ finally:
+ for event in self._done.values():
+ event.set()
+ if self.logger is not None:
+ try:
+ if failed is not None:
+ self.logger.warning(
+ "diffusion.memory: background pinning of %s stopped (%s); the rest pins on each onload",
+ self.label,
+ failed,
+ )
+ self.logger.info(
+ "diffusion.memory: pinned %.1f GiB of %s host weights in the background in %.1f s "
+ "(onloads waited %.1f s)",
+ self.pinned / 2**30,
+ self.label,
+ time.perf_counter() - start,
+ self.waited_s,
+ )
+ except Exception: # noqa: BLE001
+ pass
+
+
+def install_group_pin_wait() -> bool:
+ """Make a diffusers offload group wait for its background pin before it onloads."""
+ try:
+ import torch
+ from diffusers.hooks import group_offloading as go
+ except Exception: # noqa: BLE001
+ return False
+ group_cls = getattr(go, "ModuleGroup", None)
+ original = getattr(group_cls, "onload_", None)
+ if original is None or getattr(original, "_unsloth_pin_wait", False):
+ return False
+
+ @functools.wraps(original)
+ def onload_(self, *args: Any, **kwargs: Any) -> Any:
+ pinner = getattr(self, _BG_PIN_ATTR, None)
+ if pinner is not None:
+ pinner.wait(self)
+ return original(self, *args, **kwargs)
+
+ disable = getattr(getattr(torch, "compiler", None), "disable", None)
+ wrapped = disable(onload_) if callable(disable) else onload_
+ wrapped._unsloth_pin_wait = True
+ group_cls.onload_ = wrapped
+ return True
+
+
+def start_background_pins(pipe: Any) -> int:
+ """Start the pinners the load deferred; returns how many."""
+ pinners = list(getattr(pipe, _PENDING_PINS_ATTR, None) or ())
+ for pinner in pinners:
+ pinner.start()
+ return len(pinners)
+
+
+def finish_background_pins(pipe: Any, cancel: Any = None) -> float:
+ """Block until every deferred pinner on ``pipe`` is done; returns the seconds waited.
+
+ A render that overlaps the pinner ran its warm renders 2.5 to 3 s slower on an A100 (LTX-2.3, n=15 per arm);
+ letting the pinner finish first matched the eager pin. So the pin overlaps the idle time after the load, not
+ the first render."""
+ import time
+
+ start = time.perf_counter()
+ for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
+ pinner.start()
+ # Polled so a cancelled request leaves now; the pin itself keeps running for the next render.
+ while not pinner.join(0.25):
+ if cancel is not None and cancel.is_set():
+ return time.perf_counter() - start
+ return time.perf_counter() - start
+
+
+def stop_background_pins(pipe: Any, timeout: Optional[float] = 30.0) -> None:
+ for pinner in list(getattr(pipe, _PENDING_PINS_ATTR, None) or ()):
+ try:
+ pinner.stop(timeout)
+ except Exception: # noqa: BLE001 - teardown is best effort
+ pass
+
+
+def _drop_deferred_pinning(pipe: Any, module: Any) -> None:
+ pending = getattr(pipe, _PENDING_PINS_ATTR, None)
+ if pending:
+ pending[:] = [pinner for pinner in pending if pinner.module is not module]
+
+
+def _defer_pinning(pipe: Any, module: Any, device: Any, logger: Any) -> bool:
+ groups = _offload_groups(module)
+ if not groups:
+ return False
+ pinner = _GroupPinner(module, groups, device, logger)
+ pending = getattr(pipe, _PENDING_PINS_ATTR, None)
+ if pending is None:
+ pending = []
+ try:
+ setattr(pipe, _PENDING_PINS_ATTR, pending)
+ except Exception: # noqa: BLE001 - nowhere to park it: pin now, like the eager path
+ pinner.start()
+ return True
+ pending.append(pinner)
+ return True
+
+
def install_group_offload_buffer_restore() -> bool:
"""diffusers stream group offload restores only parameters, leaving buffers (native int8 weights) on the GPU."""
try:
@@ -1650,6 +1980,7 @@ def _apply_group_offload(
*,
stream_text_encoders: bool = False,
stream_transformer: bool = True,
+ background_pin: Optional[bool] = None,
) -> bool:
"""Stream the transformer a few blocks at a time via diffusers group offloading, keeping the
smaller components resident. Returns False (caller falls back to whole-module) on any failure.
@@ -1723,6 +2054,18 @@ def _apply_group_offload(
logger,
)
gkwargs["low_cpu_mem_usage"] = not pin_streamed[0]
+ # ``background_pin``: a module the plan pins is applied unpinned and handed to a _GroupPinner, which the caller
+ # starts with start_background_pins once the load has committed.
+ if background_pin is None:
+ background_pin = bool(getattr(pipe, _BACKGROUND_PIN_REQUEST_ATTR, False))
+ defer = (
+ background_pin
+ and use_stream
+ and "low_cpu_mem_usage" in _params
+ and _background_pin_enabled()
+ )
+ if defer:
+ install_group_pin_wait()
# Place the smaller components resident BEFORE attaching the transformer group-offload hooks: a companion .to()
# OOM then returns False with no hooks installed, and diffusers rejects enable_model_cpu_offload once group
# hooks exist.
@@ -1732,7 +2075,11 @@ def _apply_group_offload(
if isinstance(comp, torch.nn.Module):
comp.to(onload)
for module in streamed.values():
- apply_group_offloading(module, **gkwargs)
+ if defer and not gkwargs.get("low_cpu_mem_usage", False):
+ apply_group_offloading(module, **{**gkwargs, "low_cpu_mem_usage": True})
+ _defer_pinning(pipe, module, onload, logger)
+ else:
+ apply_group_offloading(module, **gkwargs)
installed += 1
# The encoders come AFTER the DiTs and are applied one by one, each failure absorbed. A text encoder is a far
# less well-trodden target for block-level group offloading than a DiT (a family whose encoder exposes no
@@ -1752,7 +2099,11 @@ def _apply_group_offload(
transformer_demoted = False
for name, module in streamed_encoders.items():
try:
- apply_group_offloading(module, **ekwargs)
+ if defer and not ekwargs.get("low_cpu_mem_usage", False):
+ apply_group_offloading(module, **{**ekwargs, "low_cpu_mem_usage": True})
+ _defer_pinning(pipe, module, onload, logger)
+ else:
+ apply_group_offloading(module, **ekwargs)
installed += 1
_pin_vision_embedding_device(module)
except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load
@@ -1786,6 +2137,10 @@ def _apply_group_offload(
name,
exc,
)
+ # a leaf-level apply can raise after hooking part of the encoder; resident means no hooks at all, or
+ # the applied VRAM floor reads the whole encoder as streamed while its unhooked layers stay on the card
+ _remove_group_offload_hooks(module)
+ _drop_deferred_pinning(pipe, module)
module.to(onload)
return True
except Exception as exc: # noqa: BLE001 - fall back to whole-module offload
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 447354c3b3c..0bd4e6d9367 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -87,8 +87,14 @@ from .diffusion_device import (
resolve_selected_cuda_ordinal,
)
from .diffusion_memory import (
+ DEFAULT_BASE_OVERHEAD_MIB,
+ DEFAULT_GROUP_BLOCKS,
+ MEMORY_MODE_AUTO,
MEMORY_MODE_FAST,
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
OFFLOAD_NONE,
+ OFFLOAD_STREAMING,
apply_memory_plan,
estimate_gguf_resident_mib,
estimate_safetensors_dense_mib,
@@ -96,12 +102,18 @@ from .diffusion_memory import (
file_size_mib,
normalize_memory_mode,
plan_diffusion_memory,
+ plan_keeps_transformer_resident,
raise_on_unified_memory_shortfall,
reclaim_host_memory,
reclaim_offload_host_memory,
+ refine_memory_plan_for_components,
+ finish_background_pins,
release_pinned_host_memory,
+ request_background_pins,
safetensors_prefix_mib,
settled_snapshot_device_memory,
+ start_background_pins,
+ stop_background_pins,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
from .media_decode_phase import decode_phase as _decode_phase
@@ -340,7 +352,7 @@ def _memory_mode_resolved(memory_mode: Any, plan: Any, offload_policy: str) -> t
return (
memory_mode,
plan.requested_mode,
- f"planned '{plan.offload_policy}' offload from the family size table",
+ f"planned '{plan.offload_policy}' offload ({_video_plan_label(plan)}) from the family size table",
)
@@ -994,6 +1006,10 @@ class _VideoLoadState:
denoiser_pinned: bool = False
# MiniMax-H3: streamed denoiser, so the VRAM floor is the largest single phase, not a sum.
denoiser_streamed: bool = False
+ # MiB the offload tier holds on the device at once; None when nothing is offloaded.
+ vram_floor_mib: Optional[int] = None
+ # MiB still on the device during the VAE decode (the streamed DiT has offloaded by then).
+ vram_decode_floor_mib: Optional[int] = None
# MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.
denoiser_host_copy: bool = False
resolved: Optional[dict] = None
@@ -1515,7 +1531,7 @@ def _video_seed_stays_resident(
base_repo: Optional[str],
reclaimable_mib: int = 0,
) -> bool:
- """True when an artifact-sized plan keeps the denoiser resident; torchao rejects offload."""
+ """True when an artifact-sized plan keeps the denoiser resident (text encoders may stream)."""
components = getattr(fam, "bf16_components_gb", None)
if not components:
return True
@@ -1543,10 +1559,9 @@ def _video_seed_stays_resident(
)
vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scale
mib_per_gb = 1000.0**3 / (1024.0 * 1024.0)
- model_dense_mib = int(
- (denoiser_gb + components[1] * te_scale * dtype_scale + components[2] * vae_scale)
- * mib_per_gb
- )
+ text_encoder_gb = components[1] * te_scale * dtype_scale
+ companions_gb = text_encoder_gb + components[2] * vae_scale
+ model_dense_mib = int((denoiser_gb + companions_gb) * mib_per_gb)
device_memory = settled_snapshot_device_memory(target)
if reclaimable_mib > 0 and device_memory.free_mib is not None:
free = device_memory.free_mib + int(reclaimable_mib)
@@ -1562,10 +1577,442 @@ def _video_seed_stays_resident(
height = fam.resolution_presets[0][1],
num_frames = fam.default_num_frames,
),
- companion_dense_mib = None,
+ companion_dense_mib = int(companions_gb * mib_per_gb),
+ text_encoder_dense_mib = int(text_encoder_gb * mib_per_gb),
requested_mode = normalize_memory_mode(memory_mode),
)
- return planned.offload_policy == "none"
+ return plan_keeps_transformer_resident(planned)
+
+
+def _render_under_no_grad(state: Any) -> bool:
+ """Render under no_grad: torchao weight subclasses fail an offload hook's in-place swap under inference_mode."""
+ if not getattr(state, "transformer_quant", None):
+ return False
+ if getattr(state, "denoiser_streamed", False):
+ return True
+ family = getattr(state, "family", None)
+ return not getattr(family, "modular_workflow", None) and (
+ getattr(state, "offload_policy", OFFLOAD_NONE) != OFFLOAD_NONE
+ )
+
+
+# Staging headroom: base overhead plus one linear's bf16, rotated and int8 copies, as a multiple of the largest.
+_STAGE_LINEAR_WORKSPACE = 3
+
+
+def _stage_units(module: Any) -> list:
+ import torch
+
+ units: list = []
+ for child in module.children():
+ if isinstance(child, torch.nn.ModuleList):
+ units.extend(child)
+ else:
+ units.append(child)
+ return units
+
+
+def _module_dense_bytes(module: Any) -> int:
+ from itertools import chain
+ return sum(
+ int(t.numel()) * int(t.element_size()) for t in chain(module.parameters(), module.buffers())
+ )
+
+
+def _stage_denoiser_for_quant(
+ transformer: Any,
+ target: Any,
+ *,
+ logger: Any = None,
+) -> list:
+ """Stage a CPU DiT onto ``target``'s CUDA device block by block as far as it fits; quantisers work per linear,
+ so a partial fit gives the same result. Returns the moved modules; never raises."""
+ try:
+ import torch
+
+ if (
+ transformer is None
+ or getattr(target, "device", None) != "cuda"
+ or not torch.cuda.is_available()
+ ):
+ return []
+ first = next(transformer.parameters(), None)
+ if first is None or first.device.type != "cpu":
+ return []
+ device = torch.device(getattr(target, "torch_device", None) or "cuda")
+ from utils.hardware import trusted_mem_get_info
+
+ free_bytes, _ = trusted_mem_get_info(device, module = torch.cuda)
+ largest_linear = max(
+ (
+ _module_dense_bytes(m)
+ for m in transformer.modules()
+ if isinstance(m, torch.nn.Linear)
+ ),
+ default = 0,
+ )
+ budget = (
+ int(free_bytes)
+ - DEFAULT_BASE_OVERHEAD_MIB * 1024 * 1024
+ - _STAGE_LINEAR_WORKSPACE * largest_linear
+ )
+ staged: list = []
+ moved_bytes = 0
+ try:
+ for unit in _stage_units(transformer):
+ size = _module_dense_bytes(unit)
+ if size == 0 or moved_bytes + size > budget:
+ continue
+ unit.to(device)
+ staged.append(unit)
+ moved_bytes += size
+ except Exception:
+ # The DiT started on the host: put every moved unit, and a half-moved one, back before the CPU fallback.
+ transformer.to("cpu")
+ clear_gpu_cache()
+ raise
+ if staged and logger is not None:
+ logger.info(
+ "video.transformer_quant: quantising on %s with %.1f of %.1f GB of the DiT staged there",
+ device,
+ moved_bytes / 1e9,
+ _module_dense_bytes(transformer) / 1e9,
+ )
+ return staged
+ except Exception as exc: # noqa: BLE001 -- quantising on the host is slower, never wrong
+ if logger is not None:
+ logger.warning("video.transformer_quant: staging on the card failed (%s)", exc)
+ return []
+
+
+def _unstage_modules(modules: list) -> None:
+ for module in modules:
+ module.to("cpu")
+ clear_gpu_cache()
+
+
+def _auto_offload_scheme(requested: Optional[str]) -> Optional[str]:
+ """auto takes int8 for an offloaded DiT: its torchao-free build rides every offload hook."""
+ scheme = normalize_transformer_quant(requested)
+ return TQ_INT8 if scheme == TQ_AUTO else scheme
+
+
+# Denoise-side base of estimate_video_runtime_mib; the rest scales with the decoded clip, held only by the VAE.
+_VIDEO_DENOISE_ACTIVATION_MIB = 4096
+
+
+def _video_prefer_whole_module(
+ plan: Any,
+ *,
+ denoiser_mib: Optional[int],
+ text_encoder_mib: Optional[int],
+ vae_mib: Optional[int],
+ runtime_mib: int,
+) -> Any:
+ """Swap a streamed-DiT plan for whole-module offload when every component fits whole, one at a time.
+
+ Wan2.2-5B at 16 GiB: 0.14-0.28 s per forward whole-module vs 0.28-0.33 s int8 streamed, so streaming only wins
+ very short calls. ``denoiser_mib`` is one expert of a dual-DiT family. An explicit memory mode is left as asked."""
+ if (
+ getattr(plan, "offload_policy", None) != OFFLOAD_GROUP
+ or not bool(getattr(plan, "stream_transformer", True))
+ or getattr(plan, "requested_mode", None) != MEMORY_MODE_AUTO
+ or denoiser_mib is None
+ or text_encoder_mib is None
+ or vae_mib is None
+ ):
+ return plan
+ budget = (getattr(plan, "estimates", None) or {}).get("safe_device_budget_mib")
+ if budget is None:
+ return plan
+ phases = (
+ int(text_encoder_mib) + DEFAULT_BASE_OVERHEAD_MIB,
+ int(denoiser_mib)
+ + min(int(runtime_mib), _VIDEO_DENOISE_ACTIVATION_MIB)
+ + DEFAULT_BASE_OVERHEAD_MIB,
+ # the decode holds only the decoded-clip share; the denoise base is already in the DiT phase
+ int(vae_mib)
+ + max(0, int(runtime_mib) - _VIDEO_DENOISE_ACTIVATION_MIB)
+ + DEFAULT_BASE_OVERHEAD_MIB,
+ )
+ if max(phases) > int(budget):
+ return plan
+ return dataclasses.replace(
+ plan,
+ estimates = {**(plan.estimates or {}), "whole_module_fits_mib": max(phases)},
+ offload_policy = OFFLOAD_MODEL,
+ stream_text_encoders = False,
+ stream_transformer = True,
+ vae_tiling = True,
+ vae_slicing = True,
+ reasons = tuple(plan.reasons)
+ + (
+ f"every component fits the card whole (largest phase {max(phases)} MiB of {int(budget)}): "
+ "whole-module offload pages the DiT once per call instead of streaming it every step",
+ ),
+ )
+
+
+def _video_plan_rank(plan: Any) -> int:
+ """Video speed rank, fastest first (0..6); see the tier comments in the tests."""
+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
+ if policy == OFFLOAD_NONE:
+ return 0
+ if policy == OFFLOAD_GROUP:
+ if not bool(getattr(plan, "stream_transformer", True)):
+ return 1
+ return 4 if bool(getattr(plan, "stream_text_encoders", False)) else 3
+ if policy == OFFLOAD_MODEL:
+ fits = (getattr(plan, "estimates", None) or {}).get("whole_module_fits_mib") is not None
+ return 2 if fits else 5
+ return 6
+
+
+def _video_plan_label(plan: Any) -> str:
+ return (
+ "every component resident",
+ "denoiser resident, text encoders streamed",
+ "whole-module CPU offload, each component fits whole",
+ "denoiser streamed block by block, companions resident",
+ "denoiser and text encoders streamed",
+ "whole-module CPU offload",
+ "denoiser blocks and text-encoder layers streamed",
+ )[_video_plan_rank(plan)]
+
+
+_VIDEO_DENOISER_ATTRS = ("transformer", "transformer_2", "unconditional_transformer")
+
+
+def _video_streamed_peak_bytes(module: Any, offload_type: str, *, prefetch: bool, size: Any) -> int:
+ """Bytes a group-offload hook holds on the device at once for ``module`` (diffusers' own grouping)."""
+ from itertools import chain
+
+ import torch
+
+ try:
+ from diffusers.hooks._common import _GO_LC_SUPPORTED_PYTORCH_LAYERS as leaf_types
+ except Exception: # noqa: BLE001 -- same layer set diffusers streams leaf by leaf
+ nn = torch.nn
+ leaf_types = (
+ nn.Conv1d,
+ nn.Conv2d,
+ nn.Conv3d,
+ nn.ConvTranspose1d,
+ nn.ConvTranspose2d,
+ nn.ConvTranspose3d,
+ nn.Linear,
+ nn.Embedding,
+ )
+
+ counted: set[int] = set()
+
+ def _own(mod: Any, recurse: bool) -> int:
+ # a tied weight (shared embedding) moves once, so count it for its first group only
+ total = 0
+ for t in chain(mod.parameters(recurse = recurse), mod.buffers(recurse = recurse)):
+ if id(t) not in counted:
+ counted.add(id(t))
+ total += size(t)
+ return total
+
+ if offload_type == "block_level":
+ windows: list[int] = []
+ grouped: set[int] = set()
+ for child in module.children():
+ if isinstance(child, (torch.nn.ModuleList, torch.nn.Sequential)):
+ blocks = list(child)
+ for i in range(0, len(blocks), DEFAULT_GROUP_BLOCKS):
+ windows.append(sum(_own(b, True) for b in blocks[i : i + DEFAULT_GROUP_BLOCKS]))
+ grouped |= {id(t) for t in chain(child.parameters(), child.buffers())}
+ # the unmatched group (embeddings, heads, loose params) stays onloaded for the whole forward
+ unmatched = sum(
+ size(t)
+ for t in {id(t): t for t in chain(module.parameters(), module.buffers())}.values()
+ if id(t) not in grouped and id(t) not in counted
+ )
+ units, fixed = windows, unmatched
+ else:
+ # (size, enclosing ancestors' loose bytes, ids of this group and its ancestors) per onload group
+ groups: list[tuple[int, int, frozenset]] = []
+ visited: set[int] = set()
+
+ def _walk(mod: Any, enclosing: int, chain: frozenset) -> None:
+ if id(mod) in visited:
+ return
+ visited.add(id(mod))
+ if isinstance(mod, leaf_types):
+ groups.append((_own(mod, True), enclosing, chain | {id(mod)}))
+ elif next(mod.children(), None) is not None:
+ # a parent's loose params onload in its pre-forward and stay until its post-forward, under every
+ # descendant; a sibling branch has offloaded its own by then, so only the enclosing chain stacks
+ loose = _own(mod, False)
+ here = chain | {id(mod)}
+ groups.append((loose, enclosing, here))
+ for child in mod.children():
+ _walk(child, enclosing + loose, here)
+ else:
+ # loose params of a childless non-leaf (a norm) form their own group
+ groups.append((_own(mod, False), enclosing, chain | {id(mod)}))
+
+ _walk(module, 0, frozenset())
+ ranked = sorted(range(len(groups)), key = lambda i: groups[i][0], reverse = True)
+ peak = 0
+ for size, enclosing, here in groups:
+ # a copy stream prefetches the next group while this one computes: any group not already held
+ other = 0
+ if prefetch:
+ for i in ranked:
+ if not (groups[i][2] <= here):
+ other = groups[i][0]
+ break
+ peak = max(peak, enclosing + size + other)
+ return peak
+ units = sorted(units, reverse = True)
+ # a copy stream prefetches the next group while the current one computes
+ return fixed + sum(units[: 2 if prefetch else 1])
+
+
+def _video_group_hooked(module: Any) -> bool:
+ """Whether group offloading actually hooked ``module`` (leaf level hooks its leaves). Unreadable counts as hooked."""
+ try:
+ for sub in module.modules():
+ hooks = getattr(getattr(sub, "_diffusers_hook", None), "hooks", None) or {}
+ if any("group_offloading" in str(key) for key in hooks):
+ return True
+ except Exception: # noqa: BLE001 -- trust the plan
+ return True
+ return False
+
+
+def _video_offload_vram_floor_mib(
+ pipe: Any,
+ plan: Any,
+ *,
+ applied: bool = False,
+ phase: str = "peak",
+) -> Optional[int]:
+ """MiB of weights co-resident on the device under ``plan``'s tier, from the loaded modules; None if not offloaded.
+ ``applied`` reads which modules the hooks actually stream: an encoder that refused leaf offload stays resident.
+ ``phase="decode"`` counts only what stays on the device during the VAE decode, after the DiT has offloaded."""
+ policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
+ if policy == OFFLOAD_NONE:
+ return None
+ try:
+ from itertools import chain
+
+ import torch
+
+ from .diffusion_prequant import tensor_payload_bytes
+
+ components = getattr(pipe, "components", None)
+ if not isinstance(components, dict):
+ return None
+ sizes: dict[str, int] = {}
+ modules: dict[str, Any] = {}
+ seen: set[int] = set()
+ for name, module in components.items():
+ if not isinstance(module, torch.nn.Module):
+ continue
+ total = 0
+ for tensor in chain(module.parameters(), module.buffers()):
+ if id(tensor) in seen or getattr(tensor, "is_meta", False):
+ continue
+ seen.add(id(tensor))
+ total += tensor_payload_bytes(tensor)
+ sizes[str(name)] = total
+ modules[str(name)] = module
+ if not sizes:
+ return None
+ denoisers = {n for n in sizes if n in _VIDEO_DENOISER_ATTRS}
+ encoders = {n for n in sizes if n.startswith("text_encoder")}
+ if policy == OFFLOAD_MODEL:
+ if phase == "decode":
+ # whole-module offload onloads one component at a time; the decode runs the decoders alone
+ decoders = [
+ size for name, size in sizes.items() if name not in denoisers | encoders
+ ]
+ return int(max(decoders, default = 0) // (1024 * 1024))
+ return int(max(sizes.values()) // (1024 * 1024))
+ if policy == OFFLOAD_GROUP:
+ streamed = (
+ set() if not bool(getattr(plan, "stream_transformer", True)) else set(denoisers)
+ )
+ if bool(getattr(plan, "stream_text_encoders", False)):
+ streamed |= encoders
+ elif policy == OFFLOAD_STREAMING:
+ streamed = denoisers | encoders
+ else:
+ return None
+ if applied:
+ # From the hooks alone: a refusing encoder stays resident, and the fallback may hook a DiT the plan kept.
+ streamed = {name for name in denoisers | encoders if _video_group_hooked(modules[name])}
+ held = sum(size for name, size in sizes.items() if name not in streamed)
+ if phase == "decode":
+ # streamed groups have all offloaded by the decode; only the residents remain
+ return int(held // (1024 * 1024))
+ backend = getattr(getattr(plan, "device_memory", None), "backend", None)
+ prefetch = backend not in ("mps", "cpu")
+
+ def _size(tensor: Any) -> int:
+ return 0 if getattr(tensor, "is_meta", False) else tensor_payload_bytes(tensor)
+
+ # streamed modules run one after another, so only the largest onloaded unit joins the residents
+ held += max(
+ (
+ _video_streamed_peak_bytes(
+ modules[name],
+ "block_level" if name in denoisers else "leaf_level",
+ prefetch = prefetch,
+ size = _size,
+ )
+ for name in streamed
+ ),
+ default = 0,
+ )
+ return int(held // (1024 * 1024))
+ except Exception: # noqa: BLE001 -- an unreadable pipeline keeps today's unchecked behaviour
+ return None
+
+
+def video_offload_shortfall_message(
+ *,
+ family: str,
+ floor_mib: int,
+ available_mib: int,
+ placement: str,
+ width: Optional[int] = None,
+ height: Optional[int] = None,
+ frames: Optional[int] = None,
+ decode_floor_mib: Optional[int] = None,
+) -> Optional[str]:
+ """Refusal when co-resident weights plus the activation headroom exceed the device, else None.
+ Denoise and VAE decode are separate phases: the decoded-clip share is held only beside the weights still on the
+ device during the decode (``decode_floor_mib``, the full floor when unknown), never beside the streamed DiT."""
+ headroom = DEFAULT_BASE_OVERHEAD_MIB
+ required = int(floor_mib) + headroom
+ if width and height and frames:
+ # decoded-clip share of the runtime estimate; its fixed 4 GiB denoise base over-refuses measured cells
+ clip_mib = max(
+ 0,
+ estimate_video_runtime_mib(width = width, height = height, num_frames = frames)
+ - _VIDEO_DENOISE_ACTIVATION_MIB,
+ )
+ decode_floor = int(
+ floor_mib if decode_floor_mib is None else min(decode_floor_mib, floor_mib)
+ )
+ if decode_floor + headroom + clip_mib > required:
+ floor_mib, headroom = decode_floor, headroom + clip_mib
+ required = decode_floor + headroom
+ if int(available_mib) >= required:
+ return None
+ shape = f" for {width}x{height} at {frames} frames" if width and height and frames else ""
+ return (
+ f"{family} needs about {required / 1024:.1f} GiB of available VRAM{shape} under this load's "
+ f"placement ({placement}: {floor_mib / 1024:.1f} GiB of weights on the GPU at once plus "
+ f"{headroom / 1024:.1f} GiB for activations); {available_mib / 1024:.1f} GiB is "
+ "available. Free VRAM held by other processes, set the text encoder or DiT precision lower, "
+ "or pick a smaller model."
+ )
def _progress(phase: Optional[str], **extra: Any) -> dict[str, Any]:
@@ -4884,7 +5331,6 @@ class VideoBackend:
vae_scale = 1.0 if getattr(fam, "vae_force_fp32", False) else dtype_scale
scaled_te_gb = text_encoder_gb * dtype_scale
scaled_vae_gb = vae_gb * vae_scale
- companions_gb = text_encoder_gb + vae_gb if components is not None else 0.0
scaled_companions_gb = scaled_te_gb + scaled_vae_gb if components is not None else 0.0
if log and scale != 1.0 and components is not None:
logger.info(
@@ -4895,6 +5341,11 @@ class VideoBackend:
text_encoder_gb,
components[1],
)
+ # Without the text-encoder split the planner can only pick all-resident or whole-module offload.
+ companion_mib = (
+ int(scaled_companions_gb * mib_per_gb) if components is not None else None
+ )
+ text_encoder_mib = int(scaled_te_gb * mib_per_gb) if components is not None else None
if kind == "pipeline":
transformer_gb = (
components[0] * dtype_scale if denoiser_gb is None else float(denoiser_gb)
@@ -4904,62 +5355,106 @@ class VideoBackend:
if components is not None
else None
)
- companion_mib = None
else:
- companion_mib = (
- int(scaled_companions_gb * mib_per_gb) if components is not None else None
- )
# Budget ALL weights: companions stay resident, so budgeting the transformer alone lets auto pick
# OFFLOAD_NONE and OOM.
model_dense_mib = (
transformer_mib + (companion_mib or 0) if transformer_mib is not None else None
)
- planned = plan_diffusion_memory(
- target = target,
- device_memory = device_memory,
- model_dense_mib = model_dense_mib,
- runtime_headroom_mib = runtime_mib,
- companion_dense_mib = companion_mib,
- requested_mode = normalize_memory_mode(memory_mode),
+ experts = 2 if getattr(fam, "is_moe", False) else 1
+ vae_mib = int(scaled_vae_gb * mib_per_gb) if components is not None else None
+ dit_mib = (
+ (int(transformer_gb * mib_per_gb) if components is not None else None)
+ if kind == "pipeline"
+ else transformer_mib
+ )
+ planned = _video_prefer_whole_module(
+ plan_diffusion_memory(
+ target = target,
+ device_memory = device_memory,
+ model_dense_mib = model_dense_mib,
+ runtime_headroom_mib = runtime_mib,
+ companion_dense_mib = companion_mib,
+ text_encoder_dense_mib = text_encoder_mib,
+ requested_mode = normalize_memory_mode(memory_mode),
+ ),
+ denoiser_mib = None if dit_mib is None else dit_mib // experts,
+ text_encoder_mib = text_encoder_mib,
+ vae_mib = vae_mib,
+ runtime_mib = runtime_mib,
)
- # Parity with the image dense-quant path: the bf16-table plan can force offload a quantised DiT would not
- # need, so re-plan with the scheme factor and keep resident if it fits.
+ # Re-plan with the scheme factor: the bf16-table plan can offload a DiT the quantised one would not.
dense_plan = planned
replanned_for_quant = False
+ requested_scheme = normalize_transformer_quant(transformer_quant)
+ # auto keeps a resident bf16 DiT: quantising costs accuracy for little speed.
+ worth_replanning = (
+ not plan_keeps_transformer_resident(planned)
+ if requested_scheme == TQ_AUTO
+ else planned.offload_policy != OFFLOAD_NONE
+ )
# Not for a unified-memory shortfall: runtime quant still peaks at the dense bf16 DiT.
if (
kind == "pipeline"
and denoiser_gb is None
- and planned.offload_policy != "none"
- and normalize_transformer_quant(transformer_quant) is not None
+ and worth_replanning
+ and requested_scheme is not None
and (
dense_transformer_supported(target)
or native_quant_scheme(target, transformer_quant, family = fam.name) is not None
+ or native_quant_scheme(
+ target,
+ _auto_offload_scheme(transformer_quant),
+ family = fam.name,
+ offload = True,
+ )
+ is not None
)
and components is not None
):
- scheme_preview = native_quant_scheme(
- target, transformer_quant, family = fam.name
- ) or select_transformer_quant_scheme(target, transformer_quant, family = fam.name)
+ scheme_preview = (
+ native_quant_scheme(target, transformer_quant, family = fam.name)
+ or (
+ select_transformer_quant_scheme(target, transformer_quant, family = fam.name)
+ if dense_transformer_supported(target)
+ else None
+ )
+ or native_quant_scheme(
+ target,
+ _auto_offload_scheme(transformer_quant),
+ family = fam.name,
+ offload = True,
+ )
+ )
factor = _QUANT_STEADY_FACTOR.get(scheme_preview) if scheme_preview else None
if factor is not None:
- quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)
- replanned = plan_diffusion_memory(
- target = target,
- device_memory = device_memory,
- model_dense_mib = quant_mib,
- runtime_headroom_mib = runtime_mib,
- companion_dense_mib = None,
- requested_mode = normalize_memory_mode(memory_mode),
+ # companions priced exactly as the split below (dtype-scaled), or the planner's DiT share
+ # (quant total minus the scaled text encoder) drifts under an fp16 to fp32 promotion
+ quant_mib = int((components[0] * factor + scaled_companions_gb) * mib_per_gb)
+ replanned = _video_prefer_whole_module(
+ plan_diffusion_memory(
+ target = target,
+ device_memory = device_memory,
+ model_dense_mib = quant_mib,
+ runtime_headroom_mib = runtime_mib,
+ companion_dense_mib = companion_mib,
+ text_encoder_dense_mib = text_encoder_mib,
+ requested_mode = normalize_memory_mode(memory_mode),
+ ),
+ denoiser_mib = int(components[0] * factor * mib_per_gb) // experts,
+ text_encoder_mib = text_encoder_mib,
+ vae_mib = vae_mib,
+ runtime_mib = runtime_mib,
)
- if replanned.offload_policy == "none":
+ if _video_plan_rank(replanned) < _video_plan_rank(planned):
if log:
logger.info(
- "video.transformer_quant: %s fits resident (%d MiB steady); "
- "dropping the bf16 plan's '%s' offload",
+ "video.transformer_quant: %s (%d MiB steady) places the DiT as '%s' "
+ "instead of the bf16 plan's '%s'",
scheme_preview,
quant_mib,
- planned.offload_policy,
+ _video_plan_label(replanned),
+ _video_plan_label(planned),
)
planned = replanned
replanned_for_quant = True
@@ -4991,8 +5486,13 @@ class VideoBackend:
plan, bf16_plan, quant_replanned = _plan_for_te_scale(
te_scale, log = True, denoiser_gb = denoiser_seed_gb
)
- if denoiser_seed_scheme is not None and plan.offload_policy != "none":
- # Offload hooks use Module.to(), which torchao tensors reject: re-plan at bf16.
+ if denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):
+ logger.info(
+ "video.denoiser_prequant: the %s denoiser would have to be offloaded ('%s') on this "
+ "card, so the hosted checkpoint is not seeded",
+ denoiser_seed_scheme,
+ _video_plan_label(plan),
+ )
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)
@@ -5052,7 +5552,7 @@ class VideoBackend:
plan, bf16_plan, quant_replanned = _plan_for_te_scale(
settled_te_scale, log = False, denoiser_gb = denoiser_seed_gb
)
- if denoiser_seed_scheme is not None and plan.offload_policy != "none":
+ if denoiser_seed_scheme is not None and not plan_keeps_transformer_resident(plan):
denoiser_seed_scheme = None
denoiser_seed_gb = None
plan, bf16_plan, quant_replanned = _plan_for_te_scale(settled_te_scale, log = False)
@@ -5142,7 +5642,7 @@ class VideoBackend:
ltx23_override = None
ltx23_scheme = normalize_transformer_quant(transformer_quant)
seeded = None
- if ltx23_prequant_pick and plan.offload_policy == "none":
+ if ltx23_prequant_pick and plan_keeps_transformer_resident(plan):
from .video_ltx2 import load_ltx23_prequant_transformer
seeded = load_ltx23_prequant_transformer(
fam,
@@ -5164,9 +5664,10 @@ class VideoBackend:
elif ltx23_prequant_pick:
# Decline before the dense assembly loads the 44 GB bf16 DiT only to refuse it.
ltx23_reason = (
- f"this GPU's '{plan.offload_policy}' memory plan offloads the DiT even at fp8 size, and "
- "torchao quantised tensors cannot be moved by the offload hooks"
- if plan.offload_policy != "none"
+ f"this GPU's memory plan ({_video_plan_label(plan)}) offloads the DiT even at fp8 "
+ "size, and the hosted fp8 DiT is kept resident only (the text encoders may stream "
+ "around it)"
+ if not plan_keeps_transformer_resident(plan)
else "the hosted fp8 DiT (unsloth/LTX-2.3-FP8) could not be loaded"
+ (" from the local cache" if local_files_only else "")
)
@@ -5245,10 +5746,16 @@ class VideoBackend:
transformer_quant_decline: Optional[str] = None
transformer_quant_decline_status = RESOLVED_FELL_BACK
transformer_quant_source: Optional[str] = None
- video_offload = plan.offload_policy != "none"
+ # Text encoders streaming around a resident DiT do not move its weights, so torchao stays valid.
+ video_offload = not plan_keeps_transformer_resident(plan)
native_scheme = (
native_quant_scheme(
- target, transformer_quant_pinned, family = fam.name, offload = video_offload
+ target,
+ _auto_offload_scheme(transformer_quant)
+ if video_offload and transformer_quant_pinned is None
+ else transformer_quant_pinned,
+ family = fam.name,
+ offload = video_offload,
)
# A seeded denoiser is a torchao checkpoint; the torchao-free path must not claim it.
if kind == "pipeline" and not denoiser_injected
@@ -5268,7 +5775,7 @@ class VideoBackend:
and normalize_transformer_quant(transformer_quant) == TQ_AUTO
and not quant_replanned
and not denoiser_injected
- and plan.offload_policy == "none"
+ and plan_keeps_transformer_resident(plan)
# An unmeasured budget also plans "none" without proving a fit.
and plan.estimates.get("safe_device_budget_mib") is not None
and plan.estimates.get("resident_required_mib") is not None
@@ -5311,22 +5818,21 @@ class VideoBackend:
kind == "pipeline"
and normalize_transformer_quant(transformer_quant) is not None
and dense_transformer_supported(target)
- and plan.offload_policy != "none"
+ and video_offload
and native_scheme is None
):
- # Offload hooks move modules with Module.to(), which torchao quantized tensors reject. Skip quant
- # (dense-under-offload beats a crash); forceable via a resident mode.
+ # torchao subclasses do not survive offload hooks under inference_mode; int8 has a torchao-free build, others skip.
logger.info(
- "video.transformer_quant: skipped (offload policy '%s' moves the "
- "DiT via Module.to(), unsupported for torchao quantized tensors); "
- "pin a resident memory mode to combine quant with this model",
- plan.offload_policy,
+ "video.transformer_quant: skipped (the memory plan offloads the DiT: %s; only "
+ "int8 runs torchao-free there)",
+ _video_plan_label(plan),
)
quant_skipped_for_offload = True
transformer_quant_decline = (
- f"the memory plan picked '{plan.offload_policy}' offload, which moves the DiT via "
- "Module.to(); torchao quantized tensors reject that. Pin a resident memory mode "
- "(Fast) to combine quant with this model"
+ f"this GPU's memory plan moves the DiT ({_video_plan_label(plan)}), and "
+ f"'{normalize_transformer_quant(transformer_quant)}' is built as torchao weights here, "
+ "which Studio keeps on a resident DiT only; int8 runs torchao-free under offload. Pick "
+ "int8, or a resident memory mode (Fast) on a card that holds the quantised DiT"
)
elif (
kind == "pipeline"
@@ -5351,16 +5857,24 @@ class VideoBackend:
):
engaged = []
for view in views:
- # Pass each expert view so both DiTs quantise with the same scheme. The family name drives
- # _FAMILY_SCHEME_DENY.
- scheme = quantize_transformer(
- view,
- target,
- mode = transformer_quant,
- family = fam.name,
- logger = logger,
- **native_kwargs,
+ # Quantise on the card: on the host int8 packing took 59 s (Wan2.2-5B) to 154 s (LTX-2) of the load.
+ staged = _stage_denoiser_for_quant(
+ getattr(view, "transformer", None), target, logger = logger
)
+ scheme = None
+ try:
+ scheme = quantize_transformer(
+ view,
+ target,
+ mode = native_scheme or transformer_quant,
+ family = fam.name,
+ logger = logger,
+ **native_kwargs,
+ )
+ finally:
+ # A declined quant can roll back to the bf16 offload plan, which must not start with dense blocks staged.
+ if staged and (video_offload or scheme is None):
+ _unstage_modules(staged)
if scheme is not None:
engaged.append(scheme)
# All experts or none: the first is mutated in place, so a second-expert failure cannot fall back to dense.
@@ -5615,6 +6129,26 @@ class VideoBackend:
del pipe
clear_gpu_cache()
raise RuntimeError("Video load was cancelled or superseded.")
+ # A component larger than the card OOMs at its first whole-module onload; fall back to leaf streaming.
+ refined = refine_memory_plan_for_components(pipe, plan)
+ if refined is not plan:
+ logger.info("video.memory: %s", refined.reasons[-1])
+ plan = refined
+ vram_floor_mib = _video_offload_vram_floor_mib(pipe, plan)
+ free_mib = getattr(getattr(plan, "device_memory", None), "free_mib", None)
+ if vram_floor_mib is not None and free_mib is not None:
+ shortfall = video_offload_shortfall_message(
+ family = fam.name,
+ floor_mib = vram_floor_mib,
+ available_mib = int(free_mib),
+ placement = _video_plan_label(plan),
+ )
+ if shortfall is not None:
+ del pipe
+ clear_gpu_cache()
+ raise RuntimeError(shortfall)
+ # the streamed modules pin after the load returns, overlapping the first encode and compile
+ request_background_pins(pipe)
offload_policy, vae_tiling = apply_memory_plan(
pipe,
plan,
@@ -5624,6 +6158,28 @@ class VideoBackend:
placement_device = target.torch_device,
logger = logger,
)
+ if (
+ offload_policy == plan.offload_policy
+ and vram_floor_mib is not None
+ and free_mib is not None
+ ):
+ # An encoder that refused leaf offload is kept resident under the same policy: re-check what landed.
+ applied_floor = _video_offload_vram_floor_mib(pipe, plan, applied = True)
+ raised = applied_floor is not None and applied_floor > vram_floor_mib
+ if applied_floor is not None:
+ # record what landed even when lower (the fallback may stream a DiT the plan kept resident)
+ vram_floor_mib = applied_floor
+ if raised:
+ shortfall = video_offload_shortfall_message(
+ family = fam.name,
+ floor_mib = applied_floor,
+ available_mib = int(free_mib),
+ placement = _video_plan_label(plan),
+ )
+ if shortfall is not None:
+ del pipe
+ clear_gpu_cache()
+ raise RuntimeError(shortfall)
# A dual-DiT MoE needs no extra per-expert pass: apply_memory_plan covers every DiT; a second pass would
# duplicate-hook.
if not vae_tiling:
@@ -5746,6 +6302,21 @@ class VideoBackend:
cache_threshold = transformer_cache_threshold,
transformer_quant = transformer_quant_engaged,
text_encoder_quant = text_encoder_quant_engaged,
+ vram_floor_mib = (
+ vram_floor_mib
+ if offload_policy == plan.offload_policy
+ else _video_offload_vram_floor_mib(
+ pipe, dataclasses.replace(plan, offload_policy = offload_policy)
+ )
+ ),
+ vram_decode_floor_mib = _video_offload_vram_floor_mib(
+ pipe,
+ plan
+ if offload_policy == plan.offload_policy
+ else dataclasses.replace(plan, offload_policy = offload_policy),
+ applied = True,
+ phase = "decode",
+ ),
resolved = resolved,
)
self._precommit_globals = None
@@ -5758,6 +6329,7 @@ class VideoBackend:
effective_speed,
transformer_quant_engaged or "off",
)
+ start_background_pins(pipe)
return self.status()
@staticmethod
@@ -7135,6 +7707,15 @@ class VideoBackend:
# is pinned the un-indexed state.device below -- the H3 memory probe and every torch.Generator --
# resolves to its own default card while the pipeline sits on the selected one.
self._state_device_target(state)
+ # A render racing the post-load pinner ran every later render slower; the wait is usually over.
+ waited = finish_background_pins(getattr(state, "pipe", None), cancel)
+ if cancel.is_set():
+ raise RuntimeError(VIDEO_CANCELLED_MSG)
+ if waited >= 1.0:
+ logger.info(
+ "video.generate: waited %.1f s for the host weights to finish pinning",
+ waited,
+ )
fam = state.family
if _resolved_inputs is None:
first_pil, last_pil, width, height, conditioning = self._resolve_keyframes(
@@ -7281,6 +7862,41 @@ class VideoBackend:
f"system RAM at this VRAM tier; {host_capacity_gb:.1f} GB is "
"available. Load the GGUF artifact instead."
)
+ elif (
+ state.engine == "diffusers"
+ and not fam.modular_workflow
+ and getattr(state, "vram_floor_mib", None) is not None
+ and str(state.device).startswith("cuda")
+ and hasattr(torch, "cuda")
+ and hasattr(torch.cuda, "mem_get_info")
+ ):
+ # Otherwise a card that cannot hold the tier's weights is admitted and dies at the first onload.
+ from utils.hardware import trusted_mem_get_info
+
+ ordinal = getattr(state, "placed_ordinal", None)
+ device_obj = (
+ torch.device("cuda", int(ordinal))
+ if ordinal is not None
+ else torch.device(state.device)
+ )
+ free_bytes, _ = trusted_mem_get_info(device_obj, module = torch.cuda)
+ reserved_bytes = (
+ torch.cuda.memory_reserved(device_obj)
+ if hasattr(torch.cuda, "memory_reserved")
+ else 0
+ )
+ shortfall = video_offload_shortfall_message(
+ family = fam.name,
+ floor_mib = int(state.vram_floor_mib),
+ available_mib = int((free_bytes + reserved_bytes) // (1024 * 1024)),
+ placement = f"'{state.offload_policy}' offload",
+ width = width,
+ height = height,
+ frames = frames,
+ decode_floor_mib = getattr(state, "vram_decode_floor_mib", None),
+ )
+ if shortfall is not None:
+ raise RuntimeError(shortfall)
# MPS seeds from the CPU generator too: diffusers reproduces a Metal seed only that way, and the
# pipelines move the noise to the device themselves.
@@ -7559,11 +8175,13 @@ class VideoBackend:
)
elif state.transformer_cache:
self._reset_step_cache(pipe)
- # torchao's aliasing check fails a streamed weight move under inference_mode.
+ # torchao weights fail a hooked weight move under inference_mode (see _render_under_no_grad).
grad_ctx = (
torch.no_grad()
if state.transformer_quant and getattr(state, "denoiser_streamed", False)
else torch.inference_mode()
+ if not _render_under_no_grad(state)
+ else torch.no_grad()
)
try:
with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:
@@ -8163,6 +8781,8 @@ class VideoBackend:
from . import diffusion_prompt_cache
diffusion_prompt_cache.release(getattr(state, "pipe", None))
+ # A pinner still copying would hold its chunks past release_pinned_host_memory().
+ stop_background_pins(getattr(state, "pipe", None))
# Before clear_gpu_cache(), or the graph pool stays reserved.
diffusion_cuda_graph.uninstall_all(
getattr(getattr(state, "pipe", None), "_unsloth_cuda_graphs", ()) or ()
diff --git a/studio/backend/tests/test_diffusion_memory.py b/studio/backend/tests/test_diffusion_memory.py
index 269af3d55c6..03d8eb01808 100644
--- a/studio/backend/tests/test_diffusion_memory.py
+++ b/studio/backend/tests/test_diffusion_memory.py
@@ -2340,6 +2340,127 @@ def test_the_resident_transformer_tier_pins_its_encoders_when_ram_allows(monkeyp
assert seen["text_encoder_2"]["low_cpu_mem_usage"] is False
+def _background_pin_calls(monkeypatch, sizes, budget, **call_kw):
+ """(low_cpu_mem_usage per module, modules handed to a deferred pinner) for one apply."""
+ import core.inference.diffusion_memory as mem
+
+ usage: dict = {}
+ deferred: list = []
+ pipe, *_ = _stream_te_pipe(monkeypatch)
+
+ def _apply(
+ module,
+ onload_device = None,
+ offload_device = None,
+ offload_type = None,
+ num_blocks_per_group = None,
+ non_blocking = False,
+ use_stream = False,
+ record_stream = False,
+ low_cpu_mem_usage = False,
+ **_,
+ ):
+ usage[module.name] = low_cpu_mem_usage
+
+ _swap_group_offloading(monkeypatch, _apply)
+ monkeypatch.delenv(mem.GROUP_OFFLOAD_PIN_ENV, raising = False)
+ monkeypatch.setattr(mem, "_pin_budget_mib", lambda: budget)
+ monkeypatch.setattr(mem, "_pinned_memory_capped", lambda: False, raising = False)
+ monkeypatch.setattr(mem, "_module_host_mib", lambda m: sizes.get(getattr(m, "name", ""), 0))
+ monkeypatch.setattr(
+ mem,
+ "_defer_pinning",
+ lambda pipe, module, device, logger: deferred.append(module.name) or True,
+ )
+ assert mem._apply_group_offload(pipe, "cuda", logger = None, **call_kw) is True
+ return usage, deferred
+
+
+_BG_SIZES = {"transformer": 7000, "text_encoder": 8000, "text_encoder_2": 500}
+
+
+def test_a_video_load_defers_every_planned_pin_off_the_load_path(monkeypatch):
+ # Pinning at load read every streamed byte from disk first: 125-139 s of LTX-2.3's DiT on Colab. The apply goes
+ # unpinned and a pinner takes the same modules, so what gets pinned is unchanged, only when.
+ usage, deferred = _background_pin_calls(
+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True
+ )
+ assert all(v is True for v in usage.values()), usage
+ assert deferred == ["transformer", "text_encoder", "text_encoder_2"]
+
+
+def test_a_module_the_ram_gate_left_unpinned_gets_no_pinner(monkeypatch):
+ usage, deferred = _background_pin_calls(
+ monkeypatch, _BG_SIZES, 10_000, stream_text_encoders = True, background_pin = True
+ )
+ assert usage == {"transformer": True, "text_encoder": True, "text_encoder_2": True}, usage
+ assert deferred == ["transformer"]
+
+
+def test_a_resident_dit_tier_defers_only_its_encoders(monkeypatch):
+ usage, deferred = _background_pin_calls(
+ monkeypatch,
+ _BG_SIZES,
+ 10_000,
+ stream_text_encoders = True,
+ stream_transformer = False,
+ background_pin = True,
+ )
+ assert "transformer" not in usage
+ assert deferred == ["text_encoder", "text_encoder_2"]
+
+
+def test_without_background_pin_the_apply_still_pins_eagerly(monkeypatch):
+ usage, deferred = _background_pin_calls(
+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True
+ )
+ assert all(v is False for v in usage.values()), usage
+ assert deferred == []
+
+
+def test_the_background_pin_env_restores_eager_pinning(monkeypatch):
+ import core.inference.diffusion_memory as mem
+
+ monkeypatch.setenv(mem.BACKGROUND_PIN_ENV, "0")
+ usage, deferred = _background_pin_calls(
+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True, background_pin = True
+ )
+ assert all(v is False for v in usage.values()), usage
+ assert deferred == []
+
+
+def test_a_pipe_request_turns_background_pinning_on(monkeypatch):
+ # video asks through the pipe, so apply_memory_plan (and every stub of it) keeps its signature
+ import core.inference.diffusion_memory as mem
+
+ usage, deferred = _background_pin_calls(
+ monkeypatch, _BG_SIZES, 40_000, stream_text_encoders = True
+ )
+ assert deferred == []
+ import sys
+
+ pipe, *_ = _stream_te_pipe(monkeypatch)
+ seen: dict = {}
+
+ def _apply(
+ module,
+ low_cpu_mem_usage = False,
+ use_stream = False,
+ **_,
+ ):
+ seen[module.name] = low_cpu_mem_usage
+
+ _swap_group_offloading(monkeypatch, _apply)
+ monkeypatch.setattr(
+ mem,
+ "_defer_pinning",
+ lambda pipe, module, device, logger: deferred.append(module.name) or True,
+ )
+ mem.request_background_pins(pipe)
+ assert mem._apply_group_offload(pipe, "cuda", logger = None, stream_text_encoders = True) is True
+ assert "transformer" in deferred, deferred
+
+
@pytest.mark.parametrize("stream_transformer", [True, False])
def test_windows_and_wsl_never_pin_the_streamed_tiers(monkeypatch, stream_transformer):
import core.inference.diffusion_memory as mem
@@ -2390,6 +2511,30 @@ def test_a_first_encoder_refusal_is_unhooked_before_whole_module_offload(monkeyp
assert unhooked == ["text_encoder"]
+def test_an_encoder_kept_resident_after_a_partial_leaf_apply_is_unhooked(monkeypatch):
+ """A refusal that raised after hooking part of the encoder must not leave those hooks on the resident encoder."""
+ import core.inference.diffusion_memory as mem
+
+ def _apply(module, **kw):
+ if getattr(module, "name", "") == "text_encoder":
+ raise RuntimeError("leaf offload failed part way")
+
+ pipe, _unused, transformer, te, te2, vae = _stream_te_pipe(monkeypatch)
+ _swap_group_offloading(monkeypatch, _apply)
+ unhooked: list[str] = []
+ monkeypatch.setattr(
+ mem, "_remove_group_offload_hooks", lambda m: unhooked.append(m.name), raising = False
+ )
+
+ assert (
+ mem._apply_group_offload(
+ pipe, "cuda", logger = None, stream_text_encoders = True, stream_transformer = True
+ )
+ is True
+ )
+ assert unhooked == ["text_encoder"]
+
+
def test_pinned_memory_cap_detection(monkeypatch, tmp_path):
import builtins
diff --git a/studio/backend/tests/test_diffusion_offload_keep_cpu.py b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
index af236c07451..7bef7fecddd 100644
--- a/studio/backend/tests/test_diffusion_offload_keep_cpu.py
+++ b/studio/backend/tests/test_diffusion_offload_keep_cpu.py
@@ -400,3 +400,75 @@ def test_the_ram_gate_is_sized_from_the_container(
monkeypatch.setattr(torch, "empty", empty)
assert dm._pin_host_weights(module, {"weight": data}) == 0
assert bool(allocated) is pinned
+
+
+class _BufferLinear(torch.nn.Module):
+ """A weight held as plain buffers, as the torchao-free int8 linear stores it."""
+
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.register_buffer("weight_q", torch.randint(-127, 127, (8, 8), dtype = torch.int8))
+ self.register_buffer("weight_scale", torch.rand(8))
+ self.bias = torch.nn.Parameter(torch.zeros(8))
+
+ def forward(self, x):
+ return x @ (self.weight_q.float() * self.weight_scale[:, None]).t() + self.bias
+
+
+def _buffer_pipe(device):
+ pipe = _pipe(device, "transformer")
+ pipe.components["transformer"] = pipe.transformer = _BufferLinear()
+ return pipe
+
+
+def _buffer_ptrs(module):
+ return [b.data_ptr() for b in module.buffers()]
+
+
+@cuda
[email protected]("pin", ["0", "1"])
+def test_buffer_weights_keep_their_host_storage_and_are_pinned(monkeypatch, pin):
+ monkeypatch.setenv(dm.OFFLOAD_PIN_ENV, pin)
+ x = torch.randn(4, 8)
+ stock = _buffer_pipe("cuda")
+ stock.enable_model_cpu_offload()
+ ref = _render(stock, x)
+ pipe = _buffer_pipe("cuda")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ assert torch.equal(_render(pipe, x), ref)
+ host = _buffer_ptrs(pipe.transformer)
+ for _ in range(3):
+ assert torch.equal(_render(pipe, x), ref)
+ assert _buffer_ptrs(pipe.transformer) == host
+ assert all(b.is_pinned() == (pin == "1") for b in pipe.transformer.buffers())
+
+
+@cuda
+def test_a_buffer_written_on_the_device_is_copied_back():
+ pipe = _buffer_pipe("cuda")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ before = pipe.transformer.weight_scale.clone()
+ pipe.transformer(torch.ones(1, 8))
+ pipe.transformer.weight_scale.add_(1)
+ pipe.transformer._hf_hook.init_hook(pipe.transformer)
+ assert pipe.transformer.weight_scale.device.type == "cpu"
+ assert torch.equal(pipe.transformer.weight_scale, before + 1)
+
+
+@cuda
+def test_buffers_onloaded_under_inference_mode_do_not_break_the_hook():
+ """A buffer moved under inference_mode is an inference tensor with no version counter; it takes the stock copy."""
+ x = torch.randn(4, 8)
+ stock = _buffer_pipe("cuda")
+ stock.enable_model_cpu_offload()
+ with torch.inference_mode():
+ ref = _render(stock, x)
+ pipe = _buffer_pipe("cuda")
+ pipe.enable_model_cpu_offload()
+ dm.keep_cpu_weights_on_offload(pipe)
+ for _ in range(3):
+ with torch.inference_mode():
+ assert torch.equal(_render(pipe, x), ref)
diff --git a/studio/backend/tests/test_group_offload_background_pin.py b/studio/backend/tests/test_group_offload_background_pin.py
new file mode 100644
index 00000000000..d14a3ae49ac
--- /dev/null
+++ b/studio/backend/tests/test_group_offload_background_pin.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Background pinning of diffusers stream group offload, on a real card."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("diffusers")
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs a CUDA device")
+
+
+class _Pipe:
+ pass
+
+
+def _net():
+ torch.manual_seed(0)
+
+ class Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList(
+ torch.nn.Sequential(
+ torch.nn.Linear(256, 384), torch.nn.GELU(), torch.nn.Linear(384, 256)
+ )
+ for _ in range(6)
+ )
+ self.register_buffer("gain", torch.full((256,), 0.5))
+
+ def forward(self, x):
+ for block in self.blocks:
+ x = x + block(x)
+ return x * self.gain
+
+ return Net().to(torch.bfloat16)
+
+
+def _offload(net, pinned: bool):
+ from diffusers.hooks import apply_group_offloading
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = True,
+ low_cpu_mem_usage = not pinned,
+ )
+ return net
+
+
+def _reference(x):
+ with torch.no_grad():
+ return _offload(_net(), pinned = True)(x)
+
+
+def _groups_pinned(net):
+ import core.inference.diffusion_memory as mem
+ groups = mem._offload_groups(net)
+ return groups, all(t.is_pinned() for g in groups for t in g.cpu_param_dict.values())
+
+
+def test_a_deferred_pin_renders_identically_and_ends_with_every_group_pinned():
+ import core.inference.diffusion_memory as mem
+
+ mem.install_group_pin_wait()
+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")
+ ref = _reference(x)
+ pipe, net = _Pipe(), _offload(_net(), pinned = False)
+ before = {id(p): p.detach().clone() for p in net.parameters()}
+ assert mem._defer_pinning(pipe, net, torch.device("cuda"), None)
+ groups, pinned = _groups_pinned(net)
+ assert groups and not pinned
+ assert mem.start_background_pins(pipe) == 1
+ with torch.no_grad():
+ out = net(x)
+ again = net(x)
+ torch.cuda.synchronize()
+ mem.stop_background_pins(pipe)
+ assert torch.equal(out, ref) and torch.equal(again, ref)
+ groups, pinned = _groups_pinned(net)
+ assert pinned
+ # the host copies were replaced, not duplicated: every offloaded parameter now IS its pinned chunk view
+ for g in groups:
+ for t, host in g.cpu_param_dict.items():
+ if isinstance(t, torch.nn.Parameter):
+ assert t.device.type == "cpu" and t.data_ptr() == host.data_ptr()
+ for p in net.parameters():
+ assert torch.equal(p.detach(), before[id(p)])
+
+
+def test_an_onload_before_the_pinner_starts_waits_for_its_group():
+ import core.inference.diffusion_memory as mem
+
+ mem.install_group_pin_wait()
+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")
+ ref = _reference(x)
+ pipe, net = _Pipe(), _offload(_net(), pinned = False)
+ mem._defer_pinning(pipe, net, torch.device("cuda"), None)
+ # never started by the caller: the first onload starts it and waits for its own group
+ with torch.no_grad():
+ out = net(x)
+ torch.cuda.synchronize()
+ mem.stop_background_pins(pipe)
+ assert torch.equal(out, ref)
+ assert _groups_pinned(net)[1]
+
+
+def test_a_stopped_pinner_leaves_the_rest_to_diffusers_and_never_blocks():
+ import core.inference.diffusion_memory as mem
+
+ mem.install_group_pin_wait()
+ x = torch.randn(8, 256, dtype = torch.bfloat16, device = "cuda")
+ ref = _reference(x)
+ pipe, net = _Pipe(), _offload(_net(), pinned = False)
+ mem._defer_pinning(pipe, net, torch.device("cuda"), None)
+ mem.stop_background_pins(pipe)
+ with torch.no_grad():
+ out = net(x)
+ torch.cuda.synchronize()
+ assert torch.equal(out, ref)
+ assert not _groups_pinned(net)[1]
diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py
index 1977c9f6567..8d25d7380c1 100644
--- a/studio/backend/tests/test_video_backend.py
+++ b/studio/backend/tests/test_video_backend.py
@@ -8143,8 +8143,8 @@ def test_pipeline_plan_budgets_a_pre_cast_text_encoder_at_its_real_size(fake_run
assert calls[0]["model_dense_mib"] == int(
(transformer_gb + text_encoder_gb * scale + vae_gb) * _MIB_PER_GB
)
- # The pipeline kind budgets one total, so companion_dense_mib stays None as before.
- assert calls[0]["companion_dense_mib"] is None
+ assert calls[0]["companion_dense_mib"] == int((text_encoder_gb * scale + vae_gb) * _MIB_PER_GB)
+ assert calls[0]["text_encoder_dense_mib"] == int(text_encoder_gb * scale * _MIB_PER_GB)
def test_plan_returns_to_bf16_when_the_pre_cast_encoder_does_not_inject(fake_runtime, monkeypatch):
@@ -11185,10 +11185,21 @@ def test_a_resident_video_int8_on_nvidia_keeps_torchao(fake_runtime, monkeypatch
backend.unload()
-def test_video_auto_under_offload_on_nvidia_is_still_skipped(fake_runtime, monkeypatch):
+def test_video_auto_under_offload_on_nvidia_takes_the_torchao_free_int8(fake_runtime, monkeypatch):
calls = _stub_nvidia_video_offload(monkeypatch)
backend = VideoBackend()
status = backend.load_pipeline("Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline")
+ assert [(call["mode"], call["offload"]) for call in calls] == [("int8", True)]
+ assert status["transformer_quant"] == "int8"
+ backend.unload()
+
+
+def test_video_auto_under_offload_on_nvidia_keeps_bf16_under_speed_off(fake_runtime, monkeypatch):
+ calls = _stub_nvidia_video_offload(monkeypatch)
+ backend = VideoBackend()
+ status = backend.load_pipeline(
+ "Wan-AI/Wan2.2-TI2V-5B-Diffusers", model_kind = "pipeline", speed_mode = "off"
+ )
assert calls == []
assert status["transformer_quant"] is None
backend.unload()
diff --git a/studio/backend/tests/test_video_quantised_offload.py b/studio/backend/tests/test_video_quantised_offload.py
new file mode 100644
index 00000000000..1f85c26c1ed
--- /dev/null
+++ b/studio/backend/tests/test_video_quantised_offload.py
@@ -0,0 +1,887 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Conventional video families (Wan2.2, HunyuanVideo-1.5, LTX-2) when the card has to offload."""
+
+import dataclasses
+import sys
+import types
+
+import pytest
+
+from .test_video_backend import ( # noqa: F401 -- fixtures
+ _assume_the_restricted_load_is_available,
+ _FakePipe,
+ _load_gguf,
+ fake_runtime,
+)
+from core.inference.video import VideoBackend
+
+GIB = 1024
+TIERS = (8, 12, 16, 24, 32, 48, 80)
+CAPS = {"sm_120": (12, 0), "sm_89": (8, 9)}
+SUPPORTED = {(12, 0): {"int8", "fp8", "mxfp8", "nvfp4"}, (8, 9): {"int8", "fp8"}}
+
+FAMILIES = {
+ "wan2.2-ti2v-5b": "Wan-AI/Wan2.2-TI2V-5B-Diffusers",
+ "wan2.2-t2v-a14b": "Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+ "hv15-480p": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
+ "hv15-720p": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v",
+ "ltx-2": "Lightricks/LTX-2",
+}
+
+# Tiers: none, dit (TE streamed), stream (DiT streamed), both, model (whole-module). Precision: bf16, ao (torchao), w8 (torchao-free int8).
+_BF, _AO, _W8 = "bf16", "ao", "w8"
+EXPECTED = {
+ "wan2.2-ti2v-5b": {
+ 8: ("model", _W8),
+ 12: ("model", _W8),
+ 16: ("model", _W8),
+ 24: ("dit", _AO),
+ 32: ("dit", _BF),
+ 48: ("none", _BF),
+ 80: ("none", _BF),
+ },
+ "wan2.2-t2v-a14b": {
+ 8: ("model", _W8),
+ 12: ("both", _W8),
+ 16: ("both", _W8),
+ 24: ("model", _W8),
+ 32: ("model", _W8),
+ 48: ("dit", _AO),
+ 80: ("dit", _BF),
+ },
+ "hv15-480p": {
+ 8: ("model", _W8),
+ 12: ("model", _W8),
+ 16: ("both", _W8),
+ 24: ("dit", _AO),
+ 32: ("dit", _BF),
+ 48: ("dit", _BF),
+ 80: ("none", _BF),
+ },
+ "ltx-2": {
+ 8: ("model", _W8),
+ 12: ("model", _W8),
+ 16: ("both", _W8),
+ 24: ("both", _W8),
+ 32: ("model", _W8),
+ 48: ("dit", _AO),
+ 80: ("dit", _BF),
+ },
+}
+EXPECTED["hv15-720p"] = EXPECTED["hv15-480p"]
+
+
+def _tier_code(plan) -> str:
+ if plan.offload_policy == "group":
+ if not plan.stream_transformer:
+ return "dit"
+ return "both" if plan.stream_text_encoders else "stream"
+ return plan.offload_policy
+
+
+def _spoof(
+ monkeypatch,
+ *,
+ tier_gib: int,
+ cap = (12, 0),
+):
+ import core.inference.video as V
+ from core.inference import diffusion_transformer_quant as tq
+ from core.inference.diffusion_device import DiffusionDeviceTarget
+ from core.inference.diffusion_memory import DeviceMemory
+
+ torch = sys.modules["torch"]
+ target = DiffusionDeviceTarget(
+ device = "cuda",
+ dtype = torch.bfloat16,
+ backend = "cuda",
+ vendor = "nvidia",
+ supports_model_cpu_offload = True,
+ supports_default_torch_compile = True,
+ supports_pinned_transfer = True,
+ )
+ total = tier_gib * GIB
+ mem = DeviceMemory(
+ backend = "cuda",
+ device = "cuda",
+ memory_kind = "discrete_vram",
+ free_mib = total - 600,
+ total_mib = total,
+ )
+ monkeypatch.setattr(V, "settled_snapshot_device_memory", lambda *a, **k: mem)
+ monkeypatch.setattr(VideoBackend, "_device_target", lambda self, ordinal = None: target)
+ monkeypatch.setattr(V, "resolve_diffusion_device_target", lambda *a, **k: target)
+ monkeypatch.setattr(V, "dense_transformer_supported", lambda t: True)
+ monkeypatch.setattr(tq, "dense_transformer_supported", lambda t: True)
+ monkeypatch.setattr(tq, "_capability", lambda ordinal = None: cap)
+ monkeypatch.setattr(
+ tq, "_scheme_supported", lambda s, d, unproven_ok = False: s in SUPPORTED[cap]
+ )
+ monkeypatch.setattr(tq, "_is_consumer_gpu", lambda d: tier_gib <= 32)
+ monkeypatch.setattr(tq, "native_quant_host", lambda t: False)
+ monkeypatch.setattr(tq, "native_offload_host", lambda t: True)
+ monkeypatch.setattr(V, "stored_denoiser_precision", lambda *a, **k: None)
+ spy = types.SimpleNamespace(plans = [], quant = [], speed = [], seeded = [])
+
+ import core.inference.video_denoiser_prequant as vdp
+
+ def _seed(fam, base, *, scheme, **kw):
+ spy.seeded.append(scheme)
+ return {"transformer": object()}
+
+ monkeypatch.setattr(vdp, "denoiser_prequant_pipe_kwargs", _seed)
+
+ def _quantize(
+ view,
+ tgt,
+ *,
+ mode,
+ family = None,
+ logger = None,
+ **kw,
+ ):
+ spy.quant.append({"mode": mode, **kw})
+ if kw.get("offload"):
+ return "int8"
+ return tq.select_transformer_quant_scheme(tgt, mode, family = family)
+
+ monkeypatch.setattr(V, "quantize_transformer", _quantize)
+ monkeypatch.setattr(V, "native_quant_reason", lambda m, s: f"native {s}")
+
+ def _speed(view, tgt, *, is_gguf, family, speed_mode, cache_active, offload_active, **kw):
+ compiled = (
+ speed_mode in ("default", "max") and not is_gguf and family.supports_torch_compile
+ )
+ spy.speed.append({"compiled": compiled, "offload_active": offload_active})
+ return {"compiled": compiled}
+
+ monkeypatch.setattr(V, "apply_speed_optims", _speed)
+
+ def _apply(
+ pipe,
+ plan,
+ *,
+ device = None,
+ placement_device = None,
+ logger = None,
+ ):
+ spy.plans.append(plan)
+ return plan.offload_policy, plan.vae_tiling
+
+ monkeypatch.setattr(V, "apply_memory_plan", _apply)
+ return spy
+
+
+def _precision_code(spy, status) -> str:
+ engaged = status.get("transformer_quant")
+ if not engaged or engaged == "off":
+ return _BF
+ assert engaged == "int8", engaged
+ return _W8 if spy.quant and spy.quant[0].get("offload") else _AO
+
+
[email protected]("cap_name", list(CAPS))
[email protected]("tier_gib", TIERS)
[email protected]("family", list(FAMILIES))
+def test_auto_planner_table(fake_runtime, monkeypatch, family, tier_gib, cap_name):
+ """Every conventional family at every card size: tier, precision, compile on, admitted."""
+ spy = _spoof(monkeypatch, tier_gib = tier_gib, cap = CAPS[cap_name])
+ status = VideoBackend().load_pipeline(FAMILIES[family])
+ tier, precision = EXPECTED[family][tier_gib]
+ assert _tier_code(spy.plans[-1]) == tier
+ assert _precision_code(spy, status) == precision
+ assert spy.speed and all(call["compiled"] for call in spy.speed)
+ assert status["loaded"] is True
+ if precision == _W8:
+ # the torchao-free build is asked for by name: auto alone would walk the torchao ladder
+ assert spy.quant[0]["mode"] == "int8"
+
+
[email protected]("family", ["wan2.2-t2v-a14b", "ltx-2"])
+def test_auto_keeps_bf16_where_the_dit_stays_resident_at_80(fake_runtime, monkeypatch, family):
+ """80 GiB: bf16 DiT stays resident with the text encoder streamed, so auto keeps bf16."""
+ spy = _spoof(monkeypatch, tier_gib = 80)
+ status = VideoBackend().load_pipeline(FAMILIES[family])
+ assert _tier_code(spy.plans[-1]) == "dit"
+ assert status["transformer_quant"] in (None, "off")
+ assert not spy.quant
+
+
+def test_explicit_fp8_resident_dit_beside_streamed_text_encoder(fake_runtime, monkeypatch):
+ """A torchao fp8 DiT on 24 GiB engages once the text encoder streams."""
+ spy = _spoof(monkeypatch, tier_gib = 24)
+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "fp8")
+ assert _tier_code(spy.plans[-1]) == "dit"
+ assert status["transformer_quant"] == "fp8"
+ assert spy.quant and not spy.quant[0].get("offload")
+
+
+def test_explicit_fp8_refused_with_reason_when_the_dit_must_move(fake_runtime, monkeypatch):
+ _spoof(monkeypatch, tier_gib = 16)
+ with pytest.raises(RuntimeError, match = r"moves the DiT .*int8 runs torchao-free"):
+ VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "fp8")
+
+
+def test_explicit_int8_under_streamed_dit_is_torchao_free(fake_runtime, monkeypatch):
+ spy = _spoof(monkeypatch, tier_gib = 24)
+ status = VideoBackend().load_pipeline(FAMILIES["ltx-2"], transformer_quant = "int8")
+ assert _tier_code(spy.plans[-1]) == "both"
+ assert status["transformer_quant"] == "int8"
+ assert spy.quant[0] == {"mode": "int8", "offload": True, "act_int8": True}
+
+
+def test_speed_off_keeps_bf16_under_offload(fake_runtime, monkeypatch):
+ """speed_mode='off' is the bit-exact contract: no auto quant even where the DiT has to move."""
+ spy = _spoof(monkeypatch, tier_gib = 16)
+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], speed_mode = "off")
+ assert status["transformer_quant"] in (None, "off")
+ assert not spy.quant
+
+
+def test_hosted_nvfp4_seeds_beside_streamed_text_encoder(fake_runtime, monkeypatch):
+ """Wan2.2-5B NVFP4 on 32 GiB: the split keeps the DiT resident and streams the 11.4 GB encoder."""
+ monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
+ spy = _spoof(monkeypatch, tier_gib = 32)
+ status = VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "nvfp4")
+ assert spy.seeded == ["nvfp4"]
+ assert status["transformer_quant"] == "nvfp4"
+ assert _tier_code(spy.plans[-1]) == "dit"
+
+
+def test_hosted_nvfp4_refused_where_the_dit_must_stream(fake_runtime, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_NVFP4_DIFFUSION", "1")
+ spy = _spoof(monkeypatch, tier_gib = 16)
+ with pytest.raises(RuntimeError, match = "moves the DiT"):
+ VideoBackend().load_pipeline(FAMILIES["wan2.2-ti2v-5b"], transformer_quant = "nvfp4")
+ assert not spy.seeded
+
+
+def test_seed_stays_resident_counts_the_streamed_text_encoder(monkeypatch):
+ torch = pytest.importorskip("torch")
+ import core.inference.video as V
+ from core.inference.diffusion_memory import DeviceMemory
+ from core.inference.video_families import detect_video_family
+
+ fam = detect_video_family(FAMILIES["wan2.2-ti2v-5b"])
+ target = types.SimpleNamespace(
+ device = "cuda", dtype = torch.bfloat16, supports_model_cpu_offload = True
+ )
+
+ def _at(gib):
+ mem = DeviceMemory(
+ backend = "cuda",
+ device = "cuda",
+ memory_kind = "discrete_vram",
+ free_mib = gib * GIB - 600,
+ total_mib = gib * GIB,
+ )
+ monkeypatch.setattr(V, "settled_snapshot_device_memory", lambda *a, **k: mem)
+ return V._video_seed_stays_resident(
+ fam,
+ target = target,
+ scheme = "nvfp4",
+ memory_mode = None,
+ text_encoder_quant = None,
+ base_repo = FAMILIES["wan2.2-ti2v-5b"],
+ )
+
+ assert _at(32) is True
+ assert _at(12) is False
+
+
+def test_offload_tiers_rank_fastest_first():
+ from core.inference.video import _video_plan_label, _video_plan_rank
+
+ def _plan(
+ policy,
+ te = False,
+ dit = True,
+ ):
+ return types.SimpleNamespace(
+ offload_policy = policy, stream_text_encoders = te, stream_transformer = dit
+ )
+
+ whole = _plan("model")
+ whole.estimates = {"whole_module_fits_mib": 20000}
+ order = [
+ _plan("none"),
+ _plan("group", te = True, dit = False),
+ whole,
+ _plan("group"),
+ _plan("group", te = True),
+ _plan("model"),
+ _plan("streaming"),
+ ]
+ assert [_video_plan_rank(p) for p in order] == [0, 1, 2, 3, 4, 5, 6]
+ assert _video_plan_label(order[1]) == "denoiser resident, text encoders streamed"
+
+
+ "state, expected",
+ [
+ # a torchao DiT under whole-module offload: the hook swap fails under inference_mode
+ (dict(transformer_quant = "int8", offload_policy = "model"), True),
+ (dict(transformer_quant = "fp8", offload_policy = "group"), True),
+ (dict(transformer_quant = "int8", offload_policy = "none"), False),
+ (dict(transformer_quant = None, offload_policy = "model"), False),
+ # MiniMax-H3: only its streamed denoiser switches; the pinned one never moves
+ (dict(transformer_quant = "int8", offload_policy = "model", modular = True), False),
+ (
+ dict(
+ transformer_quant = "int8",
+ offload_policy = "model",
+ modular = True,
+ denoiser_streamed = True,
+ ),
+ True,
+ ),
+ ],
+)
+def test_render_grad_mode(state, expected):
+ from core.inference.video import _render_under_no_grad
+
+ modular = state.pop("modular", False)
+ ns = types.SimpleNamespace(
+ family = types.SimpleNamespace(modular_workflow = "t2v" if modular else None),
+ denoiser_streamed = state.pop("denoiser_streamed", False),
+ **state,
+ )
+ assert _render_under_no_grad(ns) is expected
+
+
+def test_vram_floor_counts_what_each_tier_holds_at_once():
+ torch = pytest.importorskip("torch")
+ from core.inference.video import _video_offload_vram_floor_mib
+
+ def _module(mib):
+ module = torch.nn.Module()
+ module.register_buffer("w", torch.zeros(mib * 1024 * 1024 // 2, dtype = torch.bfloat16))
+ return module
+
+ pipe = types.SimpleNamespace(
+ components = {
+ "transformer": _module(10),
+ "transformer_2": _module(10),
+ "text_encoder": _module(12),
+ "vae": _module(3),
+ "scheduler": object(),
+ }
+ )
+
+ def _plan(
+ policy,
+ te = False,
+ dit = True,
+ ):
+ return types.SimpleNamespace(
+ offload_policy = policy, stream_text_encoders = te, stream_transformer = dit
+ )
+
+ assert _video_offload_vram_floor_mib(pipe, _plan("none")) is None
+ assert _video_offload_vram_floor_mib(pipe, _plan("model")) == 12
+ # a flat module is one group, so its whole payload is onloaded during its forward
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", te = True, dit = False)) == 35
+ assert _video_offload_vram_floor_mib(pipe, _plan("group")) == 25
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", te = True)) == 15
+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming")) == 15
+
+
+def test_vram_floor_adds_the_largest_onloaded_block_or_leaf():
+ """Streamed tiers hold the unmatched group plus the current and prefetched block (or leaf) on the device."""
+ torch = pytest.importorskip("torch")
+ from core.inference.video import _video_offload_vram_floor_mib
+
+ def _linear(mib):
+ return torch.nn.Linear(1024, mib * 512, bias = False).to(torch.bfloat16)
+
+ dit = torch.nn.Module()
+ dit.patch_embedding = _linear(1)
+ dit.blocks = torch.nn.ModuleList([_linear(2) for _ in range(4)])
+ encoder = torch.nn.Module()
+ encoder.shared = torch.nn.Embedding(1024, 3 * 512).to(torch.bfloat16)
+ encoder.layers = torch.nn.ModuleList([_linear(1) for _ in range(4)])
+ vae = _linear(3)
+ pipe = types.SimpleNamespace(
+ components = {"transformer": dit, "text_encoder": encoder, "vae": vae}
+ )
+
+ def _plan(
+ policy,
+ backend,
+ te = False,
+ dit = True,
+ ):
+ return types.SimpleNamespace(
+ offload_policy = policy,
+ stream_text_encoders = te,
+ stream_transformer = dit,
+ device_memory = types.SimpleNamespace(backend = backend),
+ )
+
+ # resident TE 7 + VAE 3, plus DiT embed 1 + current and prefetched 2 MiB blocks
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda")) == 15
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "mps")) == 13
+ # resident DiT 9 + VAE 3, plus the 3 MiB embedding and the next 1 MiB leaf
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda", te = True, dit = False)) == 16
+ # streamed modules run in turn: VAE 3 plus the larger of DiT 5 and TE 4
+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cuda")) == 8
+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cpu")) == 6
+ # the VAE decode runs after the streamed DiT has offloaded: only the residents (or the decoder alone) remain
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda"), phase = "decode") == 10
+ assert _video_offload_vram_floor_mib(pipe, _plan("group", "cuda", te = True), phase = "decode") == 3
+ assert _video_offload_vram_floor_mib(pipe, _plan("streaming", "cuda"), phase = "decode") == 3
+ assert _video_offload_vram_floor_mib(pipe, _plan("model", "cuda"), phase = "decode") == 3
+
+
+def test_shortfall_message_only_for_what_cannot_run():
+ from core.inference.video import video_offload_shortfall_message
+
+ assert (
+ video_offload_shortfall_message(
+ family = "wan2.2-t2v-a14b", floor_mib = 20000, available_mib = 24000, placement = "x"
+ )
+ is None
+ )
+ message = video_offload_shortfall_message(
+ family = "wan2.2-t2v-a14b",
+ floor_mib = 27300,
+ available_mib = 24000,
+ placement = "whole-module CPU offload",
+ width = 1280,
+ height = 720,
+ frames = 81,
+ )
+ assert "wan2.2-t2v-a14b needs about" in message and "1280x720 at 81 frames" in message
+
+
+def test_generate_requirement_grows_with_the_requested_clip():
+ import re
+
+ from core.inference.video import video_offload_shortfall_message
+
+ def _required(width, height, frames):
+ message = video_offload_shortfall_message(
+ family = "wan2.2-ti2v-5b",
+ floor_mib = 10000,
+ available_mib = 0,
+ placement = "x",
+ width = width,
+ height = height,
+ frames = frames,
+ )
+ return float(re.search(r"needs about ([0-9.]+) GiB", message).group(1))
+
+ base = _required(640, 480, 49)
+ assert _required(1280, 480, 49) > base
+ assert _required(640, 960, 49) > base
+ assert _required(640, 480, 161) > base
+ # a small clip still fits where a long 720p one does not
+ kwargs = dict(family = "f", floor_mib = 10000, available_mib = 12500, placement = "x")
+ assert video_offload_shortfall_message(**kwargs, width = 256, height = 256, frames = 9) is None
+ assert video_offload_shortfall_message(**kwargs, width = 1280, height = 720, frames = 121)
+
+
+def test_load_refuses_a_tier_whose_weights_cannot_fit(fake_runtime, monkeypatch):
+ """A tier whose co-resident weights exceed the card is refused before placement, not at the first onload."""
+ import core.inference.video as V
+
+ spy = _spoof(monkeypatch, tier_gib = 24)
+ monkeypatch.setattr(V, "_video_offload_vram_floor_mib", lambda pipe, plan: 30000)
+ with pytest.raises(RuntimeError, match = "needs about"):
+ VideoBackend().load_pipeline(FAMILIES["wan2.2-t2v-a14b"])
+ assert not spy.plans
+
+
+def test_generate_refuses_below_the_offload_floor(fake_runtime, monkeypatch, tmp_path):
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ torch = sys.modules["torch"]
+ monkeypatch.setattr(
+ torch,
+ "device",
+ lambda kind, index = None: types.SimpleNamespace(type = kind, index = index),
+ raising = False,
+ )
+ free = {"bytes": 40 * 1024**3}
+ monkeypatch.setattr(
+ torch,
+ "cuda",
+ types.SimpleNamespace(
+ is_available = lambda: False,
+ mem_get_info = lambda device = None: (free["bytes"], 80 * 1024**3),
+ memory_reserved = lambda device = None: 0,
+ ),
+ )
+ backend._state = dataclasses.replace(
+ backend._state, device = "cuda", offload_policy = "model", vram_floor_mib = 27300
+ )
+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)
+ free["bytes"] = 20 * 1024**3
+ with pytest.raises(RuntimeError, match = "needs about"):
+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)
+ # same free VRAM: a small clip runs, a long 720p clip is refused before the render
+ free["bytes"] = 31 * 1024**3
+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)
+ with pytest.raises(RuntimeError, match = "at 121 frames"):
+ backend.generate(prompt = "a sloth", width = 1280, height = 720, num_frames = 121, fps = 8)
+
+
+def test_quantise_stages_what_fits_on_the_card_and_unstages_it(monkeypatch):
+ """The DiT is quantised on the card block by block and sent back to the host when the plan moves it."""
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ import core.inference.video as V
+
+ dit = torch.nn.Module()
+ dit.blocks = torch.nn.ModuleList(
+ [torch.nn.Linear(1024, 1024, bias = False).to(torch.bfloat16) for _ in range(4)]
+ )
+ dit.proj_out = torch.nn.Linear(1024, 64, bias = False).to(torch.bfloat16)
+ block_bytes = 1024 * 1024 * 2
+ target = types.SimpleNamespace(device = "cuda", torch_device = "cuda")
+ # room for the overhead, the linear workspace and exactly two blocks
+ free = (
+ V.DEFAULT_BASE_OVERHEAD_MIB * 1024 * 1024
+ + V._STAGE_LINEAR_WORKSPACE * block_bytes
+ + 2 * block_bytes
+ + 1
+ )
+ monkeypatch.setattr(
+ "utils.hardware.trusted_mem_get_info", lambda d, module = None: (free, 10 * free)
+ )
+ staged = V._stage_denoiser_for_quant(dit, target)
+ on_card = [m for m in staged if next(m.parameters()).is_cuda]
+ assert len(on_card) == len(staged) >= 2
+ assert all(next(b.parameters()).is_cuda for b in dit.blocks[:2])
+ assert not next(dit.blocks[3].parameters()).is_cuda
+ V._unstage_modules(staged)
+ assert all(not p.is_cuda for p in dit.parameters())
+ # nothing staged off CUDA, or for a DiT already on the card
+ assert V._stage_denoiser_for_quant(dit, types.SimpleNamespace(device = "mps")) == []
+
+
+def test_leaf_streaming_peak_keeps_an_enclosing_parent_group_onloaded():
+ """A parent's own params stay onloaded across its descendant leaves, so they add to the two-leaf window."""
+ torch = pytest.importorskip("torch")
+ from core.inference.video import _video_streamed_peak_bytes
+
+ mib = 1 << 20
+
+ def _linear(n):
+ return torch.nn.Linear(1024, n * 512, bias = False).to(torch.bfloat16)
+
+ root = torch.nn.Module()
+ root.block = torch.nn.Module()
+ root.block.table = torch.nn.Parameter(torch.zeros(mib // 2, dtype = torch.bfloat16))
+ root.block.a = _linear(2)
+ root.block.b = _linear(2)
+ root.head = _linear(2)
+ root.norm = torch.nn.LayerNorm(mib // 4, dtype = torch.bfloat16)
+
+ def size(t):
+ return t.numel() * t.element_size()
+
+ # inside block: its 1 MiB table plus leaf a and the prefetched leaf b
+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 5 * mib
+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = False, size = size) == 3 * mib
+
+
+def test_decoded_clip_share_sits_beside_the_decode_floor_not_the_streamed_dit():
+ """Denoise and VAE decode are separate phases: the clip share never stacks on the streamed DiT's weights."""
+ import inspect
+ import re
+
+ import core.inference.video as V
+
+ kwargs = dict(family = "f", floor_mib = 20000, placement = "x", width = 1280, height = 720, frames = 121)
+ # denoise needs 20000 + 2048; decode needs 3000 + 2048 + the clip share, so 23000 MiB is enough
+ assert (
+ V.video_offload_shortfall_message(**kwargs, available_mib = 23000, decode_floor_mib = 3000)
+ is None
+ )
+ # unknown decode floor keeps the conservative sum
+ assert V.video_offload_shortfall_message(**kwargs, available_mib = 23000)
+ # a huge clip makes the decode phase the binding one, even with a small decode floor
+ big = dict(kwargs, floor_mib = 5000, width = 1920, height = 1080, frames = 241)
+ message = V.video_offload_shortfall_message(**big, available_mib = 12000, decode_floor_mib = 4000)
+ assert message and float(re.search(r"needs about ([0-9.]+) GiB", message).group(1)) > 20
+ # the generate guard hands the decode floor recorded at load to the check
+ src = inspect.getsource(V)
+ assert 'decode_floor_mib = getattr(state, "vram_decode_floor_mib", None)' in src
+ assert 'phase = "decode"' in src
+
+
+def test_leaf_streaming_peak_does_not_stack_sibling_branch_groups():
+ """A sibling branch has offloaded its own params before the next runs: only the enclosing chain stacks."""
+ torch = pytest.importorskip("torch")
+ from core.inference.video import _video_streamed_peak_bytes
+
+ mib = 1 << 20
+
+ def _branch():
+ branch = torch.nn.Module()
+ branch.table = torch.nn.Parameter(torch.zeros(mib // 2, dtype = torch.bfloat16))
+ branch.leaf = torch.nn.Linear(1024, 2 * 512, bias = False).to(torch.bfloat16)
+ return branch
+
+ root = torch.nn.Module()
+ root.first, root.second = _branch(), _branch()
+
+ def size(t):
+ return t.numel() * t.element_size()
+
+ # one 1 MiB enclosing table plus the current and prefetched 2 MiB leaves; not both tables
+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 5 * mib
+
+
+def test_applied_floor_is_recorded_even_when_lower_than_planned():
+ """The fallback can stream a DiT the plan kept resident: the lower applied floor must replace the planned one."""
+ import inspect
+
+ import core.inference.video as V
+
+ src = inspect.getsource(V)
+ assert "raised = applied_floor is not None and applied_floor > vram_floor_mib" in src
+ assert (
+ "if applied_floor is not None:\n # record what landed even when lower"
+ in src
+ )
+
+
+def test_whole_module_vae_phase_counts_only_the_decoded_clip_share():
+ """The VAE phase holds the decoded clip, not the denoise-side base the DiT phase already carries."""
+ import dataclasses as dc
+
+ import core.inference.video as V
+ from core.inference.diffusion_memory import MEMORY_MODE_AUTO, OFFLOAD_GROUP, OFFLOAD_MODEL
+
+ @dc.dataclass
+ class _Plan:
+ offload_policy: str = OFFLOAD_GROUP
+ stream_transformer: bool = True
+ stream_text_encoders: bool = False
+ requested_mode: str = MEMORY_MODE_AUTO
+ vae_tiling: bool = False
+ vae_slicing: bool = False
+ reasons: tuple = ()
+ estimates: dict = dc.field(default_factory = lambda: {"safe_device_budget_mib": 17000})
+
+ runtime = V.estimate_video_runtime_mib(width = 1280, height = 704, num_frames = 121)
+ # DiT phase 10000 + 4096 + 2048 = 16144; VAE phase 8000 + clip share + 2048 ~ 13.8 GiB, not 8000 + runtime + 2048
+ out = V._video_prefer_whole_module(
+ _Plan(), denoiser_mib = 10000, text_encoder_mib = 6000, vae_mib = 8000, runtime_mib = runtime
+ )
+ assert out.offload_policy == OFFLOAD_MODEL
+ assert out.estimates["whole_module_fits_mib"] == 10000 + V._VIDEO_DENOISE_ACTIVATION_MIB + 2048
+
+
+def test_quantised_replan_prices_companions_like_the_split_it_passes():
+ """The replan hands the planner the dtype-scaled text encoder, so its total must use the same scaled companions."""
+ import inspect
+
+ import core.inference.video as V
+
+ src = inspect.getsource(V)
+ assert "quant_mib = int((components[0] * factor + scaled_companions_gb) * mib_per_gb)" in src
+ assert "quant_mib = int((components[0] * factor + companions_gb) * mib_per_gb)" not in src
+
+
+def test_leaf_streaming_peak_counts_a_large_parent_group_once():
+ """A parent's loose params are held once under its leaves; they are not also a second prefetch candidate."""
+ torch = pytest.importorskip("torch")
+ from core.inference.video import _video_streamed_peak_bytes
+
+ mib = 1 << 20
+ root = torch.nn.Module()
+ root.parent = torch.nn.Module()
+ root.parent.table = torch.nn.Parameter(torch.zeros(2 * mib, dtype = torch.bfloat16))
+ root.parent.a = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)
+ root.parent.b = torch.nn.Linear(1024, 512, bias = False).to(torch.bfloat16)
+
+ def size(t):
+ return t.numel() * t.element_size()
+
+ # the 4 MiB table plus leaf a and the prefetched leaf b, not 4 + 4 + 1
+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = True, size = size) == 6 * mib
+ assert _video_streamed_peak_bytes(root, "leaf_level", prefetch = False, size = size) == 5 * mib
+
+
+def test_applied_floor_counts_an_encoder_that_refused_leaf_offload():
+ """_apply_group_offload keeps a refusing encoder resident under the same policy; the floor must follow the hooks."""
+ torch = pytest.importorskip("torch")
+ import inspect
+
+ import core.inference.video as V
+
+ def _module(mib):
+ module = torch.nn.Module()
+ module.register_buffer("w", torch.zeros(mib * 1024 * 1024 // 2, dtype = torch.bfloat16))
+ return module
+
+ dit, encoder = _module(10), _module(12)
+ dit._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})
+ pipe = types.SimpleNamespace(
+ components = {"transformer": dit, "text_encoder": encoder, "vae": _module(3)}
+ )
+ plan = types.SimpleNamespace(
+ offload_policy = "group",
+ stream_text_encoders = True,
+ stream_transformer = True,
+ device_memory = types.SimpleNamespace(backend = "cuda"),
+ )
+ # as planned: VAE 3 plus the larger streamed unit (TE 12)
+ assert V._video_offload_vram_floor_mib(pipe, plan) == 15
+ # as applied: the unhooked encoder is resident, so VAE 3 + TE 12 plus the streamed DiT 10
+ assert V._video_offload_vram_floor_mib(pipe, plan, applied = True) == 25
+ assert "_video_offload_vram_floor_mib(pipe, plan, applied = True)" in inspect.getsource(V)
+
+
+def test_failed_staging_leaves_nothing_on_the_card(monkeypatch):
+ torch = pytest.importorskip("torch")
+ if not torch.cuda.is_available():
+ pytest.skip("needs a CUDA device")
+ import core.inference.video as V
+
+ class _Refuses(torch.nn.Linear):
+ def to(self, *args, **kwargs):
+ raise RuntimeError("CUDA out of memory")
+
+ dit = torch.nn.Module()
+ dit.blocks = torch.nn.ModuleList(
+ [torch.nn.Linear(256, 256, bias = False).to(torch.bfloat16) for _ in range(2)]
+ + [_Refuses(256, 256, bias = False)]
+ )
+ monkeypatch.setattr(
+ "utils.hardware.trusted_mem_get_info", lambda d, module = None: (80 * 1024**3, 80 * 1024**3)
+ )
+ target = types.SimpleNamespace(device = "cuda", torch_device = "cuda")
+ assert V._stage_denoiser_for_quant(dit, target) == []
+ assert all(not p.is_cuda for p in dit.parameters())
+
+
+def test_applied_floor_counts_a_dit_the_fallback_hooked():
+ """A refusing encoder can make _apply_group_offload hook the DiT the plan kept resident; count it as streamed."""
+ torch = pytest.importorskip("torch")
+ import core.inference.video as V
+
+ def _linear(mib):
+ return torch.nn.Linear(1024, mib * 512, bias = False).to(torch.bfloat16)
+
+ dit = torch.nn.Module()
+ dit.patch_embedding = _linear(1)
+ dit.blocks = torch.nn.ModuleList([_linear(2) for _ in range(4)])
+ dit.blocks[0]._diffusers_hook = types.SimpleNamespace(hooks = {"group_offloading": object()})
+ encoder = torch.nn.Module()
+ encoder.register_buffer("w", torch.zeros(12 * 1024 * 1024 // 2, dtype = torch.bfloat16))
+ vae = torch.nn.Module()
+ vae.register_buffer("w", torch.zeros(3 * 1024 * 1024 // 2, dtype = torch.bfloat16))
+ pipe = types.SimpleNamespace(
+ components = {"transformer": dit, "text_encoder": encoder, "vae": vae}
+ )
+ plan = types.SimpleNamespace(
+ offload_policy = "group",
+ stream_text_encoders = True,
+ stream_transformer = False,
+ device_memory = types.SimpleNamespace(backend = "cuda"),
+ )
+ # resident TE 12 + VAE 3, plus the streamed DiT's embed 1 and current and prefetched 2 MiB blocks
+ assert V._video_offload_vram_floor_mib(pipe, plan, applied = True) == 20
+
+
+def test_a_declined_quant_unstages_before_any_bf16_rollback():
+ import inspect
+
+ import core.inference.video as V
+ assert "if staged and (video_offload or scheme is None):" in inspect.getsource(V)
+
+
+def test_a_render_starts_only_after_the_background_pins_finish(fake_runtime, monkeypatch, tmp_path):
+ """A render overlapping the post-load pinner ran every later render 2.5-3 s slower (LTX-2.3, A100): wait first."""
+ import core.inference.diffusion_memory as mem
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ pipe = backend._state.pipe
+ order: list = []
+
+ class _Pinner:
+ module = None
+
+ def start(self):
+ order.append("start")
+
+ def join(self, timeout = None):
+ order.append("pinned")
+ return True
+
+ def stop(self, timeout = None):
+ pass
+
+ setattr(pipe, mem._PENDING_PINS_ATTR, [_Pinner()])
+ monkeypatch.setattr(
+ pipe, "scheduler", types.SimpleNamespace(step = lambda *a, **k: None), raising = False
+ )
+ original_call = type(pipe).__call__
+
+ def _call(self, *args, **kwargs):
+ order.append("render")
+ return original_call(self, *args, **kwargs)
+
+ monkeypatch.setattr(type(pipe), "__call__", _call)
+ backend.generate(prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8)
+ assert order[:3] == ["start", "pinned", "render"], order
+
+
+def test_a_cancel_during_the_background_pin_wait_ends_the_render_promptly(
+ fake_runtime, monkeypatch, tmp_path
+):
+ """The wait held _generate_lock for the whole pin (minutes on LTX-2.3), so Cancel and unload waited too."""
+ import re
+ import threading
+ import time
+
+ import core.inference.diffusion_memory as mem
+ from core.inference.video import VIDEO_CANCELLED_MSG
+
+ backend = VideoBackend()
+ _load_gguf(backend, tmp_path)
+ pipe = backend._state.pipe
+ rendered: list = []
+
+ class _SlowPinner:
+ module = None
+
+ def start(self):
+ pass
+
+ def join(self, timeout = None):
+ # A pin that takes 3 s; an unbounded join waits it out, like the real worker.
+ time.sleep(3.0 if timeout is None else timeout)
+ return timeout is None
+
+ def stop(self, timeout = None):
+ pass
+
+ setattr(pipe, mem._PENDING_PINS_ATTR, [_SlowPinner()])
+ monkeypatch.setattr(
+ pipe, "scheduler", types.SimpleNamespace(step = lambda *a, **k: None), raising = False
+ )
+ original_call = type(pipe).__call__
+
+ def _call(self, *args, **kwargs):
+ rendered.append(1)
+ return original_call(self, *args, **kwargs)
+
+ monkeypatch.setattr(type(pipe), "__call__", _call)
+ cancel = threading.Event()
+ threading.Timer(0.5, cancel.set).start()
+ began = time.perf_counter()
+ with pytest.raises(RuntimeError, match = re.escape(VIDEO_CANCELLED_MSG)):
+ backend.generate(
+ prompt = "a sloth", width = 256, height = 256, num_frames = 9, fps = 8, cancel_event = cancel
+ )
+ assert time.perf_counter() - began < 10
+ assert not rendered