Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep MiniMax-H3's int8 denoiser and compile when it has to offload (#12286)
Daniel Han5879a9baad3f
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex 9af61da7e8d..bc18ff1d39f 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -1818,6 +1818,32 @@ def _same_base_model(a: str, b: str) -> bool:return a == b or _tail(a) == _tail(b)+def _unhook_from_manager(+ manager: Any,+ module: Any,+ *,+ logger: Any = None,+ what: str,+) -> bool:+ """Remove ``module`` from a ComponentsManager's offload rotation without moving it."""+ hooks = list(getattr(manager, "model_hooks", None) or ())+ target = next((hook for hook in hooks if getattr(hook, "model", None) is module), None)+ if target is None:+ return False+ try:+ target.remove()+ # Unlist it too, else another pre_forward can evict it to CPU with no hook to bring it back.+ for hook in hooks:+ others = getattr(getattr(hook, "hook", None), "other_hooks", None)+ if others:+ hook.hook.other_hooks = [item for item in others if item is not target]+ manager.model_hooks = [hook for hook in hooks if hook is not target]+ return True+ except Exception as exc: # noqa: BLE001 -- the caller still places the module+ _warn(logger, what, exc)+ return False++def pin_prequantized_module(manager: Any,module: Any,@@ -1851,23 +1877,7 @@ def pin_prequantized_module(not look the way this expects, the module is still placed on ``device`` and False is returned,which is the behaviour before pinning existed."""- hooks = list(getattr(manager, "model_hooks", None) or ())- target = next((hook for hook in hooks if getattr(hook, "model", None) is module), None)- pinned = False- if target is not None:- try:- # Drop the accelerate hook so no pre_forward/offload ever moves this module again ...- target.remove()- # ... and unlist it, so another component's pre_forward cannot pick it as the thing to evict (which would- # move it to the CPU with no hook left to bring it back).- for hook in hooks:- others = getattr(getattr(hook, "hook", None), "other_hooks", None)- if others:- hook.hook.other_hooks = [item for item in others if item is not target]- manager.model_hooks = [hook for hook in hooks if hook is not target]- pinned = True- except Exception as exc: # noqa: BLE001 -- placement below still has to happen- _warn(logger, "pin:hook", exc)+ pinned = _unhook_from_manager(manager, module, logger = logger, what = "pin:hook")module.to(device)if logger is not None:logger.info(@@ -1879,6 +1889,187 @@ def pin_prequantized_module(return pinned+def tensor_payload_bytes(tensor: Any) -> int:+ """Payload bytes; torchao subclasses report their logical bf16 size, so sum the inner tensors."""+ flatten = getattr(tensor, "__tensor_flatten__", None)+ if callable(flatten) and type(tensor).__name__ not in ("Tensor", "Parameter"):+ try:+ return sum(tensor_payload_bytes(getattr(tensor, name)) for name in flatten()[0])+ except Exception: # noqa: BLE001 -- fall through to the logical size+ pass+ return int(tensor.numel()) * int(tensor.element_size())+++def torchao_group_offload_supported() -> bool:+ """Whether group offloading swaps torchao internals (diffusers >= 0.40); older ones half-move via ``param.data``."""+ try:+ from diffusers.hooks import apply_group_offloading # noqa: F401+ from diffusers.hooks import group_offloading as go+ except Exception: # noqa: BLE001 -- no group offloading at all+ return False+ return callable(getattr(go, "_is_torchao_tensor", None)) and callable(+ getattr(go, "_swap_torchao_tensor", None)+ )+++def _weights_pinnable(module: Any) -> bool:+ """Whether every weight class answers ``is_pinned`` (torchao <= 0.17's v1 int8 class raises)."""+ from itertools import chain++ seen: set = set()+ try:+ for tensor in chain(module.parameters(), module.buffers()):+ cls = type(tensor)+ if cls in seen:+ continue+ seen.add(cls)+ tensor.is_pinned()+ except Exception: # noqa: BLE001 -- unimplemented for this subclass+ return False+ return True+++def _evict_rotation_hook(manager: Any, device: Any) -> Any:+ """Pre-hook offloading rotating components on ``device``: the manager only evicts inside another+ managed pre_forward, so the conditioner would otherwise stay beside the whole denoise loop."""+ import torch++ execution = torch.device(device)++ @torch.compiler.disable+ def _pre_forward(_module: Any, _args: Any) -> None:+ moved = False+ for hook in list(getattr(manager, "model_hooks", None) or ()):+ model = getattr(hook, "model", None)+ try:+ where = next(model.parameters()).device+ except Exception: # noqa: BLE001 -- nothing to measure, nothing to move+ continue+ if where.type != execution.type:+ continue+ if execution.index is not None and where.index not in (None, execution.index):+ continue+ hook.offload()+ moved = True+ if moved and execution.type == "cuda":+ torch.cuda.empty_cache()++ return _pre_forward+++def _move_groups_outside_inference_mode(module: Any) -> int:+ """Run group offload moves outside inference_mode (torchao v1 int8 raises+ ``Cannot set version_counter for inference tensor``). Returns the number of groups wrapped."""+ import torch++ seen: set[int] = set()+ for submodule in module.modules():+ hooks = getattr(getattr(submodule, "_diffusers_hook", None), "hooks", None) or {}+ for hook in list(hooks.values()):+ group = getattr(hook, "group", None)+ if group is None or id(group) in seen:+ continue+ seen.add(id(group))+ for name in ("onload_", "offload_"):+ move = getattr(group, name, None)+ if not callable(move):+ continue++ def _outside(_move: Any = move) -> Any:+ with torch.inference_mode(False), torch.no_grad():+ return _move()++ setattr(group, name, torch.compiler.disable(_outside))+ return len(seen)+++def stream_prequantized_module(+ manager: Any,+ module: Any,+ device: Any,+ *,+ logger: Any = None,+ label: str = "pre-quantized denoiser",+) -> Optional[str]:+ """Stream a torchao module block by block via group offloading, outside the ComponentsManager rotation.++ Returns ``"stream"`` (fully pinned, async copies), ``"stream_lazy"`` (pinned one group at a time), ``"sync"``+ (unpinnable weights) or None (nothing changed).+ Raises once the module is unhooked: the caller only streams what does not fit pinned, so a resident+ fallback would OOM or be refused on every render."""+ if not torchao_group_offload_supported():+ return None+ import inspect++ import torch+ from diffusers.hooks import apply_group_offloading++ from .diffusion_memory import (+ DEFAULT_GROUP_BLOCKS,+ _remove_group_offload_hooks,+ _streamed_pin_plan,+ install_group_offload_buffer_restore,+ )++ onload = torch.device(device)+ if not _unhook_from_manager(manager, module, logger = logger, what = "stream:hook"):+ return None+ try:+ # load_state_dict(assign=True) leaves the weights trainable; group offload's grad-mode .cpu() copy then holds+ # each weight's AccumulateGrad, and swap_tensors on onload hits Int8Tensor's missing aten.view.+ module.requires_grad_(False)+ install_group_offload_buffer_restore()+ use_stream = onload.type == "cuda" and _weights_pinnable(module)+ if onload.type == "cuda" and not use_stream:+ # Sync copies measured 16.4 s/step vs 0.8 s resident (B200): rebuild v1 int8 as pinnable Int8Tensor.+ from .prequant_legacy_int8 import convert_legacy_int8_weights+ converted = convert_legacy_int8_weights(module)+ if converted:+ use_stream = _weights_pinnable(module)+ if logger is not None:+ logger.info(+ "diffusion.prequant: rebuilt %d v1 int8 weights as Int8Tensor for streaming",+ converted,+ )+ kwargs: dict[str, Any] = {+ "onload_device": onload,+ "offload_device": torch.device("cpu"),+ "offload_type": "block_level",+ "num_blocks_per_group": DEFAULT_GROUP_BLOCKS,+ "use_stream": use_stream,+ }+ params = inspect.signature(apply_group_offloading).parameters+ if use_stream:+ if "non_blocking" in params:+ kwargs["non_blocking"] = True+ if "record_stream" in params:+ kwargs["record_stream"] = True+ if "low_cpu_mem_usage" in params:+ from itertools import chain+ payload_mib = sum(+ tensor_payload_bytes(t) for t in chain(module.parameters(), module.buffers())+ ) // (1024 * 1024)+ kwargs["low_cpu_mem_usage"] = not _streamed_pin_plan(payload_mib, 0, logger)[0]+ apply_group_offloading(module, **kwargs)+ _move_groups_outside_inference_mode(module)+ module.register_forward_pre_hook(_evict_rotation_hook(manager, onload))+ except Exception as exc:+ _remove_group_offload_hooks(module)+ raise RuntimeError(f"group offloading could not be set up for the {label}: {exc}") from exc+ # Only a full up-front pin keeps a second host copy; lazy pinning holds one group at a time.+ mode = (+ ("stream_lazy" if kwargs.get("low_cpu_mem_usage") else "stream") if use_stream else "sync"+ )+ if logger is not None:+ logger.info(+ "diffusion.prequant: %s streamed block by block on %s (%s copies)",+ label,+ device,+ "overlapped" if use_stream else "synchronous",+ )+ return mode++def _has_meta_tensors(module: Any) -> bool:"""True if any parameter or buffer is still on the meta device after loading."""from itertools import chaindiff --git a/studio/backend/core/inference/prequant_legacy_int8.py b/studio/backend/core/inference/prequant_legacy_int8.pyindex d3eb42d247d..2aa067c0de3 100644--- a/studio/backend/core/inference/prequant_legacy_int8.py+++ b/studio/backend/core/inference/prequant_legacy_int8.py@@ -250,3 +250,27 @@ def _rebuild_standins(ckpt: Any, standins: dict, api: tuple) -> Any:elif isinstance(value, stray):raise ValueError(f"legacy int8 weight {key!r} is not wrapped for activation quant")return ckpt+++def convert_legacy_int8_weights(module: Any) -> int:+ """Rebuild a loaded module's v1 int8 weights as pinnable ``Int8Tensor`` in place (torchao <= 0.17).++ All or nothing: any weight failing validation leaves the module unchanged and returns 0."""+ import torch++ api = _int8_tensor_api()+ classes = {name: _resolve(name) for name in (*LEGACY_INT8_CLASS_NAMES, _ACT_QUANT)}+ if api is None or any(value is None for value in classes.values()):+ return 0+ rebuilt = []+ try:+ for name, submodule in module.named_modules():+ # Registered parameters only: PadToMinM forwards ``weight`` to its inner Linear.+ weight = getattr(submodule, "_parameters", {}).get("weight")+ if isinstance(weight, classes[_LAQT]):+ rebuilt.append((submodule, _rebuild_weight(name, weight, classes, api)))+ except Exception: # noqa: BLE001 -- an unrecognised weight keeps the v1 class everywhere+ return 0+ for submodule, weight in rebuilt:+ submodule.weight = torch.nn.Parameter(weight, requires_grad = False)+ return len(rebuilt)diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex a53ab0158e6..447354c3b3c 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -992,6 +992,10 @@ class _VideoLoadState:# is resident alongside whatever component is running. The memory preflight has to know, because that turns its# floor from a max into a sum.denoiser_pinned: bool = False+ # MiniMax-H3: streamed denoiser, so the VRAM floor is the largest single phase, not a sum.+ denoiser_streamed: bool = False+ # MiniMax-H3: the streamed denoiser also holds a full pinned host copy, which the host floor counts twice.+ denoiser_host_copy: bool = Falseresolved: Optional[dict] = None@@ -1120,49 +1124,87 @@ def _h3_auto_precision_ok(target: Any = None) -> bool:return False+def h3_streamed_int8_supported(target: Any = None) -> bool:+ """Whether this host can run H3's hosted INT8 denoiser streamed: the auto gate (CUDA bf16, so+ sm_80+ on NVIDIA), torchao-aware group offload, and a loader that can read the checkpoint+ (else auto keeps bf16 and the tier's load is refused). Never raises."""+ if not _h3_auto_precision_ok(target):+ return False+ try:+ from .diffusion_prequant import (+ restricted_prequant_load_supported,+ torchao_group_offload_supported,+ )+ return bool(torchao_group_offload_supported()) and bool(+ restricted_prequant_load_supported(H3_AUTO_FALLBACK_SCHEME)+ )+ except Exception: # noqa: BLE001 -- an unanswerable probe hides the tier+ return False+++def _h3_others_bytes(+ fam: Any, components: Any, *, te_scheme: Optional[str], dtype: Any, rotating: bool+) -> int:+ """Bytes beside the denoiser. ``rotating``: conditioner and VAEs rotate (the 40 GB reserve evicts+ one before placing the other), so only the larger counts; otherwise they add up."""+ import torch++ scale = 2.0 if dtype is torch.float32 else 1.0+ text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])+ headroom_bytes = (+ estimate_video_runtime_mib(+ width = fam.resolution_presets[0][0],+ height = fam.resolution_presets[0][1],+ num_frames = fam.default_num_frames,+ )+ * 1024+ * 1024+ )+ beside = max(text_encoder_gb, components[2]) if rotating else text_encoder_gb + components[2]+ return int(beside * scale * 1000.0**3) + headroom_bytes++def _h3_dense_denoiser_resident_bytes(- fam: Any, *, denoiser: Any, te_scheme: Optional[str], dtype: Any+ fam: Any,+ *,+ denoiser: Any,+ te_scheme: Optional[str],+ dtype: Any,+ rotating: bool = False,) -> Optional[tuple[int, int]]:"""``(denoiser_bytes, everything_else_bytes)`` for a MiniMax-H3 modular pipeline, or None.- ``everything_else`` is the conditioner at the precision this load ENGAGED, plus the VAEs, plus- a frames-aware activation headroom: what must still fit alongside a denoiser taken out of the- rotation. The denoiser is measured off the built module, not the family table, because the- table holds the released dense size and the module in hand may not be it."""+ ``everything_else`` is ``_h3_others_bytes``. The denoiser is measured off the built module, not+ the family table, which holds the released dense size."""components = getattr(fam, "bf16_components_gb", None)if not components or denoiser is None:return Nonetry:from itertools import chain- import torch+ from .diffusion_prequant import tensor_payload_bytesdenoiser_bytes = sum(- t.numel() * t.element_size()+ tensor_payload_bytes(t)for t in chain(denoiser.parameters(), denoiser.buffers())if not getattr(t, "is_meta", False))if denoiser_bytes <= 0:return None- scale = 2.0 if dtype is torch.float32 else 1.0- text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])- headroom_bytes = (- estimate_video_runtime_mib(- width = fam.resolution_presets[0][0],- height = fam.resolution_presets[0][1],- num_frames = fam.default_num_frames,- )- * 1024- * 1024+ others = _h3_others_bytes(+ fam, components, te_scheme = te_scheme, dtype = dtype, rotating = rotating)- others = int((text_encoder_gb + components[2]) * scale * 1000.0**3) + headroom_bytesreturn int(denoiser_bytes), othersexcept Exception: # noqa: BLE001 -- an unanswerable estimate keeps the rotation as it isreturn Nonedef _h3_planned_denoiser_bytes(- fam: Any, *, te_scheme: Optional[str], dtype: Any+ fam: Any,+ *,+ te_scheme: Optional[str],+ dtype: Any,+ rotating: bool = False,) -> Optional[tuple[int, int]]:"""The same ``(denoiser_bytes, everything_else_bytes)`` split, PREDICTED from the family table.@@ -1180,17 +1222,9 @@ def _h3_planned_denoiser_bytes(scale = 2.0 if dtype is torch.float32 else 1.0denoiser_bytes = int(components[0] * scale * 1000.0**3)- text_encoder_gb = h3_te_resident_gb(te_scheme, bf16_gb = components[1])- headroom_bytes = (- estimate_video_runtime_mib(- width = fam.resolution_presets[0][0],- height = fam.resolution_presets[0][1],- num_frames = fam.default_num_frames,- )- * 1024- * 1024+ others = _h3_others_bytes(+ fam, components, te_scheme = te_scheme, dtype = dtype, rotating = rotating)- others = int((text_encoder_gb + components[2]) * scale * 1000.0**3) + headroom_bytesreturn denoiser_bytes, othersexcept Exception: # noqa: BLE001 -- an unanswerable estimate keeps the released denoiserreturn None@@ -1240,10 +1274,32 @@ def _h3_dense_denoiser_fits(sizes: Optional[tuple[int, int]], free_bytes: Optionreturn int(free_bytes) >= int(denoiser_bytes) + int(others_bytes)-# The scheme an UNSET transformer_quant falls back to when the released denoiser cannot stay resident. int8 rather than-# fp8: the two are the same 20.3 GB resident and neither is a faster GEMM, but int8 scored closer to the released-# denoiser (SSIM 0.49 against 0.43) and, unlike fp8, needs no per-row scale support from the card.+def _h3_placement_tier(+ *,+ quantised: bool,+ whole_set_sizes: Optional[tuple[int, int]],+ pinned_sizes: Optional[tuple[int, int]],+ free_bytes: Optional[int],+ speed_off: bool,+) -> str:+ """H3 denoiser placement, fastest first: ``resident``, ``pinned``, ``stream`` (quantised only) or ``rotation``.++ A quantised denoiser never rotates (torchao does not survive the mid-block move); an unreadable card keeps it pinned."""+ if not speed_off and _h3_dense_denoiser_fits(whole_set_sizes, free_bytes):+ return "resident"+ if quantised:+ if pinned_sizes is None or free_bytes is None:+ return "pinned"+ return "pinned" if _h3_dense_denoiser_fits(pinned_sizes, free_bytes) else "stream"+ if not speed_off and _h3_dense_denoiser_fits(pinned_sizes, free_bytes):+ return "pinned"+ return "rotation"+++# Unset transformer_quant resolves to int8 over fp8: same size and speed, closer to released (SSIM 0.49 vs 0.43),+# and no per-row scale support needed.H3_AUTO_FALLBACK_SCHEME = "int8"+_H3_STREAM_MODES = ("stream", "stream_lazy", "sync")def _h3_auto_denoiser_scheme(@@ -1261,8 +1317,8 @@ def _h3_auto_denoiser_scheme() -> Optional[str]:"""The hosted scheme an UNSET ``transformer_quant`` resolves to, or None to keep bfloat16.- int8 wherever the hosted checkpoint exists for this partition and fits, which is now the- DEFAULT rather than a fallback. Measured on an H200 (141 GB) and a B200 (183 GB), 960x544,+ int8 wherever the hosted checkpoint exists for this partition, which is the DEFAULT rather+ than a fallback. Measured on an H200 (141 GB) and a B200 (183 GB), 960x544,124 frames, 8 steps, every component resident in both rows:released bf16 20.06 / 12.76 / 12.77 s 102.8 GB steady@@ -1279,8 +1335,8 @@ def _h3_auto_denoiser_scheme(schedule, but the same seed and prompt render a different video. ``transformer_quant='none'``keeps the released weights, and speed_mode="off" declines this on its own below.- ``free_reader`` is how the device is measured: live free memory by default, and CAPACITY for- the pre-download decision, which runs while the previous pipeline is still resident.+ The fit decides PLACEMENT, not precision. Only a diffusers without torchao-aware group offloading+ needs the pinned fit, via ``free_reader`` (live free memory, or CAPACITY pre-download).Never raises, and every unanswerable question keeps the released denoiser."""# An explicit speed_mode="off" is a bit-exact contract, and a re-rolled sample is not bit-exact. The conventional@@ -1301,27 +1357,24 @@ def _h3_auto_denoiser_scheme(return Noneif _h3_te_canonical(base_repo) != _h3_te_canonical(canonical):return None- sizes = _h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = dtype)- free_bytes = (free_reader or _h3_free_device_bytes)(device)- if sizes is None or free_bytes is None:- # No reading is not evidence of a shortfall, and guessing wrong here changes the picture a user gets. Keep the- # released denoiser, which is what happens today.- return Noneif not video_family_prequant_available(fam, H3_AUTO_FALLBACK_SCHEME, task = task, base_repo = base_repo):# Asked per (scheme, PARTITION): a partition with no hosted checkpoint has no fallback, and serving the other# partition's would generate the wrong thing.return None- # And the replacement has to fit BEFORE it is chosen. A torchao denoiser cannot ride the offload rotation at all (it- # does not survive the mid-block move), so taking it means pinning it, which turns the memory floor from a max into- # a sum: 20.3 GB resident PLUS whatever runs beside it. Under text_encoder_quant="none" that sum is larger than the- # dense rotation it replaces, so a card that renders today would refuse every generation afterwards. Where the- # replacement does not fit either, the released denoiser in the rotation is the configuration that still runs.- hosted_bytes = int(h3_transformer_resident_gb(H3_AUTO_FALLBACK_SCHEME) * 1000.0**3)- if not _h3_dense_denoiser_fits((hosted_bytes, sizes[1]), free_bytes):- return None- from .diffusion_prequant import restricted_prequant_load_supported+ from .diffusion_prequant import (+ restricted_prequant_load_supported,+ torchao_group_offload_supported,+ )++ if not torchao_group_offload_supported():+ # Without streaming, a torchao denoiser can only be pinned beside the larger rotating component.+ sizes = _h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = dtype, rotating = True)+ free_bytes = (free_reader or _h3_free_device_bytes)(device)+ hosted_bytes = int(h3_transformer_resident_gb(H3_AUTO_FALLBACK_SCHEME) * 1000.0**3)+ if sizes is None or not _h3_dense_denoiser_fits((hosted_bytes, sizes[1]), free_bytes):+ return Noneif not restricted_prequant_load_supported(H3_AUTO_FALLBACK_SCHEME):# Runs BEFORE the download plan, so never pick an artifact the loader will refuse. Asked last so@@ -5828,15 +5881,8 @@ class VideoBackend:transformer_quant_reason = "released bfloat16 components"# "auto" asks the backend to choose. The auto LADDER is still not it -- that quantises a dense module on device,# which this workflow never materialises -- so the choice is between the released denoiser and a hosted- # checkpoint, and it turns on whether the released one can stay resident. Where it fits, auto keeps it: same- # weights, same picture, and the pin plus regional compile below make it fast without touching a single value.- # Where it does not, keeping it is not a smaller win but a cliff -- it rides the CPU-offload rotation, a module- # that moves cannot be compiled, and the same 8-step job goes 23.7 s to 194 s. The hosted checkpoint is 20.3 GB- # resident, so it pins, so the compile comes back. The trade is that the hosted denoisers RE-ROLL the sample:- # mean SSIM 0.49 (int8) / 0.43 (fp8) against the released weights, where that config against itself scores 0.99.- # No NaN, no black frames, no visible degradation at the model's own 30-step schedule, but not the same picture.- # Which is why this is a fallback and not a blanket default: it is taken only against a configuration nobody- # would choose.+ # checkpoint; auto takes hosted int8 wherever it exists (see _h3_auto_denoiser_scheme). Memory decides the+ # PLACEMENT, never the precision. transformer_quant='none' keeps the released weights.scheme = (None if transformer_quant_is_auto else normalize_transformer_quant(transformer_quant))@@ -5875,10 +5921,8 @@ class VideoBackend:if auto_fallback_scheme is not None:scheme = auto_fallback_schemelogger.info(- "video.transformer_quant: the released bfloat16 denoiser does not fit beside "- "the other components on this device, where it would stay in the CPU-offload "- "rotation and lose the regional compile with it, so auto is taking the hosted "- "%s checkpoint (ask for transformer_quant='none' to keep the released weights)",+ "video.transformer_quant: auto is taking the hosted %s denoiser checkpoint "+ "(ask for transformer_quant='none' to keep the released weights)",auto_fallback_scheme,)# Per (scheme, PARTITION), not per scheme: each workflow denoises against its own checkpoint partition and they@@ -5983,9 +6027,8 @@ class VideoBackend:f"hosted pre-quantized {scheme} denoiser checkpoint ({source.location})"if auto_fallback_scheme is Noneelse (- f"the released bfloat16 denoiser does not fit resident on this device, "- f"so auto took the hosted {scheme} checkpoint ({source.location}); "- f"transformer_quant='none' keeps the released weights"+ f"auto: hosted pre-quantized {scheme} denoiser checkpoint "+ f"({source.location}); transformer_quant='none' keeps the released weights"))logger.info(@@ -6177,6 +6220,7 @@ class VideoBackend:)offload_policy = "none"denoiser_pinned = False+ denoiser_streamed: Optional[str] = None# load_components just spent minutes building ~145 GB of components, and everything below this line either moves# weights onto the card or mutates process-wide backend flags. The conventional placement path fences on the# token for exactly that reason; without the same fence here a cancelled or superseded worker resumes into a GPU@@ -6252,13 +6296,59 @@ class VideoBackend:offload_policy = "model"if transformer_quant_engaged:# enable_auto_cpu_offload just parked every component on the CPU and will move each one back inside- # its own pre_forward. A torchao pre-quantized denoiser does not survive that mid-block move (see- # pin_prequantized_module), so place it now and take it out of the rotation. Nothing else changes:- # the encoder and the VAEs keep their hooks and still offload around it.- from .diffusion_prequant import pin_prequantized_module- denoiser_pinned = pin_prequantized_module(- manager, denoiser, device, logger = logger+ # its own pre_forward, which a torchao denoiser does not survive: pin it if it fits beside the+ # larger rotating component, else stream it via group offloading.+ from .diffusion_prequant import (+ pin_prequantized_module,+ stream_prequantized_module,+ )++ pinned_sizes = _h3_dense_denoiser_resident_bytes(+ fam,+ denoiser = denoiser,+ te_scheme = text_encoder_quant_engaged,+ dtype = dtype,+ rotating = True,+ )+ stream = (+ denoiser is not None+ and _h3_placement_tier(+ quantised = True,+ whole_set_sizes = None,+ pinned_sizes = pinned_sizes,+ free_bytes = free_before_placement,+ speed_off = False,+ )+ == "stream"+ )+ placed = (+ stream_prequantized_module(manager, denoiser, device, logger = logger)+ if stream+ else None)+ if stream and placed not in _H3_STREAM_MODES:+ # Pinning is the tier the planner already ruled out, so it would OOM or be refused later.+ raise RuntimeError(+ f"MiniMax-H3's {transformer_quant_engaged} denoiser has to stream block by block "+ f"on this GPU ({free_before_placement / 1e9:.1f} GB free, "+ f"{sum(pinned_sizes) / 1e9:.1f} GB needed pinned), but group offloading is not "+ "available in this install. Free GPU memory or pick the GGUF version."+ )+ if placed in _H3_STREAM_MODES:+ # offload_policy stays "model": the post-generation host reclaim keys on it.+ denoiser_streamed = placed+ logger.info(+ "video.h3_placement: %.1f GB free is under the %.1f GB a pinned %s "+ "denoiser plus the larger rotating component need, so it streams "+ "block by block",+ free_before_placement / 1e9,+ sum(pinned_sizes) / 1e9,+ transformer_quant_engaged,+ )+ else:+ denoiser_pinned = pin_prequantized_module(+ manager, denoiser, device, logger = logger+ )elif denoiser is not None and effective_speed != SPEED_OFF:# The RELEASED bfloat16 denoiser, for a different reason: not correctness, speed. Which is why# speed=off skips this branch and keeps the rotation: taking a module out of the offload rotation is@@ -6271,7 +6361,11 @@ class VideoBackend:# against LIVE free memory, after every component is built and parked: the only reading that# describes the card this load got.sizes = _h3_dense_denoiser_resident_bytes(- fam, denoiser = denoiser, te_scheme = text_encoder_quant_engaged, dtype = dtype+ fam,+ denoiser = denoiser,+ te_scheme = text_encoder_quant_engaged,+ dtype = dtype,+ rotating = True,)free_bytes = Noneif sizes is not None:@@ -6305,10 +6399,11 @@ class VideoBackend:# needs ``denoiser_pinned``, which only exists after the offload. ``effective_speed`` itself was resolved above# the offload, because the pin reads it.# ── the speed layer this workflow used to skip entirely.- if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not denoiser_pinned:- # Compile ONLY over a resident denoiser: inside the rotation the compiled graph fights the onload hooks,- # measured 69-85 s against 30-115 s eager on the same job at a 135.7 GB peak. The rest of the tier- # (channels_last VAE, cudnn.benchmark, attention backend) is kept.+ if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and not (+ denoiser_pinned or denoiser_streamed+ ):+ # Compile only a resident or streamed denoiser: in the rotation the graph fights the onload hooks+ # (measured 69-85 s vs 30-115 s eager). Group offload hooks are compiler-disabled.logger.info("video.speed_mode: MiniMax-H3 denoiser is in the CPU-offload rotation, so the ""regional compile is held off (measured slower than eager there); keeping the "@@ -6393,12 +6488,21 @@ class VideoBackend:memory_mode,offload_policy,"MiniMax-H3 ComponentsManager auto CPU offload"- + ("; the pre-quantized denoiser stays resident" if denoiser_pinned else ""),+ + ("; the pre-quantized denoiser stays resident" if denoiser_pinned else "")+ + (+ "; the pre-quantized denoiser streams block by block (group offload)"+ if denoiser_streamed+ else ""+ ),),"speed_mode": (speed_mode,effective_speed,- "regional compile over the resident pre-quantized denoiser"+ (+ "regional compile over the streamed pre-quantized denoiser"+ if denoiser_streamed+ else "regional compile over the resident pre-quantized denoiser"+ )if "compiled" in speed_optimselse "lossless optimisations only; the denoiser is in the CPU-offload rotation",),@@ -6490,6 +6594,8 @@ class VideoBackend:# start.text_encoder_quant = text_encoder_quant_engaged,denoiser_pinned = denoiser_pinned,+ denoiser_streamed = bool(denoiser_streamed),+ denoiser_host_copy = denoiser_streamed == "stream",resolved = resolved,)self._precommit_globals = None@@ -7142,6 +7248,7 @@ class VideoBackend:),transformer_gb = h3_transformer_resident_gb(state.transformer_quant),transformer_pinned = bool(getattr(state, "denoiser_pinned", False)),+ transformer_streamed = bool(getattr(state, "denoiser_streamed", False)),)if available_vram_gb + 0.25 < required_vram_gb:raise RuntimeError(@@ -7166,6 +7273,7 @@ class VideoBackend:state.text_encoder_quant, bf16_gb = H3_TEXT_ENCODER_BF16_GB),transformer_gb = h3_transformer_resident_gb(state.transformer_quant),+ transformer_streamed = bool(getattr(state, "denoiser_host_copy", False)),)if host_capacity_gb + 0.5 < required_host_gb:raise RuntimeError(@@ -7451,8 +7559,14 @@ class VideoBackend:)elif state.transformer_cache:self._reset_step_cache(pipe)+ # torchao's aliasing check fails a streamed weight move under inference_mode.+ grad_ctx = (+ torch.no_grad()+ if state.transformer_quant and getattr(state, "denoiser_streamed", False)+ else torch.inference_mode()+ )try:- with torch.inference_mode(), protect_ctx, progress_ctx(), sigma_ctx:+ with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:output = render_thread.run("video", lambda: pipe(**kwargs))except _VideoGenerationCancelled:# Unwinding by exception skips maybe_free_model_hooks(); under offload the onloaded modules woulddiff --git a/studio/backend/core/inference/video_minimax_h3.py b/studio/backend/core/inference/video_minimax_h3.pyindex e8b7bc489ba..20f8e802d8c 100644--- a/studio/backend/core/inference/video_minimax_h3.py+++ b/studio/backend/core/inference/video_minimax_h3.py@@ -56,6 +56,8 @@ H3_TRANSFORMER_BF16_GB = 66.3# Video + audio VAE, from the family's bf16_components_gb. Only a floor for the offloaded term: it stops a very small# conditioner from claiming a base no component rotation could actually fit in.H3_VAE_RESIDENT_GB = 11.1+# Streamed denoiser's device footprint (running + prefetched group + top-level modules); below the VAE term.+H3_TRANSFORMER_STREAMED_GB = 3.0# Resident decimal GB of each hosted pre-quantized denoiser, from the artifact sizes in unsloth/MiniMax-H3-FP8@@ -100,13 +102,26 @@ def estimate_h3_diffusers_vram_gb(text_encoder_gb: Optional[float] = None,transformer_gb: Optional[float] = None,transformer_pinned: bool = False,+ transformer_streamed: bool = False,) -> float:"""Measured available-VRAM floor for an H3 Diffusers generation.``text_encoder_gb`` / ``transformer_gb`` are the RESIDENT sizes this load actually holds and``transformer_pinned`` whether the denoiser was taken out of the offload rotation; all unset- keeps the released-bfloat16 floor this shipped with."""+ keeps the released-bfloat16 floor this shipped with.+ ``transformer_streamed``: no two large components are ever resident together."""volume_mpixel_frames = width * height * num_frames / 1_000_000+ if transformer_streamed:+ text_encoder = (+ H3_TEXT_ENCODER_BF16_GB if text_encoder_gb is None else float(text_encoder_gb)+ )+ activations = H3_DIFFUSERS_VRAM_GB_PER_MPIXEL_FRAME * volume_mpixel_frames+ return max(+ text_encoder + H3_DIFFUSERS_VRAM_OVERHEAD_GB,+ max(H3_TRANSFORMER_STREAMED_GB, H3_VAE_RESIDENT_GB)+ + H3_DIFFUSERS_VRAM_OVERHEAD_GB+ + activations,+ )base = h3_diffusers_vram_base_gb(text_encoder_gb = text_encoder_gb,transformer_gb = transformer_gb,@@ -131,6 +146,7 @@ def estimate_h3_diffusers_host_ram_gb(*,text_encoder_gb: Optional[float] = None,transformer_gb: Optional[float] = None,+ transformer_streamed: bool = False,) -> float:"""Host-RAM floor for the offload tier selected at the available VRAM.@@ -145,11 +161,15 @@ def estimate_h3_diffusers_host_ram_gb(A pinned denoiser is still counted here. It lives on the device during the generation, but itwas built on the host to get there, and keeping it in the sum errs toward refusing a load that- would have fitted rather than admitting one that will not."""- if available_vram_gb >= H3_DIFFUSERS_HOST_RAM_TIER_VRAM_GB:+ would have fitted rather than admitting one that will not.+ A streamed denoiser counts twice (pinned staging copy; measured 80.2 GB peak vs 64.5 GB single count)."""+ # A streamed load keeps its staging copy even when free VRAM later climbs past the tier.+ if available_vram_gb >= H3_DIFFUSERS_HOST_RAM_TIER_VRAM_GB and not transformer_streamed:return H3_DIFFUSERS_HOST_RAM_HIGH_VRAM_GBtext_encoder = H3_TEXT_ENCODER_BF16_GB if text_encoder_gb is None else float(text_encoder_gb)transformer = H3_TRANSFORMER_BF16_GB if transformer_gb is None else float(transformer_gb)+ if transformer_streamed:+ transformer *= 2return text_encoder + transformer + H3_VAE_RESIDENT_GB + H3_DIFFUSERS_HOST_RAM_HEADROOM_GBdiff --git a/studio/backend/main.py b/studio/backend/main.pyindex 83959cdf121..1015ce97253 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -659,6 +659,13 @@ def _post_warm_background_work(generation: Optional[int] = None) -> None:except Exception as _dq_exc: # noqa: BLE001 -- a picker label must never break the warmimport structlog as _structlog_structlog.get_logger(__name__).debug("dense quant capability skipped: %s", _dq_exc)+ try:+ _refresh_quantised_streaming_capability()+ except Exception as _qs_exc: # noqa: BLE001 -- a picker tier must never break the warm+ import structlog as _structlog+ _structlog.get_logger(__name__).debug(+ "quantised streaming capability skipped: %s", _qs_exc+ )if _post_warm_retired(generation):return@@ -2404,6 +2411,52 @@ def _dense_quant_supported() -> bool:return bool(_dense_quant_capability)+# Whether group offload can stream torchao weights (diffusers >= 0.40) on a GPU that can run the INT8+# denoiser. None until resolved off the polled path; the picker offers no streamed tier before then.+_quantised_streaming_capability: Optional[bool] = None+++def _refresh_quantised_streaming_capability() -> bool:+ """Resolve and cache the streaming bit. Imports diffusers; never call from the polled route."""+ global _quantised_streaming_capability+ from core.inference.video import h3_streamed_int8_supported++ _quantised_streaming_capability = _probe_quantised_streaming(h3_streamed_int8_supported)+ return _quantised_streaming_capability+++def _probe_quantised_streaming(supported: Any) -> bool:+ """Every visible CUDA card must qualify: a load may be pinned to any of them, and one that+ resolves to float16 keeps bf16 (the same intersection as ``_probe_dense_quant_schemes``)."""+ try:+ import torch++ count = torch.cuda.device_count() if torch.cuda.is_available() else 0+ if count <= 1:+ return bool(supported())+ from core.inference.diffusion_device import resolve_diffusion_device_target++ return all(+ bool(supported(resolve_diffusion_device_target(ordinal = ordinal)))+ for ordinal in range(count)+ )+ except Exception: # noqa: BLE001 -- an unanswerable probe hides the tier+ return False+++def _quantised_streaming() -> bool:+ """The streaming bit for ``/api/system``. Resolved here only once a load has already loaded every+ module it reads, since a cold warm (UNSLOTH_STUDIO_DISABLE_TORCH_WARM=1) never resolves it."""+ if _quantised_streaming_capability is None and all(+ name in sys.modules for name in ("torch", "diffusers", "torchao", "core.inference.video")+ ):+ try:+ return _refresh_quantised_streaming_capability()+ except Exception: # noqa: BLE001 -- the picker then keeps the tier hidden+ return False+ return bool(_quantised_streaming_capability)++def _nvfp4_diffusion_enabled() -> bool:"""Whether image and video generation may offer NVFP4 (``UNSLOTH_NVFP4_DIFFUSION``)."""try:@@ -2519,6 +2572,8 @@ def get_system_info(# pure read of that same pass."dense_quant_supported": _dense_quant_supported(),"dense_quant_schemes": _dense_quant_schemes(),+ # The streamed MiniMax-H3 tier needs group offload that swaps torchao weights.+ "quantised_streaming": _quantised_streaming(),# Torch-free env read, safe on this polled route."nvfp4_diffusion": _nvfp4_diffusion_enabled(),}diff --git a/studio/backend/tests/test_h3_quantised_offload.py b/studio/backend/tests/test_h3_quantised_offload.pynew file mode 100644index 00000000000..faed4b96293--- /dev/null+++ b/studio/backend/tests/test_h3_quantised_offload.py@@ -0,0 +1,504 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""MiniMax-H3 keeps its quantised denoiser on a small card: the fit picks resident, pinned or streamed."""++from __future__ import annotations++import copy+import types++import pytest++torch = pytest.importorskip("torch")+++def _h3_family():+ from core.inference.video_families import detect_video_family+ return detect_video_family("minimax-h3")++[email protected](autouse = True)+def _hosted_checkpoint_readable(monkeypatch):+ import core.inference.diffusion_prequant as pq+ monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)+ monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: True)+++def _precision(vid, fam, tq, te_scheme, free_gb, monkeypatch):+ if tq in ("int8", "fp8"):+ return tq+ if tq == "none":+ return None+ monkeypatch.setattr(vid, "_h3_auto_precision_ok", lambda target = None: True, raising = False)+ monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: int(free_gb * 1000**3))+ return vid._h3_auto_denoiser_scheme(+ fam,+ target = None,+ dtype = torch.bfloat16,+ device = "cuda",+ te_scheme = te_scheme,+ task = "fl2va",+ base_repo = fam.base_repo,+ )+++# (free GB, conditioner, request) -> (engaged denoiser, tier). Quantised denoiser 20.3 GB, released 66.3 GB,+# conditioner 27.2 GB int8 / 66.7 GB dense, VAEs 11.1 GB, plus the default-shape activation headroom.+_LADDER = {+ (24, "int8", "auto"): ("int8", "stream"),+ (32, "int8", "auto"): ("int8", "stream"),+ (48, "int8", "auto"): ("int8", "stream"),+ (64, "int8", "auto"): ("int8", "pinned"),+ (80, "int8", "auto"): ("int8", "resident"),+ (141, "int8", "auto"): ("int8", "resident"),+ (24, None, "auto"): ("int8", "stream"),+ (32, None, "auto"): ("int8", "stream"),+ (48, None, "auto"): ("int8", "stream"),+ (64, None, "auto"): ("int8", "stream"),+ (80, None, "auto"): ("int8", "stream"),+ (141, None, "auto"): ("int8", "resident"),+ (24, "int8", "none"): (None, "rotation"),+ (32, "int8", "none"): (None, "rotation"),+ (48, "int8", "none"): (None, "rotation"),+ (64, "int8", "none"): (None, "rotation"),+ (80, "int8", "none"): (None, "rotation"),+ (141, "int8", "none"): (None, "resident"),+ (24, None, "none"): (None, "rotation"),+ (80, None, "none"): (None, "rotation"),+ (141, None, "none"): (None, "rotation"),+}++[email protected]("free_gb", [24, 32, 48, 64, 80, 141])[email protected]("te_scheme", ["int8", None])[email protected]("tq", ["auto", "int8", "fp8", "none"])+def test_the_placement_ladder(monkeypatch, free_gb, te_scheme, tq):+ from core.inference import video as vid++ fam = _h3_family()+ engaged = _precision(vid, fam, tq, te_scheme, free_gb, monkeypatch)+ assert engaged == (None if tq == "none" else ("int8" if tq == "auto" else tq))++ from core.inference.video_minimax_h3 import h3_transformer_resident_gb++ denoiser = int(h3_transformer_resident_gb(engaged) * 1000**3)+ whole = vid._h3_planned_denoiser_bytes(fam, te_scheme = te_scheme, dtype = torch.bfloat16)+ pinned = vid._h3_planned_denoiser_bytes(+ fam, te_scheme = te_scheme, dtype = torch.bfloat16, rotating = True+ )+ tier = vid._h3_placement_tier(+ quantised = engaged is not None,+ whole_set_sizes = (denoiser, whole[1]),+ pinned_sizes = (denoiser, pinned[1]),+ free_bytes = int(free_gb * 1000**3),+ speed_off = False,+ )+ expected = _LADDER.get((free_gb, te_scheme, "none" if engaged is None else "auto"))+ if expected is not None:+ assert tier == expected[1]+ if engaged is not None:+ assert tier in ("resident", "pinned", "stream")+++def test_speed_off_and_unreadable_cards_keep_their_placements():+ from core.inference import video as vid++ sizes = (20_300_000_000, 36_000_000_000)+ whole = (20_300_000_000, 47_000_000_000)+ assert (+ vid._h3_placement_tier(+ quantised = True,+ whole_set_sizes = whole,+ pinned_sizes = sizes,+ free_bytes = 500 * 1000**3,+ speed_off = True,+ )+ == "pinned"+ )+ assert (+ vid._h3_placement_tier(+ quantised = False,+ whole_set_sizes = whole,+ pinned_sizes = sizes,+ free_bytes = 500 * 1000**3,+ speed_off = True,+ )+ == "rotation"+ )+ for quantised, tier in ((True, "pinned"), (False, "rotation")):+ assert (+ vid._h3_placement_tier(+ quantised = quantised,+ whole_set_sizes = whole,+ pinned_sizes = sizes,+ free_bytes = None,+ speed_off = False,+ )+ == tier+ )+++def test_the_quantised_denoiser_is_sized_on_its_payload_not_its_logical_shape():+ """A torchao weight reports the bf16 shape through numel / element_size, which priced the+ 20.3 GB hosted denoiser at ~40 GB and pushed a card that could pin it onto the streamed tier."""+ from core.inference.diffusion_prequant import tensor_payload_bytes++ class _Inner:+ def __init__(self, n):+ self._n = n++ def numel(self):+ return self._n++ def element_size(self):+ return 1++ class QuantisedWeight:+ def __init__(self):+ self.qdata = _Inner(1000)+ self.scale = types.SimpleNamespace(numel = lambda: 10, element_size = lambda: 4)++ def __tensor_flatten__(self):+ return ["qdata", "scale"], None++ def numel(self):+ return 1000++ def element_size(self):+ return 2++ assert tensor_payload_bytes(QuantisedWeight()) == 1000 + 40+ assert tensor_payload_bytes(torch.empty(10, dtype = torch.bfloat16)) == 20+++def test_a_32gb_card_is_admitted_with_the_streamed_quantised_denoiser():+ from core.inference.video_minimax_h3 import estimate_h3_diffusers_vram_gb++ free_5090_gb = 33.56+ streamed = estimate_h3_diffusers_vram_gb(+ 960, 544, 124, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True+ )+ assert streamed + 0 <= free_5090_gb + 0.25+ assert streamed == pytest.approx(29.0, abs = 0.01)+ pinned = estimate_h3_diffusers_vram_gb(+ 960, 544, 124, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_pinned = True+ )+ rotating_bf16 = estimate_h3_diffusers_vram_gb(960, 544, 124, text_encoder_gb = 27.2)+ assert pinned > free_5090_gb + 0.25 and rotating_bf16 > free_5090_gb + 0.25+ assert estimate_h3_diffusers_vram_gb(960, 544, 124) == pytest.approx(73.68, abs = 0.02)+ long_clip = estimate_h3_diffusers_vram_gb(+ 1344, 768, 345, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True+ )+ assert long_clip > streamed+++def test_the_host_floor_counts_the_pinned_staging_copy_of_a_streamed_denoiser():+ """Group offloading stages the streamed denoiser in pinned host memory beside the copy it was+ built as: measured 80.2 GB peak host RSS against the 64.5 GB a single count asks for."""+ from core.inference.video_minimax_h3 import estimate_h3_diffusers_host_ram_gb++ single = estimate_h3_diffusers_host_ram_gb(33.5, text_encoder_gb = 27.2, transformer_gb = 20.3)+ streamed = estimate_h3_diffusers_host_ram_gb(+ 33.5, text_encoder_gb = 27.2, transformer_gb = 20.3, transformer_streamed = True+ )+ assert single == pytest.approx(64.5, abs = 0.01)+ assert streamed == pytest.approx(84.8, abs = 0.01) and streamed >= 80.2+ assert estimate_h3_diffusers_host_ram_gb(33.5) == pytest.approx(150.0, abs = 0.01)+++def test_a_streamed_load_keeps_its_host_floor_when_free_vram_later_climbs():+ """Streamed while another tenant held the card; that tenant leaves before the render."""+ from core.inference.video_minimax_h3 import estimate_h3_diffusers_host_ram_gb++ streamed = estimate_h3_diffusers_host_ram_gb(+ 140.0, text_encoder_gb = 66.7, transformer_gb = 20.3, transformer_streamed = True+ )+ assert streamed == pytest.approx(124.3, abs = 0.01)+ assert (+ estimate_h3_diffusers_host_ram_gb(140.0, text_encoder_gb = 66.7, transformer_gb = 20.3) == 85.0+ )+++def test_the_generate_preflight_reads_the_streamed_fact_off_the_state():+ import inspect++ from core.inference import video as vid++ fields = set(vid._VideoLoadState.__dataclass_fields__)+ assert {"denoiser_streamed", "denoiser_host_copy"} <= fields+ source = inspect.getsource(vid.VideoBackend.generate)+ # VRAM floor: any streaming. Host floor: only a full pinned host copy doubles the denoiser.+ assert (+ source.count('transformer_streamed = bool(getattr(state, "denoiser_streamed", False))') == 1+ )+ assert (+ source.count('transformer_streamed = bool(getattr(state, "denoiser_host_copy", False))')+ == 1+ )+ load = inspect.getsource(vid)+ assert 'denoiser_host_copy = denoiser_streamed == "stream"' in load+++class _Net(torch.nn.Module):+ def __init__(+ self,+ d = 256,+ n = 4,+ ):+ super().__init__()+ self.blocks = torch.nn.ModuleList(+ [+ torch.nn.Sequential(torch.nn.Linear(d, d), torch.nn.GELU(), torch.nn.Linear(d, d))+ for _ in range(n)+ ]+ )+ self.proj_out = torch.nn.Linear(d, d)++ def pad_small_m(self):+ # What the hosted H3 load does to its small-M int8 linears: a wrapper that forwards ``weight`` to the Linear.+ from core.inference.diffusion_quant_pad import PadToMinM+ self.proj_out = PadToMinM(self.proj_out)+ return self++ def forward(self, x):+ for block in self.blocks:+ x = x + block(x)+ return self.proj_out(x)+++class _UserHook:+ def __init__(self, model):+ self.model = model+ self.hook = types.SimpleNamespace(other_hooks = [])+ self.removed = False++ def remove(self):+ self.removed = True++ def offload(self):+ self.model.to("cpu")+++def _torchao_configs():+ torchao = pytest.importorskip("torchao")+ from torchao.quantization import (+ Float8DynamicActivationFloat8WeightConfig,+ Int8DynamicActivationInt8WeightConfig,+ PerRow,+ )++ configs = {}+ import inspect++ int8_params = inspect.signature(Int8DynamicActivationInt8WeightConfig).parameters+ if "version" in int8_params:+ configs["int8_v2"] = lambda: Int8DynamicActivationInt8WeightConfig(version = 2)+ try:+ Int8DynamicActivationInt8WeightConfig(version = 1)+ from torchao.quantization import linear_activation_quantized_tensor # noqa: F401+ configs["int8_v1"] = lambda: Int8DynamicActivationInt8WeightConfig(version = 1)+ except Exception: # noqa: BLE001 -- torchao >= 0.18 removed the v1 classes+ pass+ else:+ configs["int8_v1"] = lambda: Int8DynamicActivationInt8WeightConfig()+ if torch.cuda.get_device_capability() >= (8, 9):+ configs["fp8_row"] = lambda: Float8DynamicActivationFloat8WeightConfig(granularity = PerRow())+ del torchao+ return configs+++def _stream(module, stream_prequantized_module):+ rotating = torch.nn.Linear(8, 8).to("cuda")+ hooks = [_UserHook(module), _UserHook(rotating)]+ manager = types.SimpleNamespace(model_hooks = list(hooks))+ mode = stream_prequantized_module(manager, module, "cuda")+ assert hooks[0].removed and [h.model for h in manager.model_hooks] == [rotating]+ return mode, rotating++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")[email protected]("grad_mode", ["inference_mode", "no_grad"])+def test_a_streamed_torchao_denoiser_matches_the_resident_one_bit_for_bit(monkeypatch, grad_mode):+ """Studio renders under inference_mode, and under no_grad where a torchao denoiser streams."""+ pytest.importorskip("diffusers")+ pytest.importorskip("torchao")+ monkeypatch.undo()+ from torchao.quantization import quantize_++ import core.inference.prequant_legacy_int8 as legacy+ from core.inference.diffusion_prequant import (+ stream_prequantized_module,+ torchao_group_offload_supported,+ )++ if not torchao_group_offload_supported():+ pytest.skip("this diffusers cannot group-offload torchao weights")+ render = torch.inference_mode if grad_mode == "inference_mode" else torch.no_grad++ torch.manual_seed(0)+ base = _Net().to(torch.bfloat16)+ x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")+ ran = []+ for name, config in _torchao_configs().items():+ resident = copy.deepcopy(base)+ quantize_(resident, config())+ resident.pad_small_m()+ v1 = type(resident.blocks[0][0].weight).__name__ == "LinearActivationQuantizedTensor"+ if v1:+ assert legacy.convert_legacy_int8_weights(resident) > 0+ resident.to("cuda")+ with render():+ expected = resident(x)+ del resident++ streamed = copy.deepcopy(base)+ quantize_(streamed, config()) # built on the CPU, like the seeded H3 load+ streamed.pad_small_m()+ mode, rotating = _stream(streamed, stream_prequantized_module)+ assert mode == "stream", (name, mode)+ with render():+ outs = [streamed(x) for _ in range(3)]+ for out in outs:+ assert torch.equal(out, expected), name+ assert rotating.weight.device.type == "cpu"+ ran.append((name, type(streamed.blocks[0][0].weight).__name__, mode))+ print("RAN", ran)+ assert ran++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_a_hosted_denoiser_left_trainable_by_its_load_still_streams(monkeypatch):+ """The hosted checkpoint loads through load_state_dict(assign=True), which leaves every weight+ requiring grad; quantize_ does not, so the test above never saw it. The real H3 load failed its+ first onload with Int8Tensor's unimplemented aten.view."""+ pytest.importorskip("diffusers")+ pytest.importorskip("torchao")+ monkeypatch.undo()+ from torchao.quantization import quantize_++ from core.inference.diffusion_prequant import (+ stream_prequantized_module,+ torchao_group_offload_supported,+ )++ if not torchao_group_offload_supported():+ pytest.skip("this diffusers cannot group-offload torchao weights")+ configs = {k: v for k, v in _torchao_configs().items() if k != "int8_v1"}+ torch.manual_seed(0)+ base = _Net().to(torch.bfloat16)+ x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")+ for name, config in configs.items():+ resident = copy.deepcopy(base)+ quantize_(resident, config())+ resident.to("cuda")+ with torch.no_grad():+ expected = resident(x)+ del resident++ streamed = copy.deepcopy(base)+ quantize_(streamed, config())+ streamed.requires_grad_(True)+ mode, _ = _stream(streamed, stream_prequantized_module)+ assert mode == "stream", (name, mode)+ with torch.no_grad():+ outs = [streamed(x) for _ in range(2)]+ for out in outs:+ assert torch.equal(out, expected), name++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")+def test_a_v1_int8_denoiser_that_cannot_be_rebuilt_moves_synchronously_and_exactly(monkeypatch):+ """The fallback when the Int8Tensor rebuild declines: synchronous copies, same values as the+ resident v1 module, including under inference_mode (the first thing it used to fail on)."""+ pytest.importorskip("diffusers")+ pytest.importorskip("torchao")+ monkeypatch.undo()+ from torchao.quantization import quantize_++ import core.inference.prequant_legacy_int8 as legacy+ from core.inference.diffusion_prequant import (+ stream_prequantized_module,+ torchao_group_offload_supported,+ )++ configs = _torchao_configs()+ if "int8_v1" not in configs or not torchao_group_offload_supported():+ pytest.skip("needs torchao <= 0.17 (v1 int8) and torchao-aware group offload")+ monkeypatch.setattr(legacy, "convert_legacy_int8_weights", lambda module: 0)+ torch.manual_seed(0)+ base = _Net().to(torch.bfloat16)+ x = torch.randn(64, 256, dtype = torch.bfloat16, device = "cuda")+ resident = copy.deepcopy(base)+ quantize_(resident, configs["int8_v1"]())+ resident.to("cuda")+ with torch.inference_mode():+ expected = resident(x)+ streamed = copy.deepcopy(base)+ quantize_(streamed, configs["int8_v1"]())+ mode, _ = _stream(streamed, stream_prequantized_module)+ assert mode == "sync"+ assert type(streamed.blocks[0][0].weight).__name__ == "LinearActivationQuantizedTensor"+ with torch.inference_mode():+ for _ in range(2):+ assert torch.equal(streamed(x), expected)+ assert streamed.blocks[-1][0].weight.device.type == "cpu"+++def test_a_streamed_torchao_render_runs_under_no_grad_and_the_rest_keep_inference_mode():+ """torchao's aliasing check fails a weight move under inference_mode, and a streamed denoiser+ moves weights inside the render; the image loader switches for the same reason."""+ import inspect++ from core.inference import video as vid++ source = inspect.getsource(vid.VideoBackend.generate)+ assert (+ 'if state.transformer_quant and getattr(state, "denoiser_streamed", False)\n'+ " else torch.inference_mode()"+ ) in source+ assert "with grad_ctx, protect_ctx, progress_ctx(), sigma_ctx:" in source+++def test_a_failed_streaming_setup_raises_instead_of_pinning(monkeypatch):+ """Streaming is only chosen when pinning does not fit, so a failed setup must not fall back to pinning."""+ pytest.importorskip("diffusers")+ import diffusers.hooks+ import core.inference.diffusion_prequant as prequant++ def _boom(*args, **kwargs):+ raise ValueError("no group offload here")++ monkeypatch.setattr(prequant, "torchao_group_offload_supported", lambda: True)+ monkeypatch.setattr(diffusers.hooks, "apply_group_offloading", _boom)+ module = torch.nn.Linear(8, 8)+ hook = _UserHook(module)+ manager = types.SimpleNamespace(model_hooks = [hook])+ with pytest.raises(RuntimeError, match = "group offloading could not be set up"):+ prequant.stream_prequantized_module(manager, module, "cpu")+ assert hook.removed and module.weight.device.type == "cpu"++[email protected](not torch.cuda.is_available(), reason = "needs CUDA")[email protected]("pin_all, expected", [(True, "stream"), (False, "stream_lazy")])+def test_the_stream_mode_says_whether_a_full_pinned_host_copy_exists(+ monkeypatch, pin_all, expected+):+ """The host floor doubles the denoiser only for a full up-front pin; lazy pinning holds one group at a time."""+ pytest.importorskip("diffusers")+ from torchao.quantization import quantize_++ import core.inference.diffusion_memory as mem+ from core.inference.diffusion_prequant import (+ stream_prequantized_module,+ torchao_group_offload_supported,+ )++ configs = _torchao_configs()+ if "int8_v2" not in configs or not torchao_group_offload_supported():+ pytest.skip("needs Int8Tensor and torchao-aware group offload")+ monkeypatch.setattr(mem, "_streamed_pin_plan", lambda *a, **k: (pin_all, pin_all))+ streamed = _Net().to(torch.bfloat16)+ quantize_(streamed, configs["int8_v2"]())+ mode, _ = _stream(streamed, stream_prequantized_module)+ assert mode == expecteddiff --git a/studio/backend/tests/test_quantised_streaming_capability.py b/studio/backend/tests/test_quantised_streaming_capability.pynew file mode 100644index 00000000000..d261d773123--- /dev/null+++ b/studio/backend/tests/test_quantised_streaming_capability.py@@ -0,0 +1,163 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""``/api/system.quantised_streaming``: whether group offload can stream torchao weights.++The picker offers MiniMax-H3's streamed 30 GiB tier only when this is true, so an install whose+diffusers predates torchao-aware group offload, or a GPU without INT8 cores, routes to the GGUF row+instead of a refused load.+"""++import ast+import sys+import types+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+_MAIN = (_BACKEND / "main.py").read_text(encoding = "utf-8")+++def _src(name: str) -> str:+ node = next(+ n for n in ast.walk(ast.parse(_MAIN)) if isinstance(n, ast.FunctionDef) and n.name == name+ )+ return ast.get_source_segment(_MAIN, node)+++def test_the_bit_is_published_on_the_system_route():+ assert '"quantised_streaming": _quantised_streaming()' in _src("get_system_info")+++def test_the_bit_defaults_to_unknown_until_resolved():+ assert "_quantised_streaming_capability: Optional[bool] = None" in _MAIN++[email protected]("cached, expected", [(None, False), (False, False), (True, True)])+def test_the_polled_reader_answers_from_the_cache_without_importing(monkeypatch, cached, expected):+ """A poll never imports diffusers: with every import poisoned it still answers the cached bit."""+ for name in ("diffusers", "diffusers.hooks", "torch", "core.inference.diffusion_prequant"):+ monkeypatch.setitem(sys.modules, name, None)+ reader = _src("_quantised_streaming")+ assert "import" not in reader+ namespace: dict = {"_quantised_streaming_capability": cached, "sys": sys}+ exec(reader, namespace) # noqa: S102 -- the real body+ assert namespace["_quantised_streaming"]() is expected+++def _exec_refresh(+ monkeypatch,+ supported,+ device_count = 1,+):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(torch.cuda, "is_available", lambda: device_count > 0)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: device_count)+ fake = types.ModuleType("core.inference.video")+ fake.h3_streamed_int8_supported = supported+ monkeypatch.setitem(sys.modules, "core.inference.video", fake)+ device = types.ModuleType("core.inference.diffusion_device")+ device.resolve_diffusion_device_target = lambda ordinal = None: ordinal+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_device", device)+ namespace: dict = {"_quantised_streaming_capability": None, "sys": sys, "Any": object}+ exec(_src("_probe_quantised_streaming"), namespace) # noqa: S102+ exec(_src("_refresh_quantised_streaming_capability"), namespace) # noqa: S102+ return namespace++[email protected]("supported", [True, False])+def test_the_refresh_caches_the_prequant_verdict(monkeypatch, supported):+ namespace = _exec_refresh(monkeypatch, lambda target = None: supported)+ exec(_src("_quantised_streaming"), namespace) # noqa: S102+ assert namespace["_refresh_quantised_streaming_capability"]() is supported+ assert namespace["_quantised_streaming_capability"] is supported+ assert namespace["_quantised_streaming"]() is supported++[email protected]("loaded", [True, False])+def test_a_cold_warm_resolves_it_once_a_load_has_loaded_the_stack(monkeypatch, loaded):+ """UNSLOTH_STUDIO_DISABLE_TORCH_WARM=1 skips the warm refresh; a later load must still publish the bit."""+ names = ("torch", "diffusers", "torchao", "core.inference.video")+ for name in names:+ if loaded:+ monkeypatch.setitem(sys.modules, name, types.ModuleType(name))+ else:+ monkeypatch.delitem(sys.modules, name, raising = False)+ calls: list = []+ namespace: dict = {"_quantised_streaming_capability": None, "sys": sys}+ exec(_src("_quantised_streaming"), namespace) # noqa: S102+ namespace["_refresh_quantised_streaming_capability"] = lambda: calls.append(1) or True+ assert namespace["_quantised_streaming"]() is loaded+ assert calls == ([1] if loaded else [])+++def test_a_mixed_host_offers_the_tier_only_when_every_card_qualifies(monkeypatch):+ """A load can be pinned to any card; one resolving to float16 would keep the bf16 denoiser."""+ namespace = _exec_refresh(monkeypatch, lambda target = None: target != 1, device_count = 2)+ assert namespace["_refresh_quantised_streaming_capability"]() is False+ namespace = _exec_refresh(monkeypatch, lambda target = None: True, device_count = 2)+ assert namespace["_refresh_quantised_streaming_capability"]() is True+++def test_the_post_warm_worker_resolves_it_behind_the_torch_guard():+ warm = _src("_post_warm_background_work")+ assert "_refresh_quantised_streaming_capability()" in warm+ guard = warm.index('if "torch" in sys.modules:')+ assert guard < warm.index("_refresh_quantised_streaming_capability()")+++def _group_offloading(**attrs):+ hooks = types.ModuleType("diffusers.hooks")+ hooks.apply_group_offloading = lambda *a, **k: None+ go = types.ModuleType("diffusers.hooks.group_offloading")+ for name, value in attrs.items():+ setattr(go, name, value)+ hooks.group_offloading = go+ return hooks, go+++ "attrs, expected",+ [+ ({"_is_torchao_tensor": lambda t: False, "_swap_torchao_tensor": lambda *a: None}, True),+ ({}, False),+ ],+)+def test_the_probe_follows_diffusers_group_offload(monkeypatch, attrs, expected):+ from core.inference.diffusion_prequant import torchao_group_offload_supported++ hooks, go = _group_offloading(**attrs)+ diffusers = types.ModuleType("diffusers")+ diffusers.hooks = hooks+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)+ monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)+ monkeypatch.setitem(sys.modules, "diffusers.hooks.group_offloading", go)+ assert torchao_group_offload_supported() is expected+++ "device, dtype_name, streaming, readable, expected",+ [+ ("cuda", "bfloat16", True, True, True),+ # Pre-Ampere NVIDIA resolves to float16, and its INT8 path is not supported.+ ("cuda", "float16", True, True, False),+ ("cuda", "bfloat16", False, True, False),+ # An unreadable hosted INT8 keeps auto on bf16, whose load the tier cannot hold.+ ("cuda", "bfloat16", True, False, False),+ ("mps", "bfloat16", True, True, False),+ ("cpu", "float32", True, True, False),+ ],+)+def test_the_streamed_tier_needs_an_int8_capable_gpu(+ monkeypatch, device, dtype_name, streaming, readable, expected+):+ torch = pytest.importorskip("torch")+ from core.inference import diffusion_prequant, video++ monkeypatch.setattr(diffusion_prequant, "torchao_group_offload_supported", lambda: streaming)+ monkeypatch.setattr(+ diffusion_prequant, "restricted_prequant_load_supported", lambda *a, **k: readable+ )+ target = types.SimpleNamespace(device = device, dtype = getattr(torch, dtype_name))+ assert video.h3_streamed_int8_supported(target) is expecteddiff --git a/studio/backend/tests/test_video_prequant.py b/studio/backend/tests/test_video_prequant.pyindex cdd64d4036b..cab0a0d4c2d 100644--- a/studio/backend/tests/test_video_prequant.py+++ b/studio/backend/tests/test_video_prequant.py@@ -38,9 +38,11 @@ def _assume_the_restricted_load_is_available(monkeypatch):Without this, a machine with no (or a skewed) torchao turns every hosted-prequant decisionbelow into "keep the dense weights". The capability is covered in test_diffusion_prequant.py."""import core.inference.diffusion_prequant as _pq+monkeypatch.setattr(_pq, "restricted_prequant_load_supported", lambda scheme = None, filename = None: True)+ monkeypatch.setattr(_pq, "torchao_group_offload_supported", lambda: True)from core.inference.video_families import (@@ -956,12 +958,16 @@ def test_the_auto_fallback_is_declined_when_nothing_can_answer(monkeypatch):assert ask() is Nonemonkeypatch.setattr(vid, "_h3_auto_precision_ok", lambda target = None: True, raising = False)- # An unreadable card decides nothing.+ # Unanswerable memory still takes the hosted checkpoint (it streams) ...monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: None)- assert ask() is None- # Neither does an unanswerable size estimate.+ assert ask() == vid.H3_AUTO_FALLBACK_SCHEMEmonkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: 80 * 1000**3)monkeypatch.setattr(vid, "_h3_planned_denoiser_bytes", lambda *a, **k: None)+ assert ask() == vid.H3_AUTO_FALLBACK_SCHEME+ # ... unless this diffusers cannot stream torchao, leaving only the pin.+ import core.inference.diffusion_prequant as pq++ monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: False)assert ask() is Nonemonkeypatch.undo()@@ -1032,20 +1038,43 @@ def test_the_automatic_substitution_needs_the_exact_base_model(monkeypatch):assert ask(None) is None-def test_the_fallback_is_declined_when_the_hosted_denoiser_cannot_be_pinned(monkeypatch):- """Taking the hosted checkpoint means PINNING it -- a torchao module does not survive the- offload rotation's mid-block move -- and a pinned denoiser turns the memory floor from a max- into a sum.+def test_auto_keeps_int8_on_every_card_size_and_lets_the_fit_pick_the_placement(monkeypatch):+ """The fit decides the TIER, not the precision. A 32 GB card is exactly where the released+ denoiser costs the most (it rides the CPU-offload rotation uncompiled and the generate-time+ floor then refuses it outright), so a small card is never a reason to take bf16. The hosted+ denoiser streams block by block through group offloading where it cannot be pinned."""+ fam, torch, vid = _shared_setup_1()+ monkeypatch.setattr(vid, "_h3_auto_precision_ok", lambda target = None: True, raising = False)++ def ask(free_gb, te_scheme):+ monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: int(free_gb * 1000**3))+ return vid._h3_auto_denoiser_scheme(+ fam,+ target = None,+ dtype = torch.bfloat16,+ device = "cuda",+ te_scheme = te_scheme,+ task = "fl2va",+ base_repo = fam.base_repo,+ )++ for free_gb in (24, 32, 48, 80, 141):+ for te_scheme in ("int8", None):+ assert ask(free_gb, te_scheme) == vid.H3_AUTO_FALLBACK_SCHEME, (free_gb, te_scheme)+++def test_without_group_offload_the_hosted_denoiser_still_has_to_fit_pinned(monkeypatch):+ """A diffusers that cannot stream a torchao module leaves PINNING as the only placement, and a+ pinned denoiser adds to the larger rotating component. There the old rule stands: where the+ pin does not fit, the released denoiser in the rotation is the configuration that runs."""+ import core.inference.diffusion_prequant as pq- With text_encoder_quant="none" the conditioner stays dense, so that sum is BIGGER than the- rotation it replaces: the card renders today and would refuse every generation afterwards.- Where the replacement does not fit either, the released denoiser in the rotation is the- configuration that still runs, so auto keeps it."""fam, torch, vid = _shared_setup_1()monkeypatch.setattr(vid, "_h3_auto_precision_ok", lambda target = None: True, raising = False)- monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: 80 * 1000**3)+ monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: False)- def ask(te_scheme):+ def ask(free_gb, te_scheme):+ monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: int(free_gb * 1000**3))return vid._h3_auto_denoiser_scheme(fam,target = None,@@ -1056,10 +1085,53 @@ def test_the_fallback_is_declined_when_the_hosted_denoiser_cannot_be_pinned(monkbase_repo = fam.base_repo,)- # Dense conditioner: 107.1 GB pinned against 80 GB free, so the substitution buys a refusal.- assert ask(None) is None- # Quantized conditioner: 67.5 GB pinned, which is what the fallback exists for.- assert ask("int8") == vid.H3_AUTO_FALLBACK_SCHEME+ # Pinned int8 needs 20.3 + max(27.2 conditioner, 11.1 VAEs) + headroom, not the sum of all three.+ pinned = vid._h3_planned_denoiser_bytes(+ fam, te_scheme = "int8", dtype = torch.bfloat16, rotating = True+ )+ need_gb = (20.3 * 1000**3 + pinned[1]) / 1000**3+ assert ask(need_gb + 0.1, "int8") == vid.H3_AUTO_FALLBACK_SCHEME+ assert ask(need_gb - 0.1, "int8") is None+ assert ask(32, "int8") is None+ # Dense conditioner: 66.7 GB rotating beside a pinned denoiser does not fit 80 GB either.+ assert ask(80, None) is None+ assert ask(141, None) == vid.H3_AUTO_FALLBACK_SCHEME+++def test_a_pinned_denoiser_is_sized_beside_the_larger_rotating_component_not_their_sum():+ """The conditioner and the VAEs stay in the rotation beside a pinned denoiser, and its 40 GB+ reserve margin evicts one before the other is placed. Summing them over-counted by the whole+ VAE term (~11 GB) and refused pins that fit. Only the whole-set-resident question sums."""+ import torch++ from core.inference.video import _h3_dense_denoiser_resident_bytes, _h3_planned_denoiser_bytes++ fam = _h3_family()+ te_gb, vae_gb = 27.2, fam.bf16_components_gb[2]++ class _Real:+ def parameters(self):+ return iter(())++ def buffers(self):+ return iter([torch.empty(1024, dtype = torch.uint8)])++ for sizer in (+ lambda **kw: _h3_planned_denoiser_bytes(fam, te_scheme = "int8", dtype = torch.bfloat16, **kw),+ lambda **kw: _h3_dense_denoiser_resident_bytes(+ fam, denoiser = _Real(), te_scheme = "int8", dtype = torch.bfloat16, **kw+ ),+ ):+ whole = sizer()[1]+ rotating = sizer(rotating = True)[1]+ assert whole - rotating == int((te_gb + vae_gb) * 1000**3) - int(+ max(te_gb, vae_gb) * 1000**3+ )+ whole = _h3_planned_denoiser_bytes(fam, te_scheme = None, dtype = torch.bfloat16)[1]+ rotating = _h3_planned_denoiser_bytes(fam, te_scheme = None, dtype = torch.bfloat16, rotating = True)[+ 1+ ]+ assert abs((whole - rotating) - vae_gb * 1000**3) < 1000def test_the_fallback_is_resolved_before_the_download_is_planned(monkeypatch):@@ -1801,8 +1873,13 @@ def test_unreadability_is_reported_only_when_it_is_what_kept_bfloat16(monkeypatcfired.clear()assert ask(speed_mode = "off") is None and fired == []assert ask(base_repo = "someone/MiniMax-H3") is None and fired == []+ # A too-small card now streams, so the unreadable checkpoint is the reason again.monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: 10 * 1000**3)+ assert ask() is None and fired == [True]+ fired.clear()+ monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: False)assert ask() is None and fired == []+ monkeypatch.setattr(pq, "torchao_group_offload_supported", lambda: True)monkeypatch.setattr(pq, "restricted_prequant_load_supported", lambda *a, **k: True)monkeypatch.setattr(vid, "_h3_free_device_bytes", lambda device: 80 * 1000**3)assert ask() == vid.H3_AUTO_FALLBACK_SCHEME and fired == []diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.tsindex 24f991faba1..22edde8fe0f 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.check.ts@@ -946,6 +946,53 @@ assert.equal(.format,"bf16",);+// The 30 GiB tier streams an int8 denoiser, which needs group offload that swaps torchao weights+// (diffusers >= 0.40). Unknown or unsupported keeps the host on the runnable GGUF row.+for (const quantisedStreaming of [undefined, false]) {+ assert.equal(+ pickDefaultArtifact(h3, {+ gpuGb: 32,+ systemRamGb: 80,+ quantisedStreaming,+ isDownloaded: notDownloaded,+ }).format,+ "gguf",+ );+ assert.equal(+ curatedArtifactFitsDevice(H3, VIDEO_CATALOG, {+ gpuGb: 32,+ systemRamGb: 80,+ quantisedStreaming,+ }),+ false,+ );+}+assert.equal(+ pickDefaultArtifact(h3, {+ gpuGb: 32,+ systemRamGb: 80,+ quantisedStreaming: true,+ isDownloaded: notDownloaded,+ }).format,+ "bf16",+);+assert.equal(+ curatedArtifactFitsDevice(H3, VIDEO_CATALOG, {+ gpuGb: 32,+ systemRamGb: 80,+ quantisedStreaming: true,+ }),+ true,+);+// The resident tiers need no streaming.+assert.equal(+ curatedArtifactFitsDevice(H3, VIDEO_CATALOG, {+ gpuGb: 74,+ systemRamGb: 140,+ quantisedStreaming: false,+ }),+ true,+);// Qwen-Image-2512 BF16 (54 GB) misses a 24/48 GB budget but fits an 80 GB GPU (budget 56)// and wins there.diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.tsindex e27ab610078..f71b306e62d 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts@@ -50,7 +50,7 @@ export interface ModelArtifact {approxSizeGb?: number;/** Measured CPU-offload fit tiers. Any tier whose GPU and available-RAM floors are both met can* auto-route this artifact without the resident 70% rule. */- offloadFitTiers?: readonly { gpuGb: number; systemRamGb: number }[];+ offloadFitTiers?: readonly OffloadFitTier[];/** Extra search tokens beyond the id/label ("4bit", "nf4", ...). */keywords?: readonly string[];/** Parameter count of THIS artifact's checkpoint, for the row's size chip. Only a fallback: the@@ -436,7 +436,9 @@ export const VIDEO_CATALOG: CatalogGroup[] = [// mirror are decimal GB (video.py divides by 1_000_000_000), so never copy figures across:// converted, these are 79.5 / 150.3 and 132.1 / 85.9 GB, matching the estimators' 78.74 / 150// and 132 / 85. Copying applies the conversion twice and sends capable hosts to GGUF.+ // 30 GiB: streamed int8 denoiser + int8 conditioner, measured 27.6 GB peak VRAM on a 32 GiB-capped B200.offloadFitTiers: [+ { gpuGb: 30, systemRamGb: 80, requiresQuantisedStreaming: true },{ gpuGb: 74, systemRamGb: 140 },{ gpuGb: 123, systemRamGb: 80 },],@@ -1087,6 +1089,18 @@ export function groupMatchesQuery(group: CatalogGroup, query: string): boolean {}+export interface OffloadFitTier {+ gpuGb: number;+ systemRamGb: number;+ /** Only offered where `/api/system.quantised_streaming` says group offload streams torchao weights. */+ requiresQuantisedStreaming?: boolean;+}++function offloadTierMet(tier: OffloadFitTier, budget: DeviceBudget): boolean {+ if (tier.requiresQuantisedStreaming && budget.quantisedStreaming !== true) return false;+ return budget.gpuGb >= tier.gpuGb && budget.systemRamGb >= tier.systemRamGb;+}+export interface DeviceBudget {/** Total GPU memory in GB (0/undefined = unknown or none). */gpuGb: number;@@ -1098,6 +1112,8 @@ export interface DeviceBudget {gpuCount?: number;/** Dense quant schemes this host runs, best first; empty keeps the bf16 sizing rule. */denseQuantSchemes?: readonly string[];+ /** Group offload can stream torchao weights; absent = unknown, so streamed tiers are not offered. */+ quantisedStreaming?: boolean;}/** GGUF fit, delegated to the one formula the Hub badge already uses. Its old private rule@@ -1291,7 +1307,7 @@ function fitsArtifactBudget(): boolean {if (artifact.offloadFitTiers?.length) {return artifact.offloadFitTiers.some(- (tier) => budget.gpuGb >= tier.gpuGb && budget.systemRamGb >= tier.systemRamGb,+ (tier) => offloadTierMet(tier, budget),);}const allowanceGb = budget.gpuGb * 0.7;@@ -1412,7 +1428,7 @@ export function catalogGroupFitsDevice(if (a.format === "gguf") return true;if (a.offloadFitTiers?.length) {return a.offloadFitTiers.some(- (tier) => budget.gpuGb >= tier.gpuGb && budget.systemRamGb >= tier.systemRamGb,+ (tier) => offloadTierMet(tier, budget),);}// The same quantised sizing the row badge and pickDefaultArtifact use: the dense figure woulddiff --git a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsxindex 1e35e5a0b58..72290833128 100644--- a/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx+++ b/studio/frontend/src/features/model-picker/components/model-selector/pickers.tsx@@ -953,12 +953,14 @@ function artifactBudget(gpu: {memoryTotalGb: number;systemRamAvailableGb: number;denseQuantSchemes?: readonly string[];+ quantisedStreaming?: boolean;}): DeviceBudget {return {gpuGb: gpu.memoryTotalGb,systemRamGb: gpu.systemRamAvailableGb,// Judges a pre-quantised row by that checkpoint's size, not the bf16 shards it replaces.denseQuantSchemes: gpu.denseQuantSchemes,+ quantisedStreaming: gpu.quantisedStreaming,};}diff --git a/studio/frontend/src/hooks/use-gpu-info.ts b/studio/frontend/src/hooks/use-gpu-info.tsindex 8c72680ef6b..ef6cbe1b32e 100644--- a/studio/frontend/src/hooks/use-gpu-info.ts+++ b/studio/frontend/src/hooks/use-gpu-info.ts@@ -54,6 +54,8 @@ export interface GpuInfo {denseQuantSchemes: readonly string[];/** False until system info arrives, and on backends that do not report it. */nvfp4Diffusion: boolean;+ /** Group offload can stream torchao weights. Absent or false until resolved and on older backends. */+ quantisedStreaming?: boolean;name: string;memoryTotalGb: number;memorySharedGb: number;@@ -92,6 +94,7 @@ const DEFAULT_GPU: GpuInfo = {denseQuantSupported: false,denseQuantSchemes: [],nvfp4Diffusion: false,+ quantisedStreaming: false,name: "Unknown",memoryTotalGb: 0,memorySharedGb: 0,@@ -120,6 +123,7 @@ function toGpuInfo(denseQuantSupported: data?.dense_quant_supported === true,denseQuantSchemes: normalizeDenseQuantSchemes(data?.dense_quant_schemes),nvfp4Diffusion: data?.nvfp4_diffusion === true,+ quantisedStreaming: data?.quantised_streaming === true,cpuCore: data?.cpu?.physical_count ?? 0,cpuThread: data?.cpu?.logical_count ?? 0,systemRamAvailableGb: data?.memory?.available_gb ?? 0,diff --git a/studio/frontend/src/hooks/use-system.ts b/studio/frontend/src/hooks/use-system.tsindex 2083b6031a6..cd4e68b6e70 100644--- a/studio/frontend/src/hooks/use-system.ts+++ b/studio/frontend/src/hooks/use-system.ts@@ -61,6 +61,8 @@ export interface SystemInfoResponse {dense_quant_schemes?: string[];/** Absent on older backends, where readers treat it as off. */nvfp4_diffusion?: boolean;+ /** Whether group offload can stream torchao weights. Absent on older backends. */+ quantised_streaming?: boolean;uptime_seconds: number | null;cpu: {logical_count: number;