Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: serve the hosted Wan2.2-A14B int8 pair to an offloaded load as torchao-free W8A8 layers (#15240)
Daniel Han1b887847e09d
diff --git a/studio/backend/core/inference/diffusion_native_hosted.py b/studio/backend/core/inference/diffusion_native_hosted.pyindex d672b9c988c..a656598b0dd 100644--- a/studio/backend/core/inference/diffusion_native_hosted.py+++ b/studio/backend/core/inference/diffusion_native_hosted.py@@ -1,12 +1,16 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""An offloaded int8 denoiser read from the hosted ConvRot int8 checkpoint as native Linears.+"""An offloaded int8 denoiser read from the hosted int8 checkpoint as native Linears.Offload hooks cannot move torchao weights, so an offloaded load used to fetch the dense bf16 shards and quantise themat load (LTX-2.5: 36 GB staged, ~100 s of build, then a 23 GB cache write). The hosted file already holds per-row int8codes and float32 scales of the same ConvRot-rotated weights, which is exactly what ``NativeWeightOnlyLinear`` stores:-this reads them into native Linears with the same layer set, rotation and activation path as the runtime build."""+this reads them into native Linears with the same layer set, rotation and activation path as the runtime build.++Wan2.2-T2V-A14B's hosted int8 pair is an older torchao W8A8 ``.pt`` pickle (no rotation, per-row scales, per-token+int8 activations): ``_load_native_from_pickle`` reads it into the same native Linears unrotated. Its codes are the+resident torchao load's, not the runtime ConvRot build's, so its frames differ from the native cache's."""from __future__ import annotations@@ -32,10 +36,15 @@ def native_hosted_enabled() -> bool:def native_hosted_family(fam: Any) -> bool:- """Whether ``fam`` opted in (its hosted int8 file is ConvRot safetensors) and the switch is on."""+ """Whether ``fam`` opted in (ConvRot safetensors or an unrotated W8A8 pickle) and the switch is on."""return bool(getattr(fam, "native_offload_from_hosted_int8", False)) and native_hosted_enabled()+def native_hosted_pickle(fam: Any) -> bool:+ """Whether ``fam``'s hosted int8 file is a torchao W8A8 pickle (reading it needs a real torchao)."""+ return bool(getattr(fam, "native_offload_hosted_int8_pickle", False))++def _log(logger: Any, level: str, msg: str) -> None:if logger is not None:getattr(logger, level)("video.native_hosted: %s", msg)@@ -95,10 +104,23 @@ def load_native_from_hosted(family: Optional[str],component: str = "transformer",config_kwargs: Optional[dict] = None,+ base: Optional[str] = None,logger: Any = None,) -> Any:"""The native ``component`` built from hosted ``path``, or None (the caller falls back to the dense build). Neverraises. Refuses unless the file holds exactly the layers, rotation and activation path the runtime build makes."""+ if not str(path).endswith(".safetensors"):+ return _load_native_from_pickle(+ path,+ transformer_cls,+ config_source,+ spec = spec,+ family = family,+ component = component,+ config_kwargs = config_kwargs,+ base = base,+ logger = logger,+ )try:import time@@ -245,3 +267,227 @@ def load_native_from_hosted(except Exception as exc: # noqa: BLE001 - an unusable file falls back to the dense build_log(logger, "warning", f"hosted int8 read failed: {type(exc).__name__}: {str(exc)[:200]}")return None+++_PICKLE_ACT_QUANT = "_int8_symm_per_token_reduced_range_quant"+++def _is_zero(t: Any) -> bool:+ import torch+ return t is None or (type(t) is torch.Tensor and not bool(torch.any(t != 0)))+++def int8_weight_parts(name: str, value: Any) -> Optional[tuple]:+ """``(codes [n, k] int8, scale [n])`` of a torchao W8A8 weight, None for a plain tensor; raises on other layouts.++ Reads the v1 ``LinearActivationQuantizedTensor`` (torchao <= 0.17) and the v2 ``Int8Tensor`` the legacy pickle+ decoder rebuilds on torchao >= 0.18. Anything this cannot describe exactly (asymmetric, weight-only, static+ activation scales, a non per-row block, a non per-token activation) raises."""+ import torch++ kind = type(value).__name__+ if type(value) in (torch.Tensor, torch.nn.Parameter):+ return None++ def bad(why: str) -> ValueError:+ return ValueError(f"hosted int8 weight {name!r} ({kind}): {why}")++ if kind == "Int8Tensor":+ qdata, scale = getattr(value, "qdata", None), getattr(value, "scale", None)+ if not _is_zero(getattr(value, "zero_point", None)):+ raise bad("asymmetric")+ if getattr(value, "act_quant_kwargs", None) is None:+ raise bad("weight-only (no activation quant)")+ granularity = getattr(value.act_quant_kwargs, "granularity", None)+ if type(granularity).__name__ != "PerRow":+ raise bad(f"activation granularity {granularity!r} is not per-token")+ for attr in ("act_quant_scale", "act_quant_zero_point", "act_pre_scale"):+ if getattr(value, attr, None) is not None:+ raise bad(f"{attr} is set")+ block = list(getattr(value, "block_size", ()) or ())+ elif kind == "LinearActivationQuantizedTensor":+ act = getattr(value, "input_quant_func", None)+ if getattr(act, "__name__", None) != _PICKLE_ACT_QUANT or getattr(+ value, "quant_kwargs", None+ ):+ raise bad("activation quant is not the per-token int8 one")+ aqt = getattr(value, "original_weight_tensor", None)+ impl = getattr(aqt, "tensor_impl", None)+ if type(getattr(impl, "_layout", None)).__name__ != "PlainLayout":+ raise bad("inner layout is not PlainLayout")+ qdata, scale = getattr(impl, "int_data", None), getattr(impl, "scale", None)+ if not _is_zero(getattr(impl, "zero_point", None)):+ raise bad("asymmetric")+ block = list(getattr(aqt, "block_size", ()) or ())+ else:+ raise bad("not an int8 W8A8 weight")+ if type(qdata) is not torch.Tensor or qdata.dtype != torch.int8 or qdata.dim() != 2:+ raise bad("codes are not a 2-D int8 tensor")+ n, k = qdata.shape+ if tuple(value.shape) != (n, k) or block != [1, k]:+ raise bad(f"block size {block} is not per-row for {(n, k)}")+ if type(scale) is not torch.Tensor or not scale.is_floating_point() or scale.numel() != n:+ raise bad("scale is not one float per output row")+ return qdata, scale.reshape(n)+++def _load_native_from_pickle(+ path: str,+ transformer_cls: Any,+ config_source: str,+ *,+ spec: dict,+ family: Optional[str],+ component: str,+ config_kwargs: Optional[dict],+ base: Optional[str],+ logger: Any,+) -> Any:+ """``load_native_from_hosted`` for a hosted torchao W8A8 ``.pt``: the same checks as the resident torchao read+ (format, scheme, base, filter, component, fingerprint), plus the runtime build's layer set. Never raises."""+ try:+ import time++ import torch++ from core._torchao_stub import is_stubbed++ t0 = time.perf_counter()+ if spec.get("scheme") != "int8" or not spec.get("act_int8"):+ _log(+ logger,+ "info",+ "hosted int8 checkpoint not used (the native build is not W8A8 int8)",+ )+ return None+ if is_stubbed("torchao"):+ _log(+ logger,+ "info",+ "hosted int8 checkpoint not used (torchao is stubbed; the pickle needs it)",+ )+ return None+ from accelerate import init_empty_weights++ from . import diffusion_prequant as pq+ from .diffusion_convrot import declares_rotation+ from .diffusion_native_quant import native_linear_class+ from .diffusion_transformer_quant import make_filter_fn++ # mmap: a full read holds both 14 GB experts as anonymous memory before offload pins its copies.+ ckpt = pq._read_prequant_for(str(path), "cpu", logger, host_copy_follows = True)+ if not pq._validate_checkpoint(+ ckpt,+ "int8",+ base,+ logger,+ min_features = int(spec["min_features"]),+ component = component,+ ):+ return None+ meta = ckpt.get("metadata") or {}+ if declares_rotation(meta):+ _log(+ logger,+ "info",+ "hosted int8 checkpoint not used (a rotated pickle is not read natively)",+ )+ return None+ if family is not None and meta.get("family") not in (None, family):+ _log(logger, "info", f"hosted int8 checkpoint not used (family {meta.get('family')!r})")+ return None+ state_dict = ckpt["state_dict"]+ if not pq._verify_packed_fingerprint(state_dict, meta, logger = logger, path = path):+ return None+ config = transformer_cls.load_config(+ config_source, subfolder = component, **(config_kwargs or {})+ )+ with init_empty_weights(include_buffers = False):+ model = transformer_cls.from_config(config)+ filter_fn = make_filter_fn(int(spec["min_features"]), tuple(spec["exclude_name_tokens"]))+ wanted = {name for name, mod in model.named_modules() if name and filter_fn(mod, name)}+ parts: dict = {}+ state: dict = {}+ for key, value in state_dict.items():+ got = int8_weight_parts(key, value)+ if got is None:+ state[key] = value+ elif key.endswith(".weight"):+ parts[key[: -len(".weight")]] = got+ else:+ raise ValueError(f"hosted int8 tensor {key!r} is not a Linear weight")+ if set(parts) != wanted:+ _log(+ logger,+ "info",+ f"hosted int8 checkpoint not used ({len(parts)} quantised layers, the native build makes "+ f"{len(wanted)}; e.g. {sorted(set(parts) ^ wanted)[:2]})",+ )+ return None+ del ckpt, state_dict+ modules = dict(model.named_modules())+ cls = native_linear_class()+ for fqn in sorted(parts):+ codes, scale = parts[fqn]+ lin = modules[fqn]+ if tuple(codes.shape) != (int(lin.out_features), int(lin.in_features)):+ _log(+ logger,+ "info",+ f"hosted int8 checkpoint not used ({fqn}: shape {tuple(codes.shape)})",+ )+ return None+ parent_name, _, leaf = fqn.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ stand_in = torch.nn.Linear(+ int(lin.in_features),+ int(lin.out_features),+ bias = lin.bias is not None,+ device = "meta",+ dtype = torch.bfloat16,+ )+ layer = cls(stand_in, "int8", act_int8 = True, rot_group = 0, codes = codes, scale = scale)+ setattr(parent, leaf, layer)+ state[fqn + ".weight_q"] = layer.weight_q+ state[fqn + ".weight_scale"] = layer.weight_scale+ model.load_state_dict(state, strict = True, assign = True)+ del state, parts+ if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):+ _log(logger, "warning", "hosted int8 checkpoint left meta tensors; not used")+ return None+ # The class's fp32 modules (Wan: time_embedder, scale_shift_table, norm2) stay fp32; other non-bf16 is refused.+ keep_fp32 = set(getattr(transformer_cls, "_keep_in_fp32_modules", None) or ())+ stray = [+ name+ for name, p in model.named_parameters()+ if p.is_floating_point()+ and p.dtype != torch.bfloat16+ and not (p.dtype == torch.float32 and keep_fp32 & set(name.split(".")))+ ]+ if stray:+ _log(+ logger,+ "info",+ f"hosted int8 checkpoint not used (non-bf16 dense tensor {stray[0]})",+ )+ return None+ model.eval()+ try:+ model._unsloth_runtime_quant = "int8"+ model._unsloth_runtime_quant_info = {+ "scheme": "int8",+ "native": True,+ "hosted_path": str(path),+ }+ except Exception: # noqa: BLE001+ pass+ _log(+ logger,+ "info",+ f"read {len(wanted)} int8 W8A8 layers of {component} from the hosted torchao checkpoint as native "+ f"Linears in {time.perf_counter() - t0:.1f} s (no dense shards, no runtime quantise)",+ )+ return model+ except Exception as exc: # noqa: BLE001 - an unusable file falls back to the dense build+ _log(logger, "warning", f"hosted int8 read failed: {type(exc).__name__}: {str(exc)[:200]}")+ return Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 92670fec5b1..4c3d6111128 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -4125,6 +4125,7 @@ def _read_native_quant_from_hosted(family = getattr(fam, "name", None),component = component,config_kwargs = config_kwargs,+ base = base or getattr(fam, "base_repo", None),logger = logger,)if module is None:@@ -4631,16 +4632,22 @@ def _video_native_hosted_scheme(fam: Any, scheme: Optional[str], target: Any) ->"""``int8`` when an offloaded load of ``scheme`` builds the W8A8 ConvRot native DiT that the family's hosted int8checkpoint already holds (``diffusion_native_hosted``), else None. The load re-checks the file; never raises."""try:- from .diffusion_native_hosted import native_hosted_family+ from .diffusion_native_hosted import native_hosted_family, native_hosted_picklefrom .diffusion_native_quant import int8_rotation_groupif scheme != TQ_INT8 or not native_hosted_family(fam):return Noneif getattr(target, "device", None) != "cuda":return None+ # A pickle unpickles through torchao: not on ROCm or a stubbed torchao (native_offload_host's set).+ if native_hosted_pickle(fam) and not native_offload_host(target):+ return Noneif native_quant_scheme(target, TQ_INT8, family = fam.name, offload = True) != TQ_INT8:return None- if not native_int8_act(target) or int(int8_rotation_group() or 0) <= 0:+ if not native_int8_act(target):+ return None+ # A ConvRot file only matches a rotated runtime build; a pickle is read unrotated, so ConvRot off keeps it.+ if not native_hosted_pickle(fam) and int(int8_rotation_group() or 0) <= 0:return Nonereturn TQ_INT8except Exception: # noqa: BLE001 -- unanswerable: the dense shards and runtime build, as beforediff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 3ea8c9819b3..10f52d5a98c 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -303,8 +303,9 @@ class VideoFamily:# Separate partition, served ONLY by its own (scheme, task, filename) row or it loads the wrong one.prequant_partition_tasks: tuple[str, ...] = field(default_factory = tuple)# An offloaded int8 load reads the hosted int8 file's ConvRot codes as native Linears (diffusion_native_hosted)- # instead of staging and quantising the dense shards. Only for a family whose hosted int8 file is ConvRot safetensors.+ # instead of staging and quantising the dense shards. Hosted file: ConvRot safetensors, or (pickle flag) unrotated W8A8.native_offload_from_hosted_int8: bool = False+ native_offload_hosted_int8_pickle: bool = False# Modular workflow loaded instead of a DiffusionPipeline; its routing blocks are not pruned.modular_workflow: Optional[str] = None# Released video and audio sigma shifts, when configurable.@@ -827,6 +828,9 @@ _FAMILIES: tuple[VideoFamily, ...] = (),# BOTH experts: the plan subtracts one denoiser term and this family builds two.prequant_resident_gb_by_scheme = (("nvfp4", 16.2), ("fp8", 29.2), ("int8", 28.8)),+ # Hosted int8 pair is an unrotated W8A8 pickle: offload reads 29 GB instead of 115 GB of dense experts.+ native_offload_from_hosted_int8 = True,+ native_offload_hosted_int8_pickle = True,# "wan2.2_t2v": ComfyUI's expert files (wan2.2_t2v_high_noise_14B_*.safetensors), paired at load.aliases = ("wan2.2-14b", "wan-t2v", "wan2.2-t2v", "wan2.2_t2v", "wan-t2v-a14b", "wan-a14b"),has_audio = False,diff --git a/studio/backend/tests/test_video_hosted_int8_offload.py b/studio/backend/tests/test_video_hosted_int8_offload.pynew file mode 100644index 00000000000..e9c92fb1246--- /dev/null+++ b/studio/backend/tests/test_video_hosted_int8_offload.py@@ -0,0 +1,594 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An offloaded Wan2.2-T2V-A14B int8 load reads the hosted torchao W8A8 pickle pair as native Linears through+main's hosted reader (``diffusion_native_hosted``) instead of fetching and quantising both dense experts."""++from __future__ import annotations++import contextlib+import enum+import sys+import types+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+pytest.importorskip("accelerate")+pytest.importorskip("torchao")++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_prequant as pq # noqa: E402+from core.inference import diffusion_transformer_quant as tq # noqa: E402+from core.inference import prequant_legacy_int8 as li # noqa: E402+from core.inference import diffusion_native_hosted as NH # noqa: E402+from core.inference import video as V # noqa: E402+from core.inference.diffusion_native_cache import native_spec # noqa: E402+from core.inference.diffusion_native_quant import is_native_linear # noqa: E402+from core.inference.video_families import detect_video_family # noqa: E402++FAMILY = "wan2.2-t2v-a14b"+BASE = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"+COMPONENTS = ("transformer", "transformer_2")++needs_decoder = pytest.mark.skipif(+ not (li.legacy_int8_classes_missing() and li.legacy_int8_decode_supported()),+ reason = "needs torchao >= 0.18 (the legacy int8 pickle decoder)",+)+++def _family():+ return detect_video_family(BASE, override = FAMILY)+++def tiny_config() -> dict:+ return dict(+ patch_size = (1, 2, 2),+ num_attention_heads = 2,+ attention_head_dim = 256,+ in_channels = 4,+ out_channels = 4,+ text_dim = 64,+ freq_dim = 32,+ ffn_dim = 1024,+ num_layers = 2,+ cross_attn_norm = True,+ qk_norm = "rms_norm_across_heads",+ eps = 1e-6,+ rope_max_seq_len = 64,+ )+++def v1_int8_classes():+ class _Base(torch.Tensor):+ __torch_function__ = torch._C._disabled_torch_function_impl++ @staticmethod+ def __new__(cls, shape, dtype):+ return torch.Tensor._make_wrapper_subclass(cls, shape, dtype = dtype)++ @classmethod+ def __torch_dispatch__(+ cls,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ raise RuntimeError("stand-in")++ def __reduce_ex__(self, proto):+ args = (+ type(self),+ self.dtype,+ tuple(self.size()),+ self.stride(),+ self.storage_offset(),+ self.layout,+ self.device,+ False,+ )+ return (+ torch._tensor._rebuild_from_type_v2,+ (torch._utils._rebuild_wrapper_subclass, type(self), args, dict(self.__dict__)),+ )++ names = {+ "torchao.quantization.linear_activation_quantized_tensor": "LinearActivationQuantizedTensor",+ "torchao.dtypes.affine_quantized_tensor": "AffineQuantizedTensor",+ "torchao.dtypes.uintx.plain_layout": "PlainAQTTensorImpl",+ }+ mods, made = {}, {}+ for mod, name in names.items():+ cls = type(name, (_Base,), {"__module__": mod, "__qualname__": name})+ mods[mod] = types.ModuleType(mod)+ setattr(mods[mod], name, cls)+ made[name] = cls+ layout = type(+ "PlainLayout", (), {"__module__": "torchao.dtypes.utils", "__qualname__": "PlainLayout"}+ )+ mods["torchao.dtypes.utils"] = types.ModuleType("torchao.dtypes.utils")+ mods["torchao.dtypes.utils"].PlainLayout = layout++ def _int8_symm_per_token_reduced_range_quant(x):+ raise RuntimeError("never called")++ _int8_symm_per_token_reduced_range_quant.__module__ = "torchao.quantization.quant_api"+ _int8_symm_per_token_reduced_range_quant.__qualname__ = (+ "_int8_symm_per_token_reduced_range_quant"+ )+ real_api = sys.modules.get("torchao.quantization.quant_api")+ api = types.ModuleType("torchao.quantization.quant_api")+ if real_api is not None:+ api.__dict__.update(real_api.__dict__)+ api._int8_symm_per_token_reduced_range_quant = _int8_symm_per_token_reduced_range_quant+ mods["torchao.quantization.quant_api"] = api+ saved = {k: sys.modules.get(k) for k in mods}+ sys.modules.update(mods)+ try:+ yield made, layout, _int8_symm_per_token_reduced_range_quant+ finally:+ for k, v in saved.items():+ if v is None:+ sys.modules.pop(k, None)+ else:+ sys.modules[k] = v+++def v1_int8_quantise(weight):+ w = weight.detach().float()+ amax = w.abs().amax(dim = 1)+ scale = (amax / 127.5).clamp(min = torch.finfo(torch.float32).eps).to(weight.dtype)+ codes = torch.round(w / scale.float()[:, None]).clamp(-128, 127).to(torch.int8)+ return codes, scale+++def write_hosted_v1(+ model,+ path,+ *,+ component,+ base = BASE,+ metadata = None,+ zero_point = None,+):+ from torchao.quantization.quant_primitives import ZeroPointDomain++ keep = tq.make_filter_fn(+ tq.DEFAULT_MIN_LINEAR_FEATURES,+ exclude_name_tokens = tq.exclude_tokens_for_scheme("int8", FAMILY),+ )+ quantised = {name: mod for name, mod in model.named_modules() if name and keep(mod, name)}+ written = {}+ with v1_int8_classes() as (made, layout, act):+ state = {}+ for key, value in model.state_dict().items():+ fqn = key[: -len(".weight")] if key.endswith(".weight") else None+ if fqn not in quantised:+ state[key] = value.detach().clone()+ continue+ codes, scale = v1_int8_quantise(value)+ n, k = codes.shape+ impl = made["PlainAQTTensorImpl"]((n, k), torch.int8)+ impl.__dict__.update(+ int_data = codes, scale = scale, zero_point = zero_point, _layout = layout()+ )+ aqt = made["AffineQuantizedTensor"]((n, k), value.dtype)+ aqt.__dict__.update(+ tensor_impl = impl,+ block_size = (1, k),+ quant_min = None,+ quant_max = None,+ zero_point_domain = ZeroPointDomain.NONE,+ )+ laqt = made["LinearActivationQuantizedTensor"]((n, k), value.dtype)+ laqt.__dict__.update(original_weight_tensor = aqt, input_quant_func = act, quant_kwargs = {})+ state[key] = laqt+ written[fqn] = (codes, scale)+ meta = {+ "scheme": "int8",+ "base_model_id": base,+ "family": FAMILY,+ "min_features": tq.DEFAULT_MIN_LINEAR_FEATURES,+ "component": component,+ **(metadata or {}),+ }+ torch.save({"format": pq.PREQUANT_FORMAT, "metadata": meta, "state_dict": state}, str(path))+ return written+++def dense_expert(seed: int):+ torch.manual_seed(seed)+ return diffusers.WanTransformer3DModel(**tiny_config()).to(torch.bfloat16).eval()+++def _spec():+ return native_spec("int8", FAMILY, act_int8 = True)+++def hosted(tmp_path):+ files, written, dense = {}, {}, {}+ cfg = tmp_path / "base"+ for seed, component in enumerate(COMPONENTS):+ dense[component] = dense_expert(seed)+ dense[component].save_config(str(cfg / component))+ path = tmp_path / f"{component}.pt"+ written[component] = write_hosted_v1(dense[component], path, component = component)+ files[component] = str(path)+ return types.SimpleNamespace(files = files, written = written, dense = dense, cfg = str(cfg))+++def _load(+ hosted,+ component,+ *,+ base = BASE,+ spec = None,+):+ return NH.load_native_from_hosted(+ hosted.files[component],+ diffusers.WanTransformer3DModel,+ hosted.cfg,+ spec = spec or _spec(),+ family = FAMILY,+ component = component,+ base = base,+ )+++@needs_decoder[email protected]("component", COMPONENTS)+def test_hosted_codes_and_scales_are_held_bit_for_bit(hosted, component):+ model = _load(hosted, component)+ assert model is not None+ layers = {n: m for n, m in model.named_modules() if is_native_linear(m)}+ assert set(layers) == set(hosted.written[component])+ for fqn, (codes, scale) in hosted.written[component].items():+ layer = layers[fqn]+ assert layer.act_int8 and layer.rot_group == 0 and layer.scheme == "int8"+ assert torch.equal(layer.weight_q, codes)+ assert torch.equal(layer.weight_scale.view(torch.float32), scale.float())+ dense = hosted.dense[component].state_dict()+ for key, value in model.state_dict().items():+ if key in dense:+ assert torch.equal(value, dense[key]), key+ assert model._unsloth_runtime_quant_info["native"] is True+ assert all(t.device.type == "cpu" for t in (*model.parameters(), *model.buffers()))+++@needs_decoder+def test_the_experts_stay_distinct(hosted):+ a, b = _load(hosted, "transformer"), _load(hosted, "transformer_2")+ qa = dict(a.named_buffers())["blocks.0.attn1.to_q.weight_q"]+ qb = dict(b.named_buffers())["blocks.0.attn1.to_q.weight_q"]+ assert not torch.equal(qa, qb)+++@needs_decoder+def test_forward_tracks_the_dense_expert(hosted):+ model = _load(hosted, "transformer")+ g = torch.Generator().manual_seed(7)+ hidden = torch.randn(1, 4, 3, 8, 8, generator = g).to(torch.bfloat16)+ text = torch.randn(1, 6, 64, generator = g).to(torch.bfloat16)+ t = torch.tensor([500])+ with torch.no_grad():+ ref = hosted.dense["transformer"](hidden, t, text, return_dict = False)[0].float()+ out = model(hidden, t, text, return_dict = False)[0].float()+ assert torch.isfinite(out).all()+ rel = float((out - ref).norm() / ref.norm())+ assert rel < 0.05, rel+++@needs_decoder+def test_a_rotated_pickle_is_refused(hosted):+ from core.inference.diffusion_convrot import CONVROT_KIND, ROTATION_KEY+ write_hosted_v1(+ hosted.dense["transformer"],+ Path(hosted.files["transformer"]),+ component = "transformer",+ metadata = {ROTATION_KEY: CONVROT_KIND},+ )+ assert _load(hosted, "transformer") is None+++@needs_decoder+def test_a_wrong_base_or_component_is_refused(hosted):+ path = Path(hosted.files["transformer"])+ write_hosted_v1(hosted.dense["transformer"], path, component = "transformer", base = "someone/else")+ assert _load(hosted, "transformer") is None+ write_hosted_v1(hosted.dense["transformer"], path, component = "transformer_2")+ assert _load(hosted, "transformer") is None+++@needs_decoder+def test_an_asymmetric_weight_is_refused(hosted):+ path = Path(hosted.files["transformer"])+ n = hosted.dense["transformer"].blocks[0].attn1.to_q.out_features+ write_hosted_v1(+ hosted.dense["transformer"],+ path,+ component = "transformer",+ zero_point = torch.ones(n, dtype = torch.int8),+ )+ assert _load(hosted, "transformer") is None+++@needs_decoder+def test_a_layer_set_other_than_the_native_builds_is_refused(hosted):+ spec = dict(_spec())+ spec["exclude_name_tokens"] = sorted(set(spec["exclude_name_tokens"]) | {"ffn"})+ assert _load(hosted, "transformer", spec = spec) is None+++@needs_decoder+def test_a_weight_only_native_build_or_stubbed_torchao_is_refused(hosted, monkeypatch):+ assert _load(hosted, "transformer", spec = native_spec("int8", FAMILY, act_int8 = False)) is None+ import core._torchao_stub as stub++ monkeypatch.setattr(stub, "is_stubbed", lambda name: True)+ assert _load(hosted, "transformer") is None+++needs_v1_classes = pytest.mark.skipif(+ li.legacy_int8_classes_missing(),+ reason = "needs torchao <= 0.17 (the v1 int8 classes)",+)+++@needs_v1_classes+def test_a_real_torchao_v1_pickle_reads_natively(tmp_path):+ """torchao <= 0.17 unpickles the hosted file as real v1 tensors: the same codes and scales come out."""+ from torchao.quantization import Int8DynamicActivationInt8WeightConfig, quantize_++ dense = dense_expert(0)+ dense.save_config(str(tmp_path / "base" / "transformer"))+ quantised = dense_expert(0)+ keep = tq.make_filter_fn(+ tq.DEFAULT_MIN_LINEAR_FEATURES,+ exclude_name_tokens = tq.exclude_tokens_for_scheme("int8", FAMILY),+ )+ quantize_(quantised, Int8DynamicActivationInt8WeightConfig(version = 1), filter_fn = keep)+ state = {k: v for k, v in quantised.state_dict().items()}+ meta = {+ "scheme": "int8",+ "base_model_id": BASE,+ "family": FAMILY,+ "min_features": tq.DEFAULT_MIN_LINEAR_FEATURES,+ "component": "transformer",+ }+ path = tmp_path / "transformer.pt"+ torch.save({"format": pq.PREQUANT_FORMAT, "metadata": meta, "state_dict": state}, str(path))+ model = NH.load_native_from_hosted(+ str(path),+ diffusers.WanTransformer3DModel,+ str(tmp_path / "base"),+ spec = _spec(),+ family = FAMILY,+ component = "transformer",+ base = BASE,+ )+ assert model is not None+ layers = {n: m for n, m in model.named_modules() if is_native_linear(m)}+ assert layers+ for fqn, layer in layers.items():+ impl = state[fqn + ".weight"].original_weight_tensor.tensor_impl+ assert layer.rot_group == 0 and layer.act_int8+ assert torch.equal(layer.weight_q, impl.int_data)+ assert torch.equal(layer.weight_scale.view(torch.float32), impl.scale.float().reshape(-1))+++@needs_decoder+def test_the_pickle_is_mapped_not_read_into_memory(hosted, monkeypatch):+ """Both experts read in full would sit in anonymous host memory (~29 GB) before offload pins its copies."""+ seen = []+ real = pq._load_prequant_checkpoint++ def spy(path, **kwargs):+ seen.append(bool(kwargs.get("mmap")))+ return real(path, **kwargs)++ monkeypatch.setattr(pq, "_load_prequant_checkpoint", spy)+ monkeypatch.setattr(sys, "platform", "linux")+ monkeypatch.delenv(pq._PREQUANT_MMAP_ENV, raising = False)+ model = _load(hosted, "transformer")+ assert model is not None and seen and seen[0] is True+ codes, _scale = hosted.written["transformer"]["blocks.0.attn1.to_q"]+ assert torch.equal(dict(model.named_buffers())["blocks.0.attn1.to_q.weight_q"], codes)+++@needs_decoder+def test_the_exports_fp32_modules_are_kept_and_other_fp32_is_refused(hosted):+ """The real hosted A14B pickle stores time_embedder, scale_shift_table and norm2 in fp32 (125 tensors)."""+ dense = hosted.dense["transformer"]+ path = Path(hosted.files["transformer"])+ for name, p in dense.named_parameters():+ if {"time_embedder", "scale_shift_table", "norm2"} & set(name.split(".")):+ p.data = p.data.float()+ write_hosted_v1(dense, path, component = "transformer")+ model = _load(hosted, "transformer")+ assert model is not None+ assert model.scale_shift_table.dtype == torch.float32+ assert model.condition_embedder.time_embedder.linear_1.weight.dtype == torch.float32+ assert model.patch_embedding.weight.dtype == torch.bfloat16+ dense.patch_embedding.weight.data = dense.patch_embedding.weight.data.float()+ write_hosted_v1(dense, path, component = "transformer")+ assert _load(hosted, "transformer") is None+++def test_an_unreadable_pickle_never_raises(tmp_path):+ bad = tmp_path / "x.pt"+ bad.write_bytes(b"not a checkpoint")+ assert (+ NH.load_native_from_hosted(+ str(bad),+ diffusers.WanTransformer3DModel,+ str(tmp_path),+ spec = _spec(),+ family = FAMILY,+ component = "transformer",+ base = BASE,+ )+ is None+ )+++def _int8_tensor(+ n = 8,+ k = 32,+ **extra,+):+ from core.inference.prequant_legacy_int8 import _int8_tensor_api, _to_int8_tensor++ t = _to_int8_tensor(+ torch.randint(-128, 128, (n, k), dtype = torch.int8),+ torch.rand(n).to(torch.bfloat16) + 0.01,+ torch.bfloat16,+ _int8_tensor_api(),+ )+ for key, value in extra.items():+ setattr(t, key, value)+ return t+++@needs_decoder+def test_weight_parts_read_int8tensor_and_refuse_what_they_cannot_describe():+ assert NH.int8_weight_parts("w", torch.ones(2, 2)) is None+ codes, scale = NH.int8_weight_parts("w", _int8_tensor())+ assert codes.dtype == torch.int8 and scale.shape == (8,)+ for extra in (+ {"act_quant_kwargs": None},+ {"act_pre_scale": torch.ones(32)},+ {"act_quant_scale": torch.ones(1)},+ {"zero_point": torch.ones(8, 1, dtype = torch.int8)},+ {"block_size": [1, 16]},+ ):+ with pytest.raises(ValueError):+ NH.int8_weight_parts("w", _int8_tensor(**extra))+++@needs_decoder+def test_the_loader_reads_both_experts_all_or_nothing(hosted, monkeypatch):+ from core.inference import video_denoiser_prequant as vdp++ sources = {c: _source(c) for c in COMPONENTS}+ monkeypatch.setattr(vdp, "denoiser_prequant_sources", lambda *a, **k: dict(sources))+ monkeypatch.setattr(+ pq, "_resolve_checkpoint_path", lambda src, *a, **k: hosted.files[src.filename]+ )+ out = V._read_native_quant_from_hosted(+ _family(), _spec(), base = BASE, source = hosted.cfg, config_kwargs = {}+ )+ assert set(out) == set(COMPONENTS)+ assert all(any(is_native_linear(m) for m in mod.modules()) for mod in out.values())+ Path(hosted.files["transformer_2"]).write_bytes(b"not a checkpoint")+ assert (+ V._read_native_quant_from_hosted(+ _family(), _spec(), base = BASE, source = hosted.cfg, config_kwargs = {}+ )+ == {}+ )+++def _source(component):+ return pq.PrequantSource(+ kind = "repo", location = "unsloth/Wan2.2-T2V-A14B-FP8", filename = component+ )+++def nvidia(monkeypatch):+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: False)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: False)+ monkeypatch.delenv(NH.NATIVE_HOSTED_ENV, raising = False)+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)+++CUDA_BF16 = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)+++def test_a14b_opts_in_as_a_pickle_family():+ fam = _family()+ assert NH.native_hosted_family(fam) and NH.native_hosted_pickle(fam)+ ltx = detect_video_family("Lightricks/LTX-2.5-Diffusers", override = "ltx-2.5")+ assert ltx is not None and NH.native_hosted_family(ltx) and not NH.native_hosted_pickle(ltx)+ ti2v = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers", override = "wan2.2-ti2v-5b")+ assert ti2v is not None and not NH.native_hosted_family(ti2v)+++ "scheme, target, env, act, expect",+ [+ ("int8", CUDA_BF16, None, None, "int8"),+ ("int8", CUDA_BF16, "0", None, None),+ ("int8", CUDA_BF16, None, "0", None),+ ("fp8", CUDA_BF16, None, None, None),+ ("nvfp4", CUDA_BF16, None, None, None),+ ("int8", types.SimpleNamespace(device = "cuda", dtype = torch.float16), None, None, None),+ ("int8", types.SimpleNamespace(device = "mps", dtype = torch.bfloat16), None, None, None),+ ],+)+def test_the_hosted_scheme_gate(nvidia, monkeypatch, scheme, target, env, act, expect):+ if env is not None:+ monkeypatch.setenv(NH.NATIVE_HOSTED_ENV, env)+ if act is not None:+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", act)+ assert V._video_native_hosted_scheme(_family(), scheme, target) == expect+++def test_convrot_off_keeps_the_unrotated_pickle_read_but_not_ltx(nvidia, monkeypatch):+ from core.inference.diffusion_native_quant import NATIVE_INT8_ROT_ENV++ monkeypatch.setenv(NATIVE_INT8_ROT_ENV, "0")+ assert V._video_native_hosted_scheme(_family(), "int8", CUDA_BF16) == "int8"+ ltx = detect_video_family("Lightricks/LTX-2.5-Diffusers", override = "ltx-2.5")+ assert V._video_native_hosted_scheme(ltx, "int8", CUDA_BF16) is None+ monkeypatch.delenv(NATIVE_INT8_ROT_ENV)+ assert V._video_native_hosted_scheme(ltx, "int8", CUDA_BF16) == "int8"++[email protected]("rocm, stubbed", [(True, False), (False, True)])[email protected]("act", [None, "1"])+def test_rocm_and_stubbed_torchao_do_not_read_the_pickle(monkeypatch, rocm, stubbed, act):+ # native_quant_host accepts these hosts (UNSLOTH_NATIVE_INT8_ACT=1 makes them W8A8); the pickle needs torchao.+ monkeypatch.setattr(tq, "torch_is_rocm", lambda: rocm)+ monkeypatch.setattr(tq, "is_stubbed", lambda name: stubbed)+ monkeypatch.delenv(NH.NATIVE_HOSTED_ENV, raising = False)+ if act is None:+ monkeypatch.delenv("UNSLOTH_NATIVE_INT8_ACT", raising = False)+ else:+ monkeypatch.setenv("UNSLOTH_NATIVE_INT8_ACT", act)+ assert tq.native_quant_host(CUDA_BF16)+ assert V._video_native_hosted_scheme(_family(), "int8", CUDA_BF16) is None+++ "offloads, expect, hosted", [(True, "declined", ["int8"]), (False, "int8", [])]+)+def test_planner_stages_the_hosted_pair_under_offload(+ nvidia, monkeypatch, offloads, expect, hosted+):+ target = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16, ordinal = 0)+ monkeypatch.setattr(V, "resolve_diffusion_device_target", lambda **k: target)+ monkeypatch.setattr(V, "_video_auto_denoiser_scheme", lambda fam, **k: "int8")+ monkeypatch.setattr(V, "_auto_seed_yields_to_resident_bf16", lambda *a: False)+ monkeypatch.setattr(V, "_video_seed_stays_resident", lambda fam, **k: not offloads)+ backend = V.VideoBackend.__new__(V.VideoBackend)+ backend._state = None+ out: list = []+ got = backend._video_planned_auto_denoiser_scheme(+ _family(),+ base = BASE,+ kind = "pipeline",+ transformer_quant = "int8",+ speed_mode = None,+ native_hosted = out,+ )+ assert got == (V.DENOISER_SEED_DECLINED if expect == "declined" else expect)+ assert out == hosted