Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio LTX-2.5: read the hosted int8 checkpoint as native Linears when the DiT is offloaded (#16886)
Daniel Han250895c051cd
diff --git a/studio/backend/core/inference/diffusion_native_hosted.py b/studio/backend/core/inference/diffusion_native_hosted.pynew file mode 100644index 00000000000..e519c5a4589--- /dev/null+++ b/studio/backend/core/inference/diffusion_native_hosted.py@@ -0,0 +1,243 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An offloaded int8 denoiser read from the hosted ConvRot int8 checkpoint as native Linears.++Offload hooks cannot move torchao weights, so an offloaded load used to fetch the dense bf16 shards and quantise them+at load (LTX-2.5: 36 GB staged, ~100 s of build, then a 23 GB cache write). The hosted file already holds per-row int8+codes and float32 scales of the same ConvRot-rotated weights, which is exactly what ``NativeWeightOnlyLinear`` stores:+this reads them into native Linears with the same layer set, rotation and activation path as the runtime build."""++from __future__ import annotations++import json+import os+from typing import Any, Optional++NATIVE_HOSTED_ENV = "UNSLOTH_VIDEO_NATIVE_HOSTED"+_QDATA = "._weight_qdata"+_SCALE = "._weight_scale"+_ZERO_POINT = "._weight_zero_point"+# Int8Tensor payloads that would change the activation math (static / pre-scaled activations): refused.+_ACT_PAYLOADS = (+ "._weight_act_quant_scale",+ "._weight_act_quant_zero_point",+ "._weight_act_pre_scale",+)+++def native_hosted_enabled() -> bool:+ raw = (os.environ.get(NATIVE_HOSTED_ENV) or "").strip().lower()+ return raw not in ("0", "off", "false", "no")+++def native_hosted_family(fam: Any) -> bool:+ """Whether ``fam`` opted in (its hosted int8 file is ConvRot safetensors) and the switch is on."""+ return bool(getattr(fam, "native_offload_from_hosted_int8", False)) and native_hosted_enabled()+++def _log(logger: Any, level: str, msg: str) -> None:+ if logger is not None:+ getattr(logger, level)("video.native_hosted: %s", msg)+++def hosted_metadata(path: str) -> Optional[dict]:+ """The checkpoint's ``unsloth_metadata``, or None. Header only."""+ try:+ from safetensors import safe_open+ with safe_open(str(path), framework = "pt") as handle:+ raw = (handle.metadata() or {}).get("unsloth_metadata")+ return json.loads(raw) if raw else None+ except Exception: # noqa: BLE001 - unreadable: not usable+ return None+++def metadata_refusal(+ meta: Any, spec: dict, *, family: Optional[str], component: str+) -> Optional[str]:+ """Why the hosted file is not the native build ``spec`` describes, or None."""+ from .diffusion_convrot import CONVROT_KIND, ROTATION_FQNS_KEY, ROTATION_GROUP_KEY, ROTATION_KEY++ if not isinstance(meta, dict):+ return "no unsloth metadata"+ if spec.get("scheme") != "int8" or not spec.get("act_int8"):+ return f"the native build is {spec.get('scheme')} (act_int8={spec.get('act_int8')}), not W8A8 int8"+ if meta.get("scheme") != "int8" or meta.get("quant_backend") != "torchao":+ return f"checkpoint scheme {meta.get('scheme')!r} / backend {meta.get('quant_backend')!r}"+ if family is not None and meta.get("family") != family:+ return f"checkpoint family {meta.get('family')!r}, not {family!r}"+ if (meta.get("component") or "transformer") != component:+ return f"checkpoint component {meta.get('component')!r}, not {component!r}"+ if str(meta.get("torch_dtype") or "bfloat16") != "bfloat16":+ return f"checkpoint dtype {meta.get('torch_dtype')!r}"+ if meta.get(ROTATION_KEY) != CONVROT_KIND:+ return f"activation rotation {meta.get(ROTATION_KEY)!r}"+ if int(meta.get(ROTATION_GROUP_KEY) or 0) != int(spec.get("rot_group") or 0):+ return f"rotation group {meta.get(ROTATION_GROUP_KEY)!r}, the native build uses {spec.get('rot_group')!r}"+ if not isinstance(meta.get(ROTATION_FQNS_KEY), (list, tuple)):+ return "no rotation fqns"+ if int(meta.get("min_features") or -1) != int(spec.get("min_features") or -2):+ return f"min_features {meta.get('min_features')!r}"+ excludes = meta.get("exclude_name_tokens")+ if not isinstance(excludes, (list, tuple)):+ return "no exclude tokens"+ if sorted(set(excludes) | {"lora_"}) != list(spec.get("exclude_name_tokens") or ()):+ return "its excluded layers differ from the native build's"+ return None+++def load_native_from_hosted(+ path: str,+ transformer_cls: Any,+ config_source: str,+ *,+ spec: dict,+ family: Optional[str],+ component: str = "transformer",+ config_kwargs: Optional[dict] = None,+ logger: Any = None,+) -> Any:+ """The native ``component`` built from hosted ``path``, or None (the caller falls back to the dense build). Never+ raises. Refuses unless the file holds exactly the layers, rotation and activation path the runtime build makes."""+ try:+ import time++ import torch++ t0 = time.perf_counter()+ meta = hosted_metadata(path)+ problem = metadata_refusal(meta, spec, family = family, component = component)+ if problem:+ _log(logger, "info", f"hosted int8 checkpoint not used ({problem})")+ return None+ from accelerate import init_empty_weights+ from safetensors.torch import load_file++ from .diffusion_convrot import ROTATION_FQNS_KEY+ from .diffusion_native_quant import native_linear_class+ from .diffusion_prequant import HostedInt8Parts, _verify_packed_fingerprint+ from .diffusion_transformer_quant import make_filter_fn++ config = transformer_cls.load_config(+ config_source, subfolder = component, **(config_kwargs or {})+ )+ with init_empty_weights(include_buffers = False):+ model = transformer_cls.from_config(config)+ group = int(spec["rot_group"])+ # The runtime build's layer set (its require_bf16 holds: the whole load is bf16, checked above).+ filter_fn = make_filter_fn(int(spec["min_features"]), tuple(spec["exclude_name_tokens"]))+ wanted = {name for name, mod in model.named_modules() if name and filter_fn(mod, name)}+ from .prequant_safetensors import UNSLOTH_ROOT_PREFIX++ state = load_file(str(path))+ # Root-level tensors are stored prefixed (the torchao unflatten splits keys on dots).+ for key in [k for k in state if k.startswith(UNSLOTH_ROOT_PREFIX)]:+ state[key[len(UNSLOTH_ROOT_PREFIX) :]] = state.pop(key)+ quantised = {key[: -len(_QDATA)] for key in state if key.endswith(_QDATA)}+ if quantised != wanted:+ _log(+ logger,+ "info",+ f"hosted int8 checkpoint not used ({len(quantised)} quantised layers, the native build makes "+ f"{len(wanted)}; e.g. {sorted(quantised ^ wanted)[:2]})",+ )+ return None+ if any(key.endswith(_ACT_PAYLOADS) for key in state):+ _log(+ logger,+ "info",+ "hosted int8 checkpoint not used (it carries static activation scales)",+ )+ return None+ modules = dict(model.named_modules())+ rotated = set(meta[ROTATION_FQNS_KEY])+ expect_rotated = {fqn for fqn in wanted if int(modules[fqn].in_features) % group == 0}+ if rotated != expect_rotated:+ _log(+ logger,+ "info",+ "hosted int8 checkpoint not used (its rotated layers differ from the native build's)",+ )+ return None+ parts: dict = {}+ for fqn in quantised:+ codes = state[fqn + _QDATA]+ scale = state[fqn + _SCALE]+ zero = state.get(fqn + _ZERO_POINT)+ lin = modules[fqn]+ shape = (int(lin.out_features), int(lin.in_features))+ if (+ codes.dtype != torch.int8+ or tuple(codes.shape) != shape+ or scale.numel() != shape[0]+ or not scale.is_floating_point()+ or (zero is not None and bool(torch.any(zero != 0)))+ ):+ _log(+ logger,+ "info",+ f"hosted int8 checkpoint not used ({fqn}: unexpected payload layout)",+ )+ return None+ parts[fqn + ".weight"] = HostedInt8Parts(codes, scale, zero)+ # The torchao load checks the same recorded digests: corruption after build must not render.+ if not _verify_packed_fingerprint(parts, meta, logger = logger, path = path):+ return None+ del parts+ cls = native_linear_class()+ for fqn in sorted(quantised):+ lin = modules[fqn]+ parent_name, _, leaf = fqn.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ stand_in = torch.nn.Linear(+ int(lin.in_features),+ int(lin.out_features),+ bias = lin.bias is not None,+ device = "meta",+ dtype = torch.bfloat16,+ )+ codes = state.pop(fqn + _QDATA)+ scale = state.pop(fqn + _SCALE).to(torch.float32).reshape(-1).contiguous()+ state.pop(fqn + _ZERO_POINT, None)+ setattr(+ parent,+ leaf,+ cls(+ stand_in,+ "int8",+ act_int8 = True,+ rot_group = group if fqn in rotated else 0,+ codes = codes,+ scale = scale,+ ),+ )+ state[fqn + ".weight_q"] = codes+ state[fqn + ".weight_scale"] = scale.view(torch.int32)+ model.load_state_dict(state, strict = True, assign = True)+ del state+ if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):+ _log(logger, "warning", "hosted int8 checkpoint left meta tensors; not used")+ return None+ if any(p.dtype != torch.bfloat16 for p in model.parameters() if p.is_floating_point()):+ _log(logger, "info", "hosted int8 checkpoint not used (a non-bf16 dense tensor)")+ return None+ model.eval()+ try:+ model._unsloth_runtime_quant = "int8"+ model._unsloth_runtime_quant_info = {+ "scheme": "int8",+ "native": True,+ "hosted_path": str(path),+ }+ except Exception: # noqa: BLE001+ pass+ _log(+ logger,+ "info",+ f"read {len(quantised)} int8 ConvRot layers of {component} from the hosted checkpoint as native "+ f"Linears in {time.perf_counter() - t0:.1f} s (no dense shards, no runtime quantise)",+ )+ return model+ except Exception as exc: # noqa: BLE001 - an unusable file falls back to the dense build+ _log(logger, "warning", f"hosted int8 read failed: {type(exc).__name__}: {str(exc)[:200]}")+ return Nonediff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.pyindex fddd6aaf859..f8bafc0afae 100644--- a/studio/backend/core/inference/diffusion_prequant.py+++ b/studio/backend/core/inference/diffusion_prequant.py@@ -1365,7 +1365,26 @@ _CANONICAL_PARTS: dict = {"Float8Tensor": (("q", "qdata"), ("scale", "scale")),"NVFP4Tensor": (("q", "qdata"), ("scale", "scale"), ("per_tensor_scale", "per_tensor_scale")),"MXTensor": (("q", "qdata"), ("scale", "scale")),+ "HostedInt8Parts": (("q", "qdata"), ("scale", "scale"), ("zero_point", "zero_point")),}+++class HostedInt8Parts:+ """An Int8Tensor's stored payload read as plain tensors (no torchao), fingerprinted like the Int8Tensor."""++ __slots__ = ("qdata", "scale", "zero_point")++ def __init__(+ self,+ qdata: Any,+ scale: Any,+ zero_point: Any = None,+ ):+ self.qdata = qdata+ self.scale = scale+ self.zero_point = zero_point++_CANONICAL_ROLE_ORDER = ("q","scale",diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 2e78f733486..887ab80d224 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -3781,6 +3781,68 @@ def _read_native_quant_from_cache(return {}+def _read_native_quant_from_hosted(+ fam: Any,+ spec: Optional[dict],+ *,+ base: Optional[str],+ source: Optional[str],+ config_kwargs: dict,+ hf_token: Optional[str] = None,+ local_files_only: bool = False,+) -> dict[str, Any]:+ """``{component: module}`` for EVERY denoiser read from the hosted int8 file as native Linears, or ``{}`` (all or+ nothing; the dense shards then load as before). Never raises."""+ if spec is None:+ return {}+ try:+ from utils.torch_warmup import close_dynamo_import_window+ close_dynamo_import_window(logger) # a no-op once the load closed it above+ except Exception as exc: # noqa: BLE001 - optimisation only+ logger.debug("dynamo pre-import skipped: %r", exc)+ try:+ import diffusers++ from .diffusion import hub_cache_dir+ from .diffusion_native_hosted import load_native_from_hosted+ from .diffusion_prequant import _resolve_checkpoint_path+ from .video_denoiser_prequant import denoiser_prequant_sources++ transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)+ sources = denoiser_prequant_sources(fam, spec.get("scheme"), base, hf_token = hf_token)+ if transformer_cls is None or not sources:+ return {}+ out: dict[str, Any] = {}+ for component, src in sources.items():+ path = _resolve_checkpoint_path(+ src,+ hf_token,+ hub_cache_dir(),+ local_files_only = local_files_only,+ scheme = spec.get("scheme"),+ logger = logger,+ )+ if path is None:+ return {}+ module = load_native_from_hosted(+ path,+ transformer_cls,+ str(source or base),+ spec = spec,+ family = getattr(fam, "name", None),+ component = component,+ config_kwargs = config_kwargs,+ logger = logger,+ )+ if module is None:+ return {}+ out[component] = module+ return out+ except Exception as exc: # noqa: BLE001 - a hosted problem keeps the dense build+ logger.warning("video.native_hosted: hosted read skipped (%s)", exc)+ return {}++def _video_native_stream_build(fam: Any,spec: Optional[dict],@@ -4163,6 +4225,26 @@ def _auto_offload_scheme(requested: Optional[str]) -> Optional[str]:return TQ_INT8 if scheme == TQ_AUTO else scheme+def _video_native_hosted_scheme(fam: Any, scheme: Optional[str], target: Any) -> Optional[str]:+ """``int8`` when an offloaded load of ``scheme`` builds the W8A8 ConvRot native DiT that the family's hosted int8+ checkpoint already holds (``diffusion_native_hosted``), else None. The load re-checks the file; never raises."""+ try:+ from .diffusion_native_hosted import native_hosted_family+ from .diffusion_native_quant import int8_rotation_group++ if scheme != TQ_INT8 or not native_hosted_family(fam):+ return None+ if getattr(target, "device", None) != "cuda":+ return None+ if native_quant_scheme(target, TQ_INT8, family = fam.name, offload = True) != TQ_INT8:+ return None+ if not native_int8_act(target) or int(int8_rotation_group() or 0) <= 0:+ return None+ return TQ_INT8+ except Exception: # noqa: BLE001 -- unanswerable: the dense shards and runtime build, as before+ return None++# Denoise-side base of estimate_video_runtime_mib; the rest scales with the decoded clip, held only by the VAE._VIDEO_DENOISE_ACTIVATION_MIB = 4096@@ -5603,6 +5685,7 @@ class VideoBackend:if kind != "single_file"else None)+ native_hosted_out: list = []video_auto_denoiser = self._video_planned_auto_denoiser_scheme(fam,base = base,@@ -5614,6 +5697,7 @@ class VideoBackend:gpu_ordinal = kwargs.get("gpu_ordinal"),hf_token = kwargs.get("hf_token"),ltx25_dev = ltx25_dev,+ native_hosted = native_hosted_out,)video_seed_declined = video_auto_denoiser == DENOISER_SEED_DECLINEDif video_seed_declined:@@ -5631,6 +5715,32 @@ class VideoBackend:local_files_only = local_files_only,kind = kind,)+ # Offloaded int8 read from the hosted file as native Linears: its dense shards are not staged either.+ native_hosted_planned = None+ if (+ video_seed_declined+ and native_hosted_out+ and not skip_transformer_weights+ and self._denoiser_prequant_verified(+ fam,+ native_hosted_out[0],+ base,+ kwargs.get("h3_task"),+ kwargs.get("hf_token"),+ local_files_only = local_files_only,+ kind = kind,+ )+ ):+ skip_transformer_weights = True+ native_hosted_planned = native_hosted_out[0]+ kwargs["_video_native_hosted_planned"] = native_hosted_planned+ # The scheme whose hosted artifact replaces the dense shards (claimed, then fetched under the cancel event).+ planned_denoiser = (+ h3_auto_denoiser+ or video_auto_denoiser+ or requested_denoiser+ or native_hosted_planned+ )skip_transformer_components = (_planned_denoiser_components(fam, kind) if skip_transformer_weights else ())@@ -5694,7 +5804,7 @@ class VideoBackend:if skip_transformer_weights:claimed = self._denoiser_prequant_repo_ids(fam,- h3_auto_denoiser or video_auto_denoiser or requested_denoiser,+ planned_denoiser,base,kwargs.get("h3_task"),)@@ -5854,7 +5964,7 @@ class VideoBackend:self._fetch_denoiser_prequant(self._denoiser_prequant_source_list(fam,- h3_auto_denoiser or video_auto_denoiser or requested_denoiser,+ planned_denoiser,base,kwargs.get("h3_task"),hf_token = kwargs.get("hf_token"),@@ -5862,7 +5972,7 @@ class VideoBackend:kwargs.get("hf_token"),cancel_event = cancel_event,local_files_only = local_files_only,- scheme = h3_auto_denoiser or video_auto_denoiser or requested_denoiser,+ scheme = planned_denoiser,)base_plan = planned_files.get(_FETCH_BASE)actual_te_skip = tuple(te_skipped) + tuple(h3_te_skipped)@@ -7035,8 +7145,12 @@ class VideoBackend:gpu_ordinal: Optional[int] = None,hf_token: Optional[str] = None,ltx25_dev: bool = False,+ native_hosted: Optional[list] = None,) -> Optional[str]:- """The pre-download auto denoiser scheme, None, or ``DENOISER_SEED_DECLINED`` if the plan would offload."""+ """The pre-download auto denoiser scheme, None, or ``DENOISER_SEED_DECLINED`` if the plan would offload.++ On a decline, ``native_hosted`` (a list) receives the scheme when the offloaded native build can be read from+ the hosted checkpoint instead (``_video_native_hosted_scheme``)."""try:if kind != "pipeline" or getattr(fam, "modular_workflow", None):return None@@ -7103,6 +7217,17 @@ class VideoBackend:ltx25_dev = ltx25_dev,device_memory_reader = _settled_once,):+ hosted = _video_native_hosted_scheme(fam, scheme, target)+ if native_hosted is not None and hosted is not None:+ native_hosted.append(hosted)+ logger.info(+ "video.denoiser_prequant: an artifact-sized plan for %s still offloads on "+ "this card, so the hosted %s checkpoint is read as offload-safe native "+ "Linears instead of quantising the dense denoiser shards",+ scheme,+ hosted,+ )+ return DENOISER_SEED_DECLINEDlogger.info("video.denoiser_prequant: an artifact-sized plan for %s still offloads on ""this card, and offload moves the DiT, so the dense denoiser shards are "@@ -7780,6 +7905,7 @@ class VideoBackend:if kind != "single_file"else None) or transformer_quant+ native_hosted_out: list = []video_planned = self._video_planned_auto_denoiser_scheme(fam,base = base,@@ -7791,9 +7917,11 @@ class VideoBackend:gpu_ordinal = load_kwargs.get("gpu_ordinal"),hf_token = hf_token,ltx25_dev = ltx25_dev,+ native_hosted = native_hosted_out,)if video_planned == DENOISER_SEED_DECLINED:- video_planned = None+ # An offloaded int8 load reads the hosted file as native Linears: plan that file, not the dense shards.+ video_planned = native_hosted_out[0] if native_hosted_out else Noneif kind == "pipeline" and not getattr(fam, "modular_workflow", None):transformer_quant = video_plannedelse:@@ -8630,6 +8758,7 @@ class VideoBackend:_te_prequant_failed: Optional[dict] = None,_h3_auto_denoiser_planned: Optional[str] = None,_video_auto_denoiser_planned: Optional[str] = None,+ _video_native_hosted_planned: Optional[str] = None,_denoiser_prequant_skipped: tuple[str, ...] = (),) -> dict[str, Any]:fam = self.validate_load_request(@@ -9415,6 +9544,25 @@ class VideoBackend:cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),)self._raise_if_video_load_superseded(_load_token)+ native_from_hosted = False+ if (+ not native_cache_hits+ and native_cache_spec is not None+ and _video_native_hosted_planned is not None+ and _video_native_hosted_planned == native_cache_spec.get("scheme")+ ):+ # The planner staged the hosted int8 file instead of the dense shards: read it as the native build.+ native_cache_hits = _read_native_quant_from_hosted(+ fam,+ native_cache_spec,+ base = fetch_base,+ source = _base_local_dir or fetch_base,+ config_kwargs = native_config_kwargs,+ hf_token = hf_token,+ local_files_only = local_files_only,+ )+ native_from_hosted = bool(native_cache_hits)+ self._raise_if_video_load_superseded(_load_token)if not native_cache_hits and native_cache_spec is not None:# A miss whose dense load would not fit host RAM: build the same native experts per shard into the cache.native_cache_hits = _video_native_stream_build(@@ -9945,6 +10093,18 @@ class VideoBackend:denoiser_seed_scheme,fam.name,)+ elif native_cached and native_from_hosted:+ transformer_quant_engaged = native_scheme+ transformer_quant_source = (+ native_quant_reason(getattr(pipe, "transformer", None), native_scheme)+ + "; codes and scales read from the hosted pre-quantized checkpoint"+ )+ logger.info(+ "video.transformer_quant: %s native denoiser(s) read from the hosted checkpoint for "+ "%s; skipping the dense shards and the runtime quantise",+ native_scheme,+ fam.name,+ )elif native_cached:transformer_quant_engaged = native_schemetransformer_quant_source = (diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 38992b969e6..58bbf3bec89 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -304,6 +304,9 @@ class VideoFamily:prequant_filenames: tuple[tuple[str, ...], ...] = field(default_factory = tuple)# Separate partition, served ONLY by its own (scheme, task, filename) row or it loads the wrong one.prequant_partition_tasks: tuple[str, ...] = field(default_factory = tuple)+ # An offloaded int8 load reads the hosted int8 file's ConvRot codes as native Linears (diffusion_native_hosted)+ # instead of staging and quantising the dense shards. Only for a family whose hosted int8 file is ConvRot safetensors.+ native_offload_from_hosted_int8: bool = False# Modular workflow loaded instead of a DiffusionPipeline; its routing blocks are not pruned.modular_workflow: Optional[str] = None# Released video and audio sigma shifts, when configurable.@@ -672,6 +675,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (("lightricks/ltx-2.5-diffusers", "int8", "unsloth/LTX-2.5-FP8"),("lightricks/ltx-2.5-diffusers", "fp8", "unsloth/LTX-2.5-FP8"),),+ native_offload_from_hosted_int8 = True,# File sizes (decimal GB): int8 keeps the audio stream, AV cross-attention and adaLN in bf16; fp8 quantizes them.prequant_resident_gb_by_scheme = (("int8", 25.11), ("fp8", 19.06)),# Dev DiT at the root, distilled under distilled/; the pack above supplies VAEs and connectors.diff --git a/studio/backend/tests/test_video_native_hosted.py b/studio/backend/tests/test_video_native_hosted.pynew file mode 100644index 00000000000..5732ef9cdce--- /dev/null+++ b/studio/backend/tests/test_video_native_hosted.py@@ -0,0 +1,290 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An offloaded LTX-2.5 int8 load reads the hosted ConvRot int8 file as native Linears instead of quantising the+dense shards: same layer set, rotation and activation path as the runtime native build, codes and scales from the+file, verified against its recorded fingerprint, and refused (dense build) whenever the file is not that build."""++from __future__ import annotations++import json+import sys+import types+from pathlib import Path++import pytest++torch = pytest.importorskip("torch")+diffusers = pytest.importorskip("diffusers")+pytest.importorskip("safetensors")+pytest.importorskip("accelerate")+if not hasattr(diffusers, "LTX2VideoTransformer3DModel"):+ pytest.skip("diffusers without LTX-2", allow_module_level = True)++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import diffusion_native_hosted as NH # noqa: E402+from core.inference import video as V # noqa: E402+from core.inference.diffusion_convrot import ( # noqa: E402+ CONVROT_KIND,+ ROTATION_FQNS_KEY,+ ROTATION_GROUP_KEY,+ ROTATION_KEY,+ quantize_convrot_int8_weight,+)+from core.inference.diffusion_native_cache import native_spec # noqa: E402+from core.inference.diffusion_native_quant import ( # noqa: E402+ apply_native_weight_quant,+ is_native_linear,+)+from core.inference.diffusion_prequant import ( # noqa: E402+ CANONICAL_FINGERPRINT_ALGO,+ HostedInt8Parts,+ canonical_weight_digest,+ canonical_weight_fingerprint,+)+from core.inference.diffusion_transformer_quant import make_filter_fn # noqa: E402+from core.inference.prequant_safetensors import UNSLOTH_ROOT_KEYS_KEY, UNSLOTH_ROOT_PREFIX # noqa: E402+from core.inference.video_families import detect_video_family # noqa: E402++FAMILY = "ltx-2.5"+CLS = diffusers.LTX2VideoTransformer3DModel+++def _config():+ # 2 heads x 256 = 512 wide: the video Linears pass the 512 filter and g256 ConvRot divides them.+ return dict(+ in_channels = 32,+ out_channels = 32,+ num_attention_heads = 2,+ attention_head_dim = 256,+ cross_attention_dim = 512,+ caption_channels = 512,+ audio_in_channels = 16,+ audio_out_channels = 16,+ audio_num_attention_heads = 2,+ audio_attention_head_dim = 32,+ audio_cross_attention_dim = 64,+ num_layers = 1,+ )+++def _spec():+ return native_spec("int8", FAMILY, act_int8 = True)+++def _dense():+ torch.manual_seed(0)+ return CLS(**_config()).to(torch.bfloat16).eval()+++def _filter(spec, require_bf16 = False):+ return make_filter_fn(+ int(spec["min_features"]), tuple(spec["exclude_name_tokens"]), require_bf16 = require_bf16+ )+++def _write_hosted(+ root: Path,+ *,+ mutate = None,+ meta_patch = None,+) -> tuple[Path, Path, dict]:+ """A tiny hosted-format int8 ConvRot file (torchao Int8Tensor layout) plus the dense config dir."""+ from safetensors.torch import save_file++ spec = _spec()+ model = _dense()+ cfg_dir = root / "base"+ model.save_config(str(cfg_dir / "transformer"))+ group = int(spec["rot_group"])+ keep = _filter(spec, require_bf16 = True)+ quantised = sorted(n for n, m in model.named_modules() if n and keep(m, n))+ state = {k: v.contiguous() for k, v in model.state_dict().items()}+ rotated = []+ parts = {}+ for fqn in quantised:+ weight = state.pop(fqn + ".weight")+ if weight.shape[1] % group == 0:+ codes, scale = quantize_convrot_int8_weight(weight, group)+ rotated.append(fqn)+ else:+ w = weight.float()+ scale = w.abs().amax(dim = 1, keepdim = True) / 127.0+ codes = (w / scale).round().clamp(-128, 127).to(torch.int8)+ zero = torch.zeros(codes.shape[0], 1, dtype = torch.int8)+ state[fqn + "._weight_qdata"] = codes.contiguous()+ state[fqn + "._weight_scale"] = scale.float().contiguous()+ state[fqn + "._weight_zero_point"] = zero+ parts[fqn + ".weight"] = HostedInt8Parts(codes, scale.float(), zero)+ fingerprint = canonical_weight_fingerprint(parts)+ meta = {+ "base_model_id": "Lightricks/LTX-2.5-Diffusers",+ "family": FAMILY,+ "scheme": "int8",+ "min_features": spec["min_features"],+ "exclude_name_tokens": [t for t in spec["exclude_name_tokens"] if t != "lora_"],+ "torch_dtype": "bfloat16",+ "quant_backend": "torchao",+ "transformer_class": "LTX2VideoTransformer3DModel",+ "component": "transformer",+ "fingerprint": {+ "algo": CANONICAL_FINGERPRINT_ALGO,+ "count": fingerprint["count"],+ "canonical_modules": fingerprint["canonical_modules"],+ },+ ROTATION_KEY: CONVROT_KIND,+ ROTATION_GROUP_KEY: group,+ ROTATION_FQNS_KEY: rotated,+ }+ if meta_patch:+ meta.update(meta_patch)+ if mutate:+ mutate(state)+ # The hosted layout: root-level tensors under a prefix (unflatten splits keys on dots).+ roots = [k for k in state if "." not in k]+ assert roots, "the tiny model should have a root-level tensor (scale_shift_table)"+ stored = {(f"{UNSLOTH_ROOT_PREFIX}{k}" if k in roots else k): v for k, v in state.items()}+ path = root / "LTX-2.5-INT8-ConvRot.safetensors"+ save_file(+ stored,+ str(path),+ metadata = {"unsloth_metadata": json.dumps(meta), UNSLOTH_ROOT_KEYS_KEY: json.dumps(roots)},+ )+ return path, cfg_dir, {"quantised": quantised, "rotated": rotated, "state": state}+++def _load(+ path,+ cfg_dir,+ spec = None,+ family = FAMILY,+):+ return NH.load_native_from_hosted(+ str(path), CLS, str(cfg_dir), spec = spec or _spec(), family = family, logger = None+ )+++def test_reads_the_hosted_codes_into_the_runtime_native_layout(tmp_path):+ path, cfg_dir, built = _write_hosted(tmp_path)+ model = _load(path, cfg_dir)+ assert model is not None+ native = {n: m for n, m in model.named_modules() if is_native_linear(m)}+ assert sorted(native) == built["quantised"]+ for fqn, layer in native.items():+ assert layer.act_int8 and layer.scheme == "int8"+ assert layer.rot_group == (256 if fqn in built["rotated"] else 0)+ assert torch.equal(layer.weight_q, built["state"][fqn + "._weight_qdata"])+ assert torch.equal(+ layer.weight_scale.view(torch.float32),+ built["state"][fqn + "._weight_scale"].reshape(-1),+ )+ # The runtime native build of the dense model swaps exactly these layers, the same way.+ runtime = _dense()+ apply_native_weight_quant(+ runtime, "int8", filter_fn = _filter(_spec(), require_bf16 = True), act_int8 = True+ )+ expect = {n: m for n, m in runtime.named_modules() if is_native_linear(m)}+ assert sorted(expect) == sorted(native)+ for fqn, layer in expect.items():+ assert (layer.rot_group, layer.act_int8, layer.in_features, layer.out_features) == (+ native[fqn].rot_group,+ native[fqn].act_int8,+ native[fqn].in_features,+ native[fqn].out_features,+ )+ # Every dense (non-quantised) tensor is the dense model's, bit for bit, in bf16.+ dense = runtime.state_dict()+ for key, value in model.state_dict().items():+ if key.endswith((".weight_q", ".weight_scale")):+ continue+ assert value.dtype == dense[key].dtype and torch.equal(value, dense[key]), key+ assert model._unsloth_runtime_quant == "int8"+++ "case",+ [+ "rotation_group",+ "family",+ "unrotated_kind",+ "excludes",+ "missing_layer",+ "static_act",+ "corrupt_codes",+ ],+)+def test_refuses_a_file_that_is_not_the_native_build(tmp_path, case):+ first = None++ def drop_layer(state):+ nonlocal first+ first = sorted(k for k in state if k.endswith("._weight_qdata"))[0][+ : -len("._weight_qdata")+ ]+ for suffix in ("._weight_qdata", "._weight_scale", "._weight_zero_point"):+ state.pop(first + suffix)++ def static_act(state):+ key = sorted(k for k in state if k.endswith("._weight_qdata"))[0]+ state[key[: -len("._weight_qdata")] + "._weight_act_quant_scale"] = torch.ones(1)++ def corrupt(state):+ key = sorted(k for k in state if k.endswith("._weight_qdata"))[0]+ state[key] = state[key].clone()+ state[key][0, 0] = -state[key][0, 0] if state[key][0, 0] != 0 else 1++ kwargs = {+ "rotation_group": {"meta_patch": {ROTATION_GROUP_KEY: 64}},+ "family": {"meta_patch": {"family": "ltx-2"}},+ "unrotated_kind": {"meta_patch": {ROTATION_KEY: None}},+ "excludes": {"meta_patch": {"exclude_name_tokens": ["norm"]}},+ "missing_layer": {"mutate": drop_layer},+ "static_act": {"mutate": static_act},+ "corrupt_codes": {"mutate": corrupt},+ }[case]+ path, cfg_dir, _ = _write_hosted(tmp_path, **kwargs)+ assert _load(path, cfg_dir) is None+++def test_refuses_a_weight_only_or_unrotated_native_build(tmp_path):+ path, cfg_dir, _ = _write_hosted(tmp_path)+ assert _load(path, cfg_dir, spec = native_spec("int8", FAMILY, act_int8 = False)) is None+ assert _load(path, cfg_dir, spec = native_spec("fp8", FAMILY, act_int8 = False)) is None+++def test_hosted_parts_fingerprint_like_the_torchao_int8_tensor():+ torchao_q = pytest.importorskip("torchao.quantization")+ if not hasattr(torchao_q, "Int8Tensor"):+ pytest.skip("torchao without Int8Tensor")+ from torchao.quantization.granularity import PerRow++ torch.manual_seed(0)+ weight = torch.randn(64, 256, dtype = torch.bfloat16)+ tensor = torchao_q.Int8Tensor.from_hp(weight, PerRow())+ parts = HostedInt8Parts(tensor.qdata, tensor.scale, getattr(tensor, "zero_point", None))+ assert canonical_weight_digest(tensor) == canonical_weight_digest(parts) is not None+++def _target():+ return types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16, ordinal = 0)+++def test_only_an_opted_in_family_offload_int8_takes_the_hosted_file(monkeypatch):+ monkeypatch.setattr(V, "native_quant_scheme", lambda *a, **k: "int8")+ monkeypatch.setattr(V, "native_int8_act", lambda target: True)+ ltx25 = detect_video_family("Lightricks/LTX-2.5-Diffusers")+ assert ltx25 is not None and ltx25.name == FAMILY+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) == "int8"+ assert V._video_native_hosted_scheme(ltx25, "fp8", _target()) is None+ cpu = types.SimpleNamespace(device = "cpu", dtype = torch.bfloat16, ordinal = None)+ assert V._video_native_hosted_scheme(ltx25, "int8", cpu) is None+ wan = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers")+ assert wan is not None and V._video_native_hosted_scheme(wan, "int8", _target()) is None+ monkeypatch.setattr(V, "native_int8_act", lambda target: False)+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) is None+ monkeypatch.setattr(V, "native_int8_act", lambda target: True)+ monkeypatch.setenv(NH.NATIVE_HOSTED_ENV, "0")+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) is None