Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio LTX-2.5: read the hosted int8 checkpoint as native Linears when the DiT is offloaded (#16886)

Daniel Han250895c051cd
diff --git a/studio/backend/core/inference/diffusion_native_hosted.py b/studio/backend/core/inference/diffusion_native_hosted.py
new file mode 100644
index 00000000000..e519c5a4589
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_native_hosted.py
@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An offloaded int8 denoiser read from the hosted ConvRot int8 checkpoint as native Linears.
+
+Offload hooks cannot move torchao weights, so an offloaded load used to fetch the dense bf16 shards and quantise them
+at load (LTX-2.5: 36 GB staged, ~100 s of build, then a 23 GB cache write). The hosted file already holds per-row int8
+codes and float32 scales of the same ConvRot-rotated weights, which is exactly what ``NativeWeightOnlyLinear`` stores:
+this reads them into native Linears with the same layer set, rotation and activation path as the runtime build."""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Optional
+
+NATIVE_HOSTED_ENV = "UNSLOTH_VIDEO_NATIVE_HOSTED"
+_QDATA = "._weight_qdata"
+_SCALE = "._weight_scale"
+_ZERO_POINT = "._weight_zero_point"
+# Int8Tensor payloads that would change the activation math (static / pre-scaled activations): refused.
+_ACT_PAYLOADS = (
+ "._weight_act_quant_scale",
+ "._weight_act_quant_zero_point",
+ "._weight_act_pre_scale",
+)
+
+
+def native_hosted_enabled() -> bool:
+ raw = (os.environ.get(NATIVE_HOSTED_ENV) or "").strip().lower()
+ return raw not in ("0", "off", "false", "no")
+
+
+def native_hosted_family(fam: Any) -> bool:
+ """Whether ``fam`` opted in (its hosted int8 file is ConvRot safetensors) and the switch is on."""
+ return bool(getattr(fam, "native_offload_from_hosted_int8", False)) and native_hosted_enabled()
+
+
+def _log(logger: Any, level: str, msg: str) -> None:
+ if logger is not None:
+ getattr(logger, level)("video.native_hosted: %s", msg)
+
+
+def hosted_metadata(path: str) -> Optional[dict]:
+ """The checkpoint's ``unsloth_metadata``, or None. Header only."""
+ try:
+ from safetensors import safe_open
+ with safe_open(str(path), framework = "pt") as handle:
+ raw = (handle.metadata() or {}).get("unsloth_metadata")
+ return json.loads(raw) if raw else None
+ except Exception: # noqa: BLE001 - unreadable: not usable
+ return None
+
+
+def metadata_refusal(
+ meta: Any, spec: dict, *, family: Optional[str], component: str
+) -> Optional[str]:
+ """Why the hosted file is not the native build ``spec`` describes, or None."""
+ from .diffusion_convrot import CONVROT_KIND, ROTATION_FQNS_KEY, ROTATION_GROUP_KEY, ROTATION_KEY
+
+ if not isinstance(meta, dict):
+ return "no unsloth metadata"
+ if spec.get("scheme") != "int8" or not spec.get("act_int8"):
+ return f"the native build is {spec.get('scheme')} (act_int8={spec.get('act_int8')}), not W8A8 int8"
+ if meta.get("scheme") != "int8" or meta.get("quant_backend") != "torchao":
+ return f"checkpoint scheme {meta.get('scheme')!r} / backend {meta.get('quant_backend')!r}"
+ if family is not None and meta.get("family") != family:
+ return f"checkpoint family {meta.get('family')!r}, not {family!r}"
+ if (meta.get("component") or "transformer") != component:
+ return f"checkpoint component {meta.get('component')!r}, not {component!r}"
+ if str(meta.get("torch_dtype") or "bfloat16") != "bfloat16":
+ return f"checkpoint dtype {meta.get('torch_dtype')!r}"
+ if meta.get(ROTATION_KEY) != CONVROT_KIND:
+ return f"activation rotation {meta.get(ROTATION_KEY)!r}"
+ if int(meta.get(ROTATION_GROUP_KEY) or 0) != int(spec.get("rot_group") or 0):
+ return f"rotation group {meta.get(ROTATION_GROUP_KEY)!r}, the native build uses {spec.get('rot_group')!r}"
+ if not isinstance(meta.get(ROTATION_FQNS_KEY), (list, tuple)):
+ return "no rotation fqns"
+ if int(meta.get("min_features") or -1) != int(spec.get("min_features") or -2):
+ return f"min_features {meta.get('min_features')!r}"
+ excludes = meta.get("exclude_name_tokens")
+ if not isinstance(excludes, (list, tuple)):
+ return "no exclude tokens"
+ if sorted(set(excludes) | {"lora_"}) != list(spec.get("exclude_name_tokens") or ()):
+ return "its excluded layers differ from the native build's"
+ return None
+
+
+def load_native_from_hosted(
+ path: str,
+ transformer_cls: Any,
+ config_source: str,
+ *,
+ spec: dict,
+ family: Optional[str],
+ component: str = "transformer",
+ config_kwargs: Optional[dict] = None,
+ logger: Any = None,
+) -> Any:
+ """The native ``component`` built from hosted ``path``, or None (the caller falls back to the dense build). Never
+ raises. Refuses unless the file holds exactly the layers, rotation and activation path the runtime build makes."""
+ try:
+ import time
+
+ import torch
+
+ t0 = time.perf_counter()
+ meta = hosted_metadata(path)
+ problem = metadata_refusal(meta, spec, family = family, component = component)
+ if problem:
+ _log(logger, "info", f"hosted int8 checkpoint not used ({problem})")
+ return None
+ from accelerate import init_empty_weights
+ from safetensors.torch import load_file
+
+ from .diffusion_convrot import ROTATION_FQNS_KEY
+ from .diffusion_native_quant import native_linear_class
+ from .diffusion_prequant import HostedInt8Parts, _verify_packed_fingerprint
+ from .diffusion_transformer_quant import make_filter_fn
+
+ config = transformer_cls.load_config(
+ config_source, subfolder = component, **(config_kwargs or {})
+ )
+ with init_empty_weights(include_buffers = False):
+ model = transformer_cls.from_config(config)
+ group = int(spec["rot_group"])
+ # The runtime build's layer set (its require_bf16 holds: the whole load is bf16, checked above).
+ filter_fn = make_filter_fn(int(spec["min_features"]), tuple(spec["exclude_name_tokens"]))
+ wanted = {name for name, mod in model.named_modules() if name and filter_fn(mod, name)}
+ from .prequant_safetensors import UNSLOTH_ROOT_PREFIX
+
+ state = load_file(str(path))
+ # Root-level tensors are stored prefixed (the torchao unflatten splits keys on dots).
+ for key in [k for k in state if k.startswith(UNSLOTH_ROOT_PREFIX)]:
+ state[key[len(UNSLOTH_ROOT_PREFIX) :]] = state.pop(key)
+ quantised = {key[: -len(_QDATA)] for key in state if key.endswith(_QDATA)}
+ if quantised != wanted:
+ _log(
+ logger,
+ "info",
+ f"hosted int8 checkpoint not used ({len(quantised)} quantised layers, the native build makes "
+ f"{len(wanted)}; e.g. {sorted(quantised ^ wanted)[:2]})",
+ )
+ return None
+ if any(key.endswith(_ACT_PAYLOADS) for key in state):
+ _log(
+ logger,
+ "info",
+ "hosted int8 checkpoint not used (it carries static activation scales)",
+ )
+ return None
+ modules = dict(model.named_modules())
+ rotated = set(meta[ROTATION_FQNS_KEY])
+ expect_rotated = {fqn for fqn in wanted if int(modules[fqn].in_features) % group == 0}
+ if rotated != expect_rotated:
+ _log(
+ logger,
+ "info",
+ "hosted int8 checkpoint not used (its rotated layers differ from the native build's)",
+ )
+ return None
+ parts: dict = {}
+ for fqn in quantised:
+ codes = state[fqn + _QDATA]
+ scale = state[fqn + _SCALE]
+ zero = state.get(fqn + _ZERO_POINT)
+ lin = modules[fqn]
+ shape = (int(lin.out_features), int(lin.in_features))
+ if (
+ codes.dtype != torch.int8
+ or tuple(codes.shape) != shape
+ or scale.numel() != shape[0]
+ or not scale.is_floating_point()
+ or (zero is not None and bool(torch.any(zero != 0)))
+ ):
+ _log(
+ logger,
+ "info",
+ f"hosted int8 checkpoint not used ({fqn}: unexpected payload layout)",
+ )
+ return None
+ parts[fqn + ".weight"] = HostedInt8Parts(codes, scale, zero)
+ # The torchao load checks the same recorded digests: corruption after build must not render.
+ if not _verify_packed_fingerprint(parts, meta, logger = logger, path = path):
+ return None
+ del parts
+ cls = native_linear_class()
+ for fqn in sorted(quantised):
+ lin = modules[fqn]
+ parent_name, _, leaf = fqn.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ stand_in = torch.nn.Linear(
+ int(lin.in_features),
+ int(lin.out_features),
+ bias = lin.bias is not None,
+ device = "meta",
+ dtype = torch.bfloat16,
+ )
+ codes = state.pop(fqn + _QDATA)
+ scale = state.pop(fqn + _SCALE).to(torch.float32).reshape(-1).contiguous()
+ state.pop(fqn + _ZERO_POINT, None)
+ setattr(
+ parent,
+ leaf,
+ cls(
+ stand_in,
+ "int8",
+ act_int8 = True,
+ rot_group = group if fqn in rotated else 0,
+ codes = codes,
+ scale = scale,
+ ),
+ )
+ state[fqn + ".weight_q"] = codes
+ state[fqn + ".weight_scale"] = scale.view(torch.int32)
+ model.load_state_dict(state, strict = True, assign = True)
+ del state
+ if any(t.device.type == "meta" for t in (*model.parameters(), *model.buffers())):
+ _log(logger, "warning", "hosted int8 checkpoint left meta tensors; not used")
+ return None
+ if any(p.dtype != torch.bfloat16 for p in model.parameters() if p.is_floating_point()):
+ _log(logger, "info", "hosted int8 checkpoint not used (a non-bf16 dense tensor)")
+ return None
+ model.eval()
+ try:
+ model._unsloth_runtime_quant = "int8"
+ model._unsloth_runtime_quant_info = {
+ "scheme": "int8",
+ "native": True,
+ "hosted_path": str(path),
+ }
+ except Exception: # noqa: BLE001
+ pass
+ _log(
+ logger,
+ "info",
+ f"read {len(quantised)} int8 ConvRot layers of {component} from the hosted checkpoint as native "
+ f"Linears in {time.perf_counter() - t0:.1f} s (no dense shards, no runtime quantise)",
+ )
+ return model
+ except Exception as exc: # noqa: BLE001 - an unusable file falls back to the dense build
+ _log(logger, "warning", f"hosted int8 read failed: {type(exc).__name__}: {str(exc)[:200]}")
+ return None
diff --git a/studio/backend/core/inference/diffusion_prequant.py b/studio/backend/core/inference/diffusion_prequant.py
index fddd6aaf859..f8bafc0afae 100644
--- a/studio/backend/core/inference/diffusion_prequant.py
+++ b/studio/backend/core/inference/diffusion_prequant.py
@@ -1365,7 +1365,26 @@ _CANONICAL_PARTS: dict = {
"Float8Tensor": (("q", "qdata"), ("scale", "scale")),
"NVFP4Tensor": (("q", "qdata"), ("scale", "scale"), ("per_tensor_scale", "per_tensor_scale")),
"MXTensor": (("q", "qdata"), ("scale", "scale")),
+ "HostedInt8Parts": (("q", "qdata"), ("scale", "scale"), ("zero_point", "zero_point")),
}
+
+
+class HostedInt8Parts:
+ """An Int8Tensor's stored payload read as plain tensors (no torchao), fingerprinted like the Int8Tensor."""
+
+ __slots__ = ("qdata", "scale", "zero_point")
+
+ def __init__(
+ self,
+ qdata: Any,
+ scale: Any,
+ zero_point: Any = None,
+ ):
+ self.qdata = qdata
+ self.scale = scale
+ self.zero_point = zero_point
+
+
_CANONICAL_ROLE_ORDER = (
"q",
"scale",
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 2e78f733486..887ab80d224 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -3781,6 +3781,68 @@ def _read_native_quant_from_cache(
return {}
+def _read_native_quant_from_hosted(
+ fam: Any,
+ spec: Optional[dict],
+ *,
+ base: Optional[str],
+ source: Optional[str],
+ config_kwargs: dict,
+ hf_token: Optional[str] = None,
+ local_files_only: bool = False,
+) -> dict[str, Any]:
+ """``{component: module}`` for EVERY denoiser read from the hosted int8 file as native Linears, or ``{}`` (all or
+ nothing; the dense shards then load as before). Never raises."""
+ if spec is None:
+ return {}
+ try:
+ from utils.torch_warmup import close_dynamo_import_window
+ close_dynamo_import_window(logger) # a no-op once the load closed it above
+ except Exception as exc: # noqa: BLE001 - optimisation only
+ logger.debug("dynamo pre-import skipped: %r", exc)
+ try:
+ import diffusers
+
+ from .diffusion import hub_cache_dir
+ from .diffusion_native_hosted import load_native_from_hosted
+ from .diffusion_prequant import _resolve_checkpoint_path
+ from .video_denoiser_prequant import denoiser_prequant_sources
+
+ transformer_cls = getattr(diffusers, str(getattr(fam, "transformer_class", "")), None)
+ sources = denoiser_prequant_sources(fam, spec.get("scheme"), base, hf_token = hf_token)
+ if transformer_cls is None or not sources:
+ return {}
+ out: dict[str, Any] = {}
+ for component, src in sources.items():
+ path = _resolve_checkpoint_path(
+ src,
+ hf_token,
+ hub_cache_dir(),
+ local_files_only = local_files_only,
+ scheme = spec.get("scheme"),
+ logger = logger,
+ )
+ if path is None:
+ return {}
+ module = load_native_from_hosted(
+ path,
+ transformer_cls,
+ str(source or base),
+ spec = spec,
+ family = getattr(fam, "name", None),
+ component = component,
+ config_kwargs = config_kwargs,
+ logger = logger,
+ )
+ if module is None:
+ return {}
+ out[component] = module
+ return out
+ except Exception as exc: # noqa: BLE001 - a hosted problem keeps the dense build
+ logger.warning("video.native_hosted: hosted read skipped (%s)", exc)
+ return {}
+
+
def _video_native_stream_build(
fam: Any,
spec: Optional[dict],
@@ -4163,6 +4225,26 @@ def _auto_offload_scheme(requested: Optional[str]) -> Optional[str]:
return TQ_INT8 if scheme == TQ_AUTO else scheme
+def _video_native_hosted_scheme(fam: Any, scheme: Optional[str], target: Any) -> Optional[str]:
+ """``int8`` when an offloaded load of ``scheme`` builds the W8A8 ConvRot native DiT that the family's hosted int8
+ checkpoint already holds (``diffusion_native_hosted``), else None. The load re-checks the file; never raises."""
+ try:
+ from .diffusion_native_hosted import native_hosted_family
+ from .diffusion_native_quant import int8_rotation_group
+
+ if scheme != TQ_INT8 or not native_hosted_family(fam):
+ return None
+ if getattr(target, "device", None) != "cuda":
+ return None
+ if native_quant_scheme(target, TQ_INT8, family = fam.name, offload = True) != TQ_INT8:
+ return None
+ if not native_int8_act(target) or int(int8_rotation_group() or 0) <= 0:
+ return None
+ return TQ_INT8
+ except Exception: # noqa: BLE001 -- unanswerable: the dense shards and runtime build, as before
+ return None
+
+
# Denoise-side base of estimate_video_runtime_mib; the rest scales with the decoded clip, held only by the VAE.
_VIDEO_DENOISE_ACTIVATION_MIB = 4096
@@ -5603,6 +5685,7 @@ class VideoBackend:
if kind != "single_file"
else None
)
+ native_hosted_out: list = []
video_auto_denoiser = self._video_planned_auto_denoiser_scheme(
fam,
base = base,
@@ -5614,6 +5697,7 @@ class VideoBackend:
gpu_ordinal = kwargs.get("gpu_ordinal"),
hf_token = kwargs.get("hf_token"),
ltx25_dev = ltx25_dev,
+ native_hosted = native_hosted_out,
)
video_seed_declined = video_auto_denoiser == DENOISER_SEED_DECLINED
if video_seed_declined:
@@ -5631,6 +5715,32 @@ class VideoBackend:
local_files_only = local_files_only,
kind = kind,
)
+ # Offloaded int8 read from the hosted file as native Linears: its dense shards are not staged either.
+ native_hosted_planned = None
+ if (
+ video_seed_declined
+ and native_hosted_out
+ and not skip_transformer_weights
+ and self._denoiser_prequant_verified(
+ fam,
+ native_hosted_out[0],
+ base,
+ kwargs.get("h3_task"),
+ kwargs.get("hf_token"),
+ local_files_only = local_files_only,
+ kind = kind,
+ )
+ ):
+ skip_transformer_weights = True
+ native_hosted_planned = native_hosted_out[0]
+ kwargs["_video_native_hosted_planned"] = native_hosted_planned
+ # The scheme whose hosted artifact replaces the dense shards (claimed, then fetched under the cancel event).
+ planned_denoiser = (
+ h3_auto_denoiser
+ or video_auto_denoiser
+ or requested_denoiser
+ or native_hosted_planned
+ )
skip_transformer_components = (
_planned_denoiser_components(fam, kind) if skip_transformer_weights else ()
)
@@ -5694,7 +5804,7 @@ class VideoBackend:
if skip_transformer_weights:
claimed = self._denoiser_prequant_repo_ids(
fam,
- h3_auto_denoiser or video_auto_denoiser or requested_denoiser,
+ planned_denoiser,
base,
kwargs.get("h3_task"),
)
@@ -5854,7 +5964,7 @@ class VideoBackend:
self._fetch_denoiser_prequant(
self._denoiser_prequant_source_list(
fam,
- h3_auto_denoiser or video_auto_denoiser or requested_denoiser,
+ planned_denoiser,
base,
kwargs.get("h3_task"),
hf_token = kwargs.get("hf_token"),
@@ -5862,7 +5972,7 @@ class VideoBackend:
kwargs.get("hf_token"),
cancel_event = cancel_event,
local_files_only = local_files_only,
- scheme = h3_auto_denoiser or video_auto_denoiser or requested_denoiser,
+ scheme = planned_denoiser,
)
base_plan = planned_files.get(_FETCH_BASE)
actual_te_skip = tuple(te_skipped) + tuple(h3_te_skipped)
@@ -7035,8 +7145,12 @@ class VideoBackend:
gpu_ordinal: Optional[int] = None,
hf_token: Optional[str] = None,
ltx25_dev: bool = False,
+ native_hosted: Optional[list] = None,
) -> Optional[str]:
- """The pre-download auto denoiser scheme, None, or ``DENOISER_SEED_DECLINED`` if the plan would offload."""
+ """The pre-download auto denoiser scheme, None, or ``DENOISER_SEED_DECLINED`` if the plan would offload.
+
+ On a decline, ``native_hosted`` (a list) receives the scheme when the offloaded native build can be read from
+ the hosted checkpoint instead (``_video_native_hosted_scheme``)."""
try:
if kind != "pipeline" or getattr(fam, "modular_workflow", None):
return None
@@ -7103,6 +7217,17 @@ class VideoBackend:
ltx25_dev = ltx25_dev,
device_memory_reader = _settled_once,
):
+ hosted = _video_native_hosted_scheme(fam, scheme, target)
+ if native_hosted is not None and hosted is not None:
+ native_hosted.append(hosted)
+ logger.info(
+ "video.denoiser_prequant: an artifact-sized plan for %s still offloads on "
+ "this card, so the hosted %s checkpoint is read as offload-safe native "
+ "Linears instead of quantising the dense denoiser shards",
+ scheme,
+ hosted,
+ )
+ return DENOISER_SEED_DECLINED
logger.info(
"video.denoiser_prequant: an artifact-sized plan for %s still offloads on "
"this card, and offload moves the DiT, so the dense denoiser shards are "
@@ -7780,6 +7905,7 @@ class VideoBackend:
if kind != "single_file"
else None
) or transformer_quant
+ native_hosted_out: list = []
video_planned = self._video_planned_auto_denoiser_scheme(
fam,
base = base,
@@ -7791,9 +7917,11 @@ class VideoBackend:
gpu_ordinal = load_kwargs.get("gpu_ordinal"),
hf_token = hf_token,
ltx25_dev = ltx25_dev,
+ native_hosted = native_hosted_out,
)
if video_planned == DENOISER_SEED_DECLINED:
- video_planned = None
+ # An offloaded int8 load reads the hosted file as native Linears: plan that file, not the dense shards.
+ video_planned = native_hosted_out[0] if native_hosted_out else None
if kind == "pipeline" and not getattr(fam, "modular_workflow", None):
transformer_quant = video_planned
else:
@@ -8630,6 +8758,7 @@ class VideoBackend:
_te_prequant_failed: Optional[dict] = None,
_h3_auto_denoiser_planned: Optional[str] = None,
_video_auto_denoiser_planned: Optional[str] = None,
+ _video_native_hosted_planned: Optional[str] = None,
_denoiser_prequant_skipped: tuple[str, ...] = (),
) -> dict[str, Any]:
fam = self.validate_load_request(
@@ -9415,6 +9544,25 @@ class VideoBackend:
cancel_check = lambda: self._raise_if_video_load_superseded(_load_token),
)
self._raise_if_video_load_superseded(_load_token)
+ native_from_hosted = False
+ if (
+ not native_cache_hits
+ and native_cache_spec is not None
+ and _video_native_hosted_planned is not None
+ and _video_native_hosted_planned == native_cache_spec.get("scheme")
+ ):
+ # The planner staged the hosted int8 file instead of the dense shards: read it as the native build.
+ native_cache_hits = _read_native_quant_from_hosted(
+ fam,
+ native_cache_spec,
+ base = fetch_base,
+ source = _base_local_dir or fetch_base,
+ config_kwargs = native_config_kwargs,
+ hf_token = hf_token,
+ local_files_only = local_files_only,
+ )
+ native_from_hosted = bool(native_cache_hits)
+ self._raise_if_video_load_superseded(_load_token)
if not native_cache_hits and native_cache_spec is not None:
# A miss whose dense load would not fit host RAM: build the same native experts per shard into the cache.
native_cache_hits = _video_native_stream_build(
@@ -9945,6 +10093,18 @@ class VideoBackend:
denoiser_seed_scheme,
fam.name,
)
+ elif native_cached and native_from_hosted:
+ transformer_quant_engaged = native_scheme
+ transformer_quant_source = (
+ native_quant_reason(getattr(pipe, "transformer", None), native_scheme)
+ + "; codes and scales read from the hosted pre-quantized checkpoint"
+ )
+ logger.info(
+ "video.transformer_quant: %s native denoiser(s) read from the hosted checkpoint for "
+ "%s; skipping the dense shards and the runtime quantise",
+ native_scheme,
+ fam.name,
+ )
elif native_cached:
transformer_quant_engaged = native_scheme
transformer_quant_source = (
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index 38992b969e6..58bbf3bec89 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -304,6 +304,9 @@ class VideoFamily:
prequant_filenames: tuple[tuple[str, ...], ...] = field(default_factory = tuple)
# Separate partition, served ONLY by its own (scheme, task, filename) row or it loads the wrong one.
prequant_partition_tasks: tuple[str, ...] = field(default_factory = tuple)
+ # An offloaded int8 load reads the hosted int8 file's ConvRot codes as native Linears (diffusion_native_hosted)
+ # instead of staging and quantising the dense shards. Only for a family whose hosted int8 file is ConvRot safetensors.
+ native_offload_from_hosted_int8: bool = False
# Modular workflow loaded instead of a DiffusionPipeline; its routing blocks are not pruned.
modular_workflow: Optional[str] = None
# Released video and audio sigma shifts, when configurable.
@@ -672,6 +675,7 @@ _FAMILIES: tuple[VideoFamily, ...] = (
("lightricks/ltx-2.5-diffusers", "int8", "unsloth/LTX-2.5-FP8"),
("lightricks/ltx-2.5-diffusers", "fp8", "unsloth/LTX-2.5-FP8"),
),
+ native_offload_from_hosted_int8 = True,
# File sizes (decimal GB): int8 keeps the audio stream, AV cross-attention and adaLN in bf16; fp8 quantizes them.
prequant_resident_gb_by_scheme = (("int8", 25.11), ("fp8", 19.06)),
# Dev DiT at the root, distilled under distilled/; the pack above supplies VAEs and connectors.
diff --git a/studio/backend/tests/test_video_native_hosted.py b/studio/backend/tests/test_video_native_hosted.py
new file mode 100644
index 00000000000..5732ef9cdce
--- /dev/null
+++ b/studio/backend/tests/test_video_native_hosted.py
@@ -0,0 +1,290 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An offloaded LTX-2.5 int8 load reads the hosted ConvRot int8 file as native Linears instead of quantising the
+dense shards: same layer set, rotation and activation path as the runtime native build, codes and scales from the
+file, verified against its recorded fingerprint, and refused (dense build) whenever the file is not that build."""
+
+from __future__ import annotations
+
+import json
+import sys
+import types
+from pathlib import Path
+
+import pytest
+
+torch = pytest.importorskip("torch")
+diffusers = pytest.importorskip("diffusers")
+pytest.importorskip("safetensors")
+pytest.importorskip("accelerate")
+if not hasattr(diffusers, "LTX2VideoTransformer3DModel"):
+ pytest.skip("diffusers without LTX-2", allow_module_level = True)
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import diffusion_native_hosted as NH # noqa: E402
+from core.inference import video as V # noqa: E402
+from core.inference.diffusion_convrot import ( # noqa: E402
+ CONVROT_KIND,
+ ROTATION_FQNS_KEY,
+ ROTATION_GROUP_KEY,
+ ROTATION_KEY,
+ quantize_convrot_int8_weight,
+)
+from core.inference.diffusion_native_cache import native_spec # noqa: E402
+from core.inference.diffusion_native_quant import ( # noqa: E402
+ apply_native_weight_quant,
+ is_native_linear,
+)
+from core.inference.diffusion_prequant import ( # noqa: E402
+ CANONICAL_FINGERPRINT_ALGO,
+ HostedInt8Parts,
+ canonical_weight_digest,
+ canonical_weight_fingerprint,
+)
+from core.inference.diffusion_transformer_quant import make_filter_fn # noqa: E402
+from core.inference.prequant_safetensors import UNSLOTH_ROOT_KEYS_KEY, UNSLOTH_ROOT_PREFIX # noqa: E402
+from core.inference.video_families import detect_video_family # noqa: E402
+
+FAMILY = "ltx-2.5"
+CLS = diffusers.LTX2VideoTransformer3DModel
+
+
+def _config():
+ # 2 heads x 256 = 512 wide: the video Linears pass the 512 filter and g256 ConvRot divides them.
+ return dict(
+ in_channels = 32,
+ out_channels = 32,
+ num_attention_heads = 2,
+ attention_head_dim = 256,
+ cross_attention_dim = 512,
+ caption_channels = 512,
+ audio_in_channels = 16,
+ audio_out_channels = 16,
+ audio_num_attention_heads = 2,
+ audio_attention_head_dim = 32,
+ audio_cross_attention_dim = 64,
+ num_layers = 1,
+ )
+
+
+def _spec():
+ return native_spec("int8", FAMILY, act_int8 = True)
+
+
+def _dense():
+ torch.manual_seed(0)
+ return CLS(**_config()).to(torch.bfloat16).eval()
+
+
+def _filter(spec, require_bf16 = False):
+ return make_filter_fn(
+ int(spec["min_features"]), tuple(spec["exclude_name_tokens"]), require_bf16 = require_bf16
+ )
+
+
+def _write_hosted(
+ root: Path,
+ *,
+ mutate = None,
+ meta_patch = None,
+) -> tuple[Path, Path, dict]:
+ """A tiny hosted-format int8 ConvRot file (torchao Int8Tensor layout) plus the dense config dir."""
+ from safetensors.torch import save_file
+
+ spec = _spec()
+ model = _dense()
+ cfg_dir = root / "base"
+ model.save_config(str(cfg_dir / "transformer"))
+ group = int(spec["rot_group"])
+ keep = _filter(spec, require_bf16 = True)
+ quantised = sorted(n for n, m in model.named_modules() if n and keep(m, n))
+ state = {k: v.contiguous() for k, v in model.state_dict().items()}
+ rotated = []
+ parts = {}
+ for fqn in quantised:
+ weight = state.pop(fqn + ".weight")
+ if weight.shape[1] % group == 0:
+ codes, scale = quantize_convrot_int8_weight(weight, group)
+ rotated.append(fqn)
+ else:
+ w = weight.float()
+ scale = w.abs().amax(dim = 1, keepdim = True) / 127.0
+ codes = (w / scale).round().clamp(-128, 127).to(torch.int8)
+ zero = torch.zeros(codes.shape[0], 1, dtype = torch.int8)
+ state[fqn + "._weight_qdata"] = codes.contiguous()
+ state[fqn + "._weight_scale"] = scale.float().contiguous()
+ state[fqn + "._weight_zero_point"] = zero
+ parts[fqn + ".weight"] = HostedInt8Parts(codes, scale.float(), zero)
+ fingerprint = canonical_weight_fingerprint(parts)
+ meta = {
+ "base_model_id": "Lightricks/LTX-2.5-Diffusers",
+ "family": FAMILY,
+ "scheme": "int8",
+ "min_features": spec["min_features"],
+ "exclude_name_tokens": [t for t in spec["exclude_name_tokens"] if t != "lora_"],
+ "torch_dtype": "bfloat16",
+ "quant_backend": "torchao",
+ "transformer_class": "LTX2VideoTransformer3DModel",
+ "component": "transformer",
+ "fingerprint": {
+ "algo": CANONICAL_FINGERPRINT_ALGO,
+ "count": fingerprint["count"],
+ "canonical_modules": fingerprint["canonical_modules"],
+ },
+ ROTATION_KEY: CONVROT_KIND,
+ ROTATION_GROUP_KEY: group,
+ ROTATION_FQNS_KEY: rotated,
+ }
+ if meta_patch:
+ meta.update(meta_patch)
+ if mutate:
+ mutate(state)
+ # The hosted layout: root-level tensors under a prefix (unflatten splits keys on dots).
+ roots = [k for k in state if "." not in k]
+ assert roots, "the tiny model should have a root-level tensor (scale_shift_table)"
+ stored = {(f"{UNSLOTH_ROOT_PREFIX}{k}" if k in roots else k): v for k, v in state.items()}
+ path = root / "LTX-2.5-INT8-ConvRot.safetensors"
+ save_file(
+ stored,
+ str(path),
+ metadata = {"unsloth_metadata": json.dumps(meta), UNSLOTH_ROOT_KEYS_KEY: json.dumps(roots)},
+ )
+ return path, cfg_dir, {"quantised": quantised, "rotated": rotated, "state": state}
+
+
+def _load(
+ path,
+ cfg_dir,
+ spec = None,
+ family = FAMILY,
+):
+ return NH.load_native_from_hosted(
+ str(path), CLS, str(cfg_dir), spec = spec or _spec(), family = family, logger = None
+ )
+
+
+def test_reads_the_hosted_codes_into_the_runtime_native_layout(tmp_path):
+ path, cfg_dir, built = _write_hosted(tmp_path)
+ model = _load(path, cfg_dir)
+ assert model is not None
+ native = {n: m for n, m in model.named_modules() if is_native_linear(m)}
+ assert sorted(native) == built["quantised"]
+ for fqn, layer in native.items():
+ assert layer.act_int8 and layer.scheme == "int8"
+ assert layer.rot_group == (256 if fqn in built["rotated"] else 0)
+ assert torch.equal(layer.weight_q, built["state"][fqn + "._weight_qdata"])
+ assert torch.equal(
+ layer.weight_scale.view(torch.float32),
+ built["state"][fqn + "._weight_scale"].reshape(-1),
+ )
+ # The runtime native build of the dense model swaps exactly these layers, the same way.
+ runtime = _dense()
+ apply_native_weight_quant(
+ runtime, "int8", filter_fn = _filter(_spec(), require_bf16 = True), act_int8 = True
+ )
+ expect = {n: m for n, m in runtime.named_modules() if is_native_linear(m)}
+ assert sorted(expect) == sorted(native)
+ for fqn, layer in expect.items():
+ assert (layer.rot_group, layer.act_int8, layer.in_features, layer.out_features) == (
+ native[fqn].rot_group,
+ native[fqn].act_int8,
+ native[fqn].in_features,
+ native[fqn].out_features,
+ )
+ # Every dense (non-quantised) tensor is the dense model's, bit for bit, in bf16.
+ dense = runtime.state_dict()
+ for key, value in model.state_dict().items():
+ if key.endswith((".weight_q", ".weight_scale")):
+ continue
+ assert value.dtype == dense[key].dtype and torch.equal(value, dense[key]), key
+ assert model._unsloth_runtime_quant == "int8"
+
+
+ "case",
+ [
+ "rotation_group",
+ "family",
+ "unrotated_kind",
+ "excludes",
+ "missing_layer",
+ "static_act",
+ "corrupt_codes",
+ ],
+)
+def test_refuses_a_file_that_is_not_the_native_build(tmp_path, case):
+ first = None
+
+ def drop_layer(state):
+ nonlocal first
+ first = sorted(k for k in state if k.endswith("._weight_qdata"))[0][
+ : -len("._weight_qdata")
+ ]
+ for suffix in ("._weight_qdata", "._weight_scale", "._weight_zero_point"):
+ state.pop(first + suffix)
+
+ def static_act(state):
+ key = sorted(k for k in state if k.endswith("._weight_qdata"))[0]
+ state[key[: -len("._weight_qdata")] + "._weight_act_quant_scale"] = torch.ones(1)
+
+ def corrupt(state):
+ key = sorted(k for k in state if k.endswith("._weight_qdata"))[0]
+ state[key] = state[key].clone()
+ state[key][0, 0] = -state[key][0, 0] if state[key][0, 0] != 0 else 1
+
+ kwargs = {
+ "rotation_group": {"meta_patch": {ROTATION_GROUP_KEY: 64}},
+ "family": {"meta_patch": {"family": "ltx-2"}},
+ "unrotated_kind": {"meta_patch": {ROTATION_KEY: None}},
+ "excludes": {"meta_patch": {"exclude_name_tokens": ["norm"]}},
+ "missing_layer": {"mutate": drop_layer},
+ "static_act": {"mutate": static_act},
+ "corrupt_codes": {"mutate": corrupt},
+ }[case]
+ path, cfg_dir, _ = _write_hosted(tmp_path, **kwargs)
+ assert _load(path, cfg_dir) is None
+
+
+def test_refuses_a_weight_only_or_unrotated_native_build(tmp_path):
+ path, cfg_dir, _ = _write_hosted(tmp_path)
+ assert _load(path, cfg_dir, spec = native_spec("int8", FAMILY, act_int8 = False)) is None
+ assert _load(path, cfg_dir, spec = native_spec("fp8", FAMILY, act_int8 = False)) is None
+
+
+def test_hosted_parts_fingerprint_like_the_torchao_int8_tensor():
+ torchao_q = pytest.importorskip("torchao.quantization")
+ if not hasattr(torchao_q, "Int8Tensor"):
+ pytest.skip("torchao without Int8Tensor")
+ from torchao.quantization.granularity import PerRow
+
+ torch.manual_seed(0)
+ weight = torch.randn(64, 256, dtype = torch.bfloat16)
+ tensor = torchao_q.Int8Tensor.from_hp(weight, PerRow())
+ parts = HostedInt8Parts(tensor.qdata, tensor.scale, getattr(tensor, "zero_point", None))
+ assert canonical_weight_digest(tensor) == canonical_weight_digest(parts) is not None
+
+
+def _target():
+ return types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16, ordinal = 0)
+
+
+def test_only_an_opted_in_family_offload_int8_takes_the_hosted_file(monkeypatch):
+ monkeypatch.setattr(V, "native_quant_scheme", lambda *a, **k: "int8")
+ monkeypatch.setattr(V, "native_int8_act", lambda target: True)
+ ltx25 = detect_video_family("Lightricks/LTX-2.5-Diffusers")
+ assert ltx25 is not None and ltx25.name == FAMILY
+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) == "int8"
+ assert V._video_native_hosted_scheme(ltx25, "fp8", _target()) is None
+ cpu = types.SimpleNamespace(device = "cpu", dtype = torch.bfloat16, ordinal = None)
+ assert V._video_native_hosted_scheme(ltx25, "int8", cpu) is None
+ wan = detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers")
+ assert wan is not None and V._video_native_hosted_scheme(wan, "int8", _target()) is None
+ monkeypatch.setattr(V, "native_int8_act", lambda target: False)
+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) is None
+ monkeypatch.setattr(V, "native_int8_act", lambda target: True)
+ monkeypatch.setenv(NH.NATIVE_HOSTED_ENV, "0")
+ assert V._video_native_hosted_scheme(ltx25, "int8", _target()) is None