Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: LTX-2.5 defaults follow ComfyUI (hosted INT8-ConvRot DiT, opt-in int8 text encoder) (#13618)
Daniel Han4e282d4f0666
diff --git a/studio/backend/core/inference/diffusion_convrot.py b/studio/backend/core/inference/diffusion_convrot.pyindex a1962f0fed3..95643f16f61 100644--- a/studio/backend/core/inference/diffusion_convrot.py+++ b/studio/backend/core/inference/diffusion_convrot.py@@ -162,6 +162,30 @@ def rotate_convrot_weight_(module: Any, group_size: int) -> None:module.weight.data = (blocks @ h.T).reshape(out_features, in_features).to(weight.dtype)+def quantize_convrot_int8_weight(weight: Any, group_size: int) -> tuple[Any, Any]:+ """``(int8 codes, float32 per-row scale)`` of ``W @ blockdiag(H).T``, matching comfy-kitchen+ ``quantize_int8_convrot_weight``. Float32 with TF32 off on purpose: bf16 rotation changes ~9% of the codes."""+ import torch++ out_features, in_features = weight.shape+ if in_features % group_size:+ raise ValueError(+ f"in_features {in_features} is not divisible by the ConvRot group {group_size}"+ )+ h = build_convrot_hadamard(group_size, device = weight.device, dtype = torch.float32)+ blocks = weight.float().reshape(out_features, in_features // group_size, group_size)+ tf32 = torch.backends.cuda.matmul.allow_tf32+ torch.backends.cuda.matmul.allow_tf32 = False+ try:+ rotated = (blocks @ h.T).reshape(out_features, in_features)+ finally:+ torch.backends.cuda.matmul.allow_tf32 = tf32+ del blocks+ scale = (rotated.abs().amax(dim = -1, keepdim = True) / 127.0).clamp(min = 1e-30)+ codes = rotated.div_(scale).round_().clamp_(-128.0, 127.0).to(torch.int8)+ return codes, scale++@lru_cache(maxsize = None)def convrot_linear_class() -> Any:"""The ``nn.Linear`` subclass that rotates its input, built lazily so importing this modulediff --git a/studio/backend/core/inference/diffusion_fast_load.py b/studio/backend/core/inference/diffusion_fast_load.pyindex 8df29698729..aa3d477fbb9 100644--- a/studio/backend/core/inference/diffusion_fast_load.py+++ b/studio/backend/core/inference/diffusion_fast_load.py@@ -3,8 +3,9 @@"""Cold-start I/O for the image load path, byte-for-byte neutral: ``start_load_prefetch`` warms the pagecache with the files the load reads next (parallel slice reads beat mmap page faults); ``fast_upload``-moves plain host tensors through a pinned ring and hands them to the existing ``.to`` via a-``TorchFunctionMode``, so placement semantics stay the module's own. Kill switches+moves plain host tensors (and the inner tensors of torchao-style wrapper subclasses) through a pinned+ring and hands them to the existing ``.to`` via a ``TorchFunctionMode``, so placement semantics stay+the module's own. Kill switches``UNSLOTH_DIFFUSION_PREFETCH=0`` / ``UNSLOTH_DIFFUSION_FAST_UPLOAD=0``."""@@ -372,6 +373,63 @@ def _upload_candidates(modules: Sequence[Any], device: Any) -> list[Any]:return out+def _wrapper_subclass_parts(tensor: Any) -> Optional[tuple[list[str], Any, list[Any]]]:+ """``(inner names, flatten context, inner tensors)`` of a CPU torchao-style wrapper subclass with plain dense+ contiguous CPU inner tensors; else None. Its own ``.to`` copies inside ``__torch_dispatch__``, bypassing the ring."""+ import torch++ try:+ from torch.utils._python_dispatch import is_traceable_wrapper_subclass+ except ImportError: # noqa: BLE001 - an older torch keeps the subclass's own .to+ return None+ try:+ if not is_traceable_wrapper_subclass(tensor) or tensor.device.type != "cpu":+ return None+ if not callable(getattr(type(tensor), "__tensor_unflatten__", None)):+ return None+ names, ctx = tensor.__tensor_flatten__()+ inner = [getattr(tensor, name) for name in names]+ except Exception: # noqa: BLE001+ return None+ # A context that records a device (torchao's ``device`` attribute) would come back still naming the host.+ values = (+ ctx.values() if isinstance(ctx, dict) else ctx if isinstance(ctx, (list, tuple)) else (ctx,)+ )+ if (isinstance(ctx, dict) and "device" in ctx) or any(+ isinstance(v, torch.device) for v in values+ ):+ return None+ for part in inner:+ if type(part) is not torch.Tensor or part.device.type != "cpu":+ return None+ if (+ part.is_sparse+ or part.is_quantized+ or not part.is_contiguous()+ or part.storage_offset() < 0+ ):+ return None+ return list(names), ctx, inner+++def _subclass_candidates(+ modules: Sequence[Any], plain_seen: set[int]+) -> list[tuple[Any, list[str], Any, list[Any]]]:+ out: list[tuple[Any, list[str], Any, list[Any]]] = []+ seen: set[int] = set()+ for module in modules:+ if module is None or not hasattr(module, "parameters"):+ continue+ for tensor in list(module.parameters()) + list(module.buffers()):+ if id(tensor) in seen or id(tensor) in plain_seen:+ continue+ seen.add(id(tensor))+ parts = _wrapper_subclass_parts(tensor)+ if parts is not None:+ out.append((tensor, *parts))+ return out++def _ring_copy(sources: list[Any], device: Any) -> list[Any]:"""Device copies of ``sources`` (byte-identical), read through a pinned ring on worker threads."""from concurrent.futures import ThreadPoolExecutor@@ -472,18 +530,30 @@ def fast_upload(if target.index is None:target = torch.device("cuda", torch.cuda.current_device())sources = _upload_candidates(modules, target)+ subclasses = _subclass_candidates(modules, {id(t) for t in sources})+ known = {id(t) for t in sources}+ for _tensor, _names, _ctx, inner in subclasses:+ for part in inner:+ if id(part) not in known:+ known.add(id(part))+ sources.append(part)total = sum(t.numel() * t.element_size() for t in sources)if sources and total >= _UPLOAD_MIN_TOTAL_BYTES:with torch.no_grad():copies = _ring_copy(sources, target)staged = {id(src): (src, dst) for src, dst in zip(sources, copies)}- mode = _StagedTo(staged, target)+ wrapped = {+ id(tensor): (tensor, names, ctx, inner)+ for tensor, names, ctx, inner in subclasses+ }+ mode = _StagedTo(staged, target, wrapped)if logger is not None:logger.info(- "diffusion.fast_upload: staged %.1f GiB in %d tensor(s) onto %s",+ "diffusion.fast_upload: staged %.1f GiB in %d tensor(s) onto %s%s",total / 2**30,len(sources),target,+ f" ({len(wrapped)} quantized weight(s))" if wrapped else "",)except Exception as exc: # noqa: BLE001 - the stock .to() still runs inside the blockif logger is not None:@@ -507,10 +577,36 @@ def _staged_to_class():class _StagedToMode(TorchFunctionMode):"""Answers ``Tensor.to`` for a staged source with its device copy; every other call runs as usual."""- def __init__(self, staged: dict[int, tuple[Any, Any]], device: Any):+ def __init__(+ self,+ staged: dict[int, tuple[Any, Any]],+ device: Any,+ wrapped: Optional[dict[int, tuple[Any, list[str], Any, list[Any]]]] = None,+ ):super().__init__()self.staged = stagedself.device = device+ self.wrapped = wrapped or {}++ def _wrapped_copy(self, tensor: Any, device: Any, dtype: Any) -> Any:+ hit = self.wrapped.get(id(tensor))+ if hit is None or hit[0] is not tensor:+ return None+ if not _same_device(device, self.device) or (+ dtype is not None and dtype != tensor.dtype+ ):+ return None+ _tensor, names, ctx, inner = hit+ parts = {}+ for name, part in zip(names, inner):+ staged = self.staged.get(id(part))+ if staged is None or staged[0] is not part:+ return None+ parts[name] = staged[1]+ out = type(tensor).__tensor_unflatten__(parts, ctx, tensor.size(), tensor.stride())+ if type(out) is not type(tensor) or not _same_device(out.device, self.device):+ return None+ return outdef __torch_function__(self,@@ -542,10 +638,30 @@ def _staged_to_class():if args[0].requires_grad and torch.is_grad_enabled():out = out.requires_grad_(True)return out+ elif self.wrapped and id(args[0]) in self.wrapped:+ try:+ device, dtype, _non_blocking, memory_format = torch._C._nn._parse_to(+ *args[1:], **kwargs+ )+ out = None+ if (+ device is not None+ and memory_format is None+ and not kwargs.get("copy", False)+ ):+ out = self._wrapped_copy(args[0], device, dtype)+ except Exception: # noqa: BLE001 - the subclass's own .to below+ out = None+ if out is not None:+ return outreturn func(*args, **kwargs)return _StagedToMode-def _StagedTo(staged: dict[int, tuple[Any, Any]], device: Any) -> Any:- return _staged_to_class()(staged, device)+def _StagedTo(+ staged: dict[int, tuple[Any, Any]],+ device: Any,+ wrapped: Optional[dict[int, tuple[Any, list[str], Any, list[Any]]]] = None,+) -> Any:+ return _staged_to_class()(staged, device, wrapped)diff --git a/studio/backend/core/inference/diffusion_te_prequant.py b/studio/backend/core/inference/diffusion_te_prequant.pyindex b7f63ce38f7..10811d336c1 100644--- a/studio/backend/core/inference/diffusion_te_prequant.py+++ b/studio/backend/core/inference/diffusion_te_prequant.py@@ -150,6 +150,10 @@ def te_prequant_budget_scale(Best-effort like the rest of this module: anything unresolvable returns 1.0, i.e. today'sbf16 budget."""+ if getattr(fam, "name", None) == "ltx-2.5":+ from .video_ltx25 import ltx25_te_budget_scale+ components = getattr(fam, "bf16_components_gb", None) or (0.0, 0.0)+ return ltx25_te_budget_scale(te_quant_mode, components[1])try:sources = te_prequant_sources_for_base(fam,diff --git a/studio/backend/core/inference/family_profile.py b/studio/backend/core/inference/family_profile.pyindex 59f2aab71a1..2e6f93c6007 100644--- a/studio/backend/core/inference/family_profile.py+++ b/studio/backend/core/inference/family_profile.py@@ -83,6 +83,8 @@ class FamilyProfile:auto_prefer: Optional[AutoPrefer] = None# Reason auto keeps bf16 while bf16 fits (INT8 still picked once bf16 would offload); None = not this family.auto_bf16_when_resident: Optional[str] = None+ # Video: prequant DiT schemes AUTO seeds even where bf16 fits resident (else auto keeps resident bf16).+ auto_prequant_when_resident: frozenset[str] = frozenset()# Text-encoder int8 keep-bf16 schedule (skip_first, skip_last) decoder blocks; None = int8 falls back to fp8.te_int8_skip: Optional[tuple[int, int]] = Noneuntiled_vae: Optional[UntiledVae] = Nonediff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 79b22bb5b05..dc2d06116af 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -692,6 +692,8 @@ def _assert_video_precision_for_target(and h3_te_quant_scheme(te_mode) is not None):return+ if getattr(fam, "name", None) == "ltx-2.5":+ returnte_effective = effective_te_quant(te_mode, getattr(fam, "name", None))te_reason = Noneif te_quant_needs_resident_weights(te_effective) and not torchao_quantize_importable():@@ -2098,13 +2100,21 @@ def _pipeline_device_mib(pipe: Any, ordinal: Optional[int] = None) -> int:return total // (1024 * 1024)-def _auto_seed_yields_to_resident_bf16(scheme: Optional[str], requested: Optional[str]) -> bool:+def _auto_seed_yields_to_resident_bf16(+ scheme: Optional[str],+ requested: Optional[str],+ fam: Any = None,+) -> bool:"""Whether an AUTO seed of ``scheme`` is bound by "auto keeps a resident bf16 DiT".nvfp4 is exempt: it reaches auto only through a gated family preference (``_FAMILY_AUTO_PREFER``) that is- measured to be worth taking on a card that holds bf16. An explicit scheme is a request, honored as asked."""+ measured to be worth taking on a card that holds bf16. So is a scheme in the family's+ ``auto_prequant_when_resident``. An explicit scheme is a request, honored as asked."""if scheme is None or scheme == DENOISER_SEED_DECLINED or scheme == TQ_NVFP4:return False+ profile = getattr(fam, "profile", None)+ if scheme in (getattr(profile, "auto_prequant_when_resident", None) or ()):+ return Falsereturn requested is None or normalize_transformer_quant(requested) == TQ_AUTO@@ -3330,11 +3340,9 @@ class VideoBackend:normalize_transformer_quant(transformer_quant)# Reject a malformed text_encoder_quant the same way. Every kind: an unsupported scheme is a bad request# regardless of whether this family has a hosted quantized encoder for it.- if normalize_te_quant(text_encoder_quant) is not None and fam.name == "ltx-2.5":- raise ValueError(- "LTX-2.5's Gemma 4 text encoder runs in its own worker at bf16, so a text encoder "- "precision cannot be applied to it yet. Leave the text encoder precision unset."- )+ if fam.name == "ltx-2.5":+ from .video_ltx25 import ltx25_te_quant_scheme+ ltx25_te_quant_scheme(text_encoder_quant)_ensure_mp4_encoder_available()return fam@@ -4932,7 +4940,7 @@ class VideoBackend:ordinal = _target_ordinal(target),)if _auto_seed_yields_to_resident_bf16(- scheme, transformer_quant+ scheme, transformer_quant, fam) and _video_seed_stays_resident(fam,target = target,@@ -6994,7 +7002,7 @@ class VideoBackend:denoiser_seed_gb = Noneplan, bf16_plan, quant_replanned = _plan_for_te_scale(te_scale, log = False)if _auto_seed_yields_to_resident_bf16(- denoiser_seed_scheme, transformer_quant+ denoiser_seed_scheme, transformer_quant, fam) and _auto_keeps_resident_bf16(_plan_for_te_scale(te_scale, log = False)[1], target):# Auto keeps a resident bf16 DiT; seeding would change auto's precision wherever bf16 fits.logger.info(@@ -7049,7 +7057,11 @@ class VideoBackend:ltx25_dev = Falseif fam.name == "ltx-2.5":from .video_families import ltx25_is_dev- from .video_ltx25 import encoder_resident_for_plan, start_text_encoder+ from .video_ltx25 import (+ encoder_resident_for_plan,+ ltx25_te_quant_scheme,+ start_text_encoder,+ )ltx25_dev = _ltx25_dev_load(fam, kind, gguf_filename, repo_id, display_repo_id)# A stale load's pending worker goes only while this load is current; a superseded load that resumes here@@ -7069,6 +7081,7 @@ class VideoBackend:dtype = str(dtype).replace("torch.", ""),resident = encoder_resident_for_plan(plan),budget_mib = (getattr(plan, "estimates", None) or {}).get("safe_device_budget_mib"),+ te_quant = ltx25_te_quant_scheme(text_encoder_quant),)# Installed only while this load is current: a load superseded during the start must not overwrite the# newer load's worker (whose token-scoped cleanup could then never find it).@@ -7120,7 +7133,7 @@ class VideoBackend:# an fp8 budget under-states the resident requirement by ~11 GB for ltx-2, which is the direction that OOMs.# Placement is applied post-assembly (apply_memory_plan below), so re-plan at bf16 here, exactly as the# dense-quant fallback below restores bf16_plan.- if te_scale != 1.0 and not te_injected:+ if te_scale != 1.0 and not te_injected and fam.name != "ltx-2.5":logger.warning("video.te_prequant_budget: no pre-cast encoder was injected; re-planning ""memory at the dense bf16 text-encoder size"@@ -7698,14 +7711,18 @@ class VideoBackend:# dense text-encoder quant (opt-in): the companion encoder is often the largest resident, so quantise it in# place before placement, for every kind. Best-effort.- text_encoder_outcome = quantize_text_encoders(- pipe,- target,- mode = text_encoder_quant,- family = fam.name,- offload_active = plan.offload_policy != "none",- logger = logger,- )+ if ltx25_client is not None:+ from .video_ltx25 import ltx25_te_outcome+ text_encoder_outcome = ltx25_te_outcome(ltx25_client)+ else:+ text_encoder_outcome = quantize_text_encoders(+ pipe,+ target,+ mode = text_encoder_quant,+ family = fam.name,+ offload_active = plan.offload_policy != "none",+ logger = logger,+ )text_encoder_quant_engaged = text_encoder_outcome.modeif ((text_encoder_quant_engaged is None or text_encoder_outcome.partial)diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex 895d1b6a343..fb64b73a8bb 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -568,6 +568,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (variant_repos = ("unsloth/LTX-2.5-FP8",),default_on = True,),+ # ComfyUI's LTX-2.5 templates run the int8-convrot DiT; LPIPS 0.265 = compile floor (FP8 0.366).+ auto_prequant_when_resident = frozenset({"int8"}),untiled_vae = _LTX2_VAE_UNTILED,# 2.5 keeps the 2.3 conv VAE encoder (byte-identical weights), so the same latent space.compile_static = True,diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex 3b636742bba..002d019e454 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -84,6 +84,11 @@ TEXT_ENCODER_MIN_TRANSFORMERS = "5.10.1"# What the encoder worker holds on the card beyond its weights, on every plan tier: its CUDA context (~0.6 GB on# Hopper / Blackwell) plus allocator slack. The weights are the family table's text-encoder term.LTX25_WORKER_RESIDENT_GB = 1.0+# int8 encoder is opt-in: LPIPS 0.310 vs bf16, above the 0.256 compile floor.+LTX25_TE_QUANT_DEFAULT = "int8"+LTX25_TE_QUANT_ENV = "UNSLOTH_LTX25_TE_INT8"+# Measured int8 encoder weights (12,424 MiB vs 22,811 MiB bf16).+LTX25_TE_INT8_GB = 13.1# The pack's latent_upsampler/ (995,743,320 bytes), part of the family table's companions; only two-stage loads hold it.LTX25_LATENT_UPSAMPLER_GB = 1.0@@ -666,6 +671,44 @@ def encoder_resident_for_plan(plan: Any) -> bool:return policy == OFFLOAD_GROUP and not bool(getattr(plan, "stream_text_encoders", False))+def ltx25_te_quant_scheme(text_encoder_quant: Optional[str]) -> Optional[str]:+ """``"int8"`` or None (bf16); any other scheme raises ValueError (the worker has only the ConvRot INT8 build)."""+ from .diffusion_precision import normalize_te_quant, te_quant_is_auto++ if te_quant_is_auto(text_encoder_quant):+ if (os.environ.get(LTX25_TE_QUANT_ENV) or "").strip().lower() in ("1", "true", "yes", "on"):+ return LTX25_TE_QUANT_DEFAULT+ return None+ scheme = normalize_te_quant(text_encoder_quant)+ if scheme is None or scheme == LTX25_TE_QUANT_DEFAULT:+ return scheme+ raise ValueError(+ f"LTX-2.5's Gemma 4 text encoder runs int8 (ConvRot, as in ComfyUI's LTX-2.5 workflows) or bf16, not "+ f"'{scheme}'. Pick int8 for int8, or leave it unset / Off for bf16."+ )+++def ltx25_te_budget_scale(text_encoder_quant: Optional[str], bf16_gb: float) -> float:+ try:+ scheme = ltx25_te_quant_scheme(text_encoder_quant)+ except ValueError:+ return 1.0+ if scheme == LTX25_TE_QUANT_DEFAULT and bf16_gb > 0:+ return min(1.0, LTX25_TE_INT8_GB / float(bf16_gb))+ return 1.0+++def ltx25_te_outcome(client: Any) -> Any:+ from .diffusion_precision import TEQuantOutcome++ scheme = (getattr(client, "config", None) or {}).get("te_quant")+ if scheme:+ return TEQuantOutcome(+ scheme, "int8 ConvRot weight-only Gemma 4 encoder (ComfyUI's LTX-2.5 encoder)"+ )+ return TEQuantOutcome(None)++def sidecar_missing_message(venv_dir: str) -> str:return ("LTX-2.5's text encoder is a Gemma 4 model (Gemma4UnifiedForConditionalGeneration) that needs transformers "@@ -730,6 +773,7 @@ class Ltx25TextEncoderClient:cancel_event: Optional[threading.Event] = None,single_file: Optional[str] = None,budget_mib: Optional[int] = None,+ te_quant: Optional[str] = None,) -> None:self.config = {"repo": repo,@@ -745,6 +789,7 @@ class Ltx25TextEncoderClient:"venv_dir": venv_dir,# The load's memory budget: a non-resident encoder onloads only into what the server leaves of it."budget_mib": budget_mib,+ "te_quant": te_quant,}self.cancel_event = cancel_eventself._proc: Any = None@@ -1023,6 +1068,7 @@ def start_text_encoder(dtype: str,resident: bool,budget_mib: Optional[int] = None,+ te_quant: Optional[str] = None,) -> Ltx25TextEncoderClient:"""Check the sidecar (a named refusal before any weight is read), then spawn the encoder worker so the Gemma 4load overlaps the DiT load."""@@ -1038,6 +1084,7 @@ def start_text_encoder(venv_dir = venv_dir,single_file = LTX25_SINGLE_TEXT_ENCODER if is_ltx25_single_file_base(repo) else None,budget_mib = budget_mib,+ te_quant = te_quant,)client.start()return clientdiff --git a/studio/backend/core/inference/video_ltx25_te_worker.py b/studio/backend/core/inference/video_ltx25_te_worker.pyindex 5fe9e10ce04..392a0aa4b17 100644--- a/studio/backend/core/inference/video_ltx25_te_worker.py+++ b/studio/backend/core/inference/video_ltx25_te_worker.py@@ -154,11 +154,170 @@ def _load(config: dict) -> tuple[Any, Any]:)model.eval()model.requires_grad_(False)+ if config.get("te_quant") == TE_QUANT_INT8_CONVROT:+ quantize_int8_convrot_(model, config.get("device") or "cpu", bool(config.get("resident")))+ elif config.get("te_quant"):+ raise RuntimeError(f"unsupported text encoder precision '{config.get('te_quant')}'")if config.get("resident"):model.to(config["device"])return tokenizer, model+# Matches ComfyUI's gemma4 int8-convrot encoder, built from the bf16 weights on disk (no second 15 GB download).+TE_QUANT_INT8_CONVROT = "int8"+CONVROT_GROUP = 256+_PROJECTIONS = ("q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj")+++def _decoder_layers(model: Any) -> Any:+ inner = getattr(model, "model", model)+ language = getattr(inner, "language_model", inner)+ layers = getattr(language, "layers", None)+ if layers is None:+ raise RuntimeError("the Gemma 4 encoder has no language_model.layers to quantize")+ return layers+++_MADV_DONTNEED = 4+++def _file_mapping_holds(start: int, end: int) -> bool:+ """Whether ``[start, end)`` lies inside one file-backed mapping (Hub blobs have no suffix, so any file qualifies)."""+ try:+ with open("/proc/self/maps", "r", encoding = "utf-8", errors = "replace") as fh:+ for line in fh:+ fields = line.split(None, 5)+ if len(fields) < 6:+ continue+ lo, _, hi = fields[0].partition("-")+ if int(lo, 16) <= start and end <= int(hi, 16):+ return fields[4] != "0" and fields[5].strip().startswith("/")+ except (OSError, ValueError):+ return False+ return False+++def _release_mapped_pages(tensor: Any) -> bool:+ """MADV_DONTNEED file-mapped pages only (DONTNEED zeroes anonymous memory); whether it did."""+ if not sys.platform.startswith("linux"):+ return False+ try:+ import ctypes+ import mmap++ if tensor.device.type != "cpu" or not tensor.is_contiguous():+ return False+ start = int(tensor.data_ptr())+ end = start + tensor.numel() * tensor.element_size()+ if end <= start or not _file_mapping_holds(start, end):+ return False+ page = mmap.PAGESIZE+ lo = (start + page - 1) // page * page+ hi = end // page * page+ if hi <= lo:+ return True+ libc = ctypes.CDLL(None, use_errno = True)+ libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)+ libc.madvise.restype = ctypes.c_int+ return libc.madvise(lo, hi - lo, _MADV_DONTNEED) == 0+ except Exception: # noqa: BLE001 - best effort: the pages then stay mapped until the copy-out below+ return False+++def _empty_device_cache(device: Any) -> None:+ import torch+ empty = getattr(getattr(torch, device.type, None), "empty_cache", None)+ if callable(empty):+ empty()+++def _quantize_off_card(weight: Any, work: Any) -> tuple[Any, Any, Any]:+ """``(codes, scale, work)``; the card only speeds this up, so its OOM moves this and later projections to the host."""+ import torch++ from core.inference.diffusion_convrot import quantize_convrot_int8_weight++ if work.type != "cpu":+ oom = getattr(torch, "OutOfMemoryError", torch.cuda.OutOfMemoryError)+ try:+ return (*quantize_convrot_int8_weight(weight.detach().to(work), CONVROT_GROUP), work)+ except oom:+ pass+ # Outside the handler: the failed attempt's tensors are unreferenced by now.+ _empty_device_cache(work)+ work = torch.device("cpu")+ return (*quantize_convrot_int8_weight(weight.detach(), CONVROT_GROUP), work)+++def quantize_int8_convrot_(+ model: Any,+ device: Any = "cpu",+ resident: bool = False,+) -> int:+ """Swap every decoder-layer projection for the ConvRot INT8 linear in place; the number swapped. ``resident``+ quantizes layer by layer on the card. Otherwise mapped bf16 pages are released per projection; if any cannot be,+ host tensors are copied out of the map so it can close."""+ import torch+ from torch import nn++ from core.inference.diffusion_convrot import quantize_convrot_int8_weight+ from core.inference.video_minimax_h3_te import _int8_convrot_linear_class++ cls = _int8_convrot_linear_class()+ work = torch.device(str(device))+ if work.type not in ("cuda", "xpu") or not getattr(torch, work.type).is_available():+ work = torch.device("cpu")+ on_card = resident and work.type != "cpu"+ swapped = 0+ released_all = True+ quantized: set = set()+ for layer in _decoder_layers(model):+ if on_card:+ layer.to(work)+ for parent in (getattr(layer, "self_attn", None), getattr(layer, "mlp", None)):+ if parent is None:+ continue+ for name in _PROJECTIONS:+ linear = getattr(parent, name, None)+ if not isinstance(linear, nn.Linear):+ continue+ home = linear.weight.device+ if on_card:+ codes, scale = quantize_convrot_int8_weight(+ linear.weight.detach().to(work), CONVROT_GROUP+ )+ else:+ codes, scale, work = _quantize_off_card(linear.weight, work)+ if not on_card:+ released_all = _release_mapped_pages(linear.weight) and released_all+ new = cls(+ linear.in_features, linear.out_features, linear.bias is not None, CONVROT_GROUP+ )+ new.weight = codes.to(home)+ new.weight_scale = scale.to(home)+ if linear.bias is not None:+ new.bias = linear.bias.detach().to(device = home, dtype = torch.bfloat16)+ setattr(parent, name, new)+ quantized.add(id(new))+ swapped += 1+ del linear, codes, scale+ if swapped == 0:+ raise RuntimeError("no Gemma 4 decoder projection could be quantized")+ if not on_card and work.type != "cpu":+ # The plan never priced this transient.+ _empty_device_cache(work)+ if not on_card and not released_all:+ with torch.no_grad():+ for module in model.modules():+ if id(module) in quantized:+ continue+ for table in (module._parameters, module._buffers):+ for key, tensor in table.items():+ if tensor is not None and tensor.device.type == "cpu":+ tensor.data = tensor.data.clone()+ return swapped++# Room an onloaded encode needs beyond the weights: every hidden state stacked for the packed embeddings (49 x 1024 x# 3840 bf16 is ~0.4 GB per prompt, twice over while stacking) plus the forward's own activations.ENCODE_MARGIN_MIB = 4096@@ -173,6 +332,13 @@ def _weights_mib(model: Any) -> int:return total >> 20+def _weights_mib_or_none(model: Any) -> Optional[int]:+ try:+ return _weights_mib(model)+ except Exception: # noqa: BLE001 -- reporting only+ return None++def _encode_device(config: dict,model: Any,@@ -301,6 +467,8 @@ def run_text_encoder_worker(*, cmd_queue: Any, resp_queue: Any, config: dict) ->getattr(model.config, "text_config", model.config), "hidden_size", None),"resident": bool(config.get("resident")),+ "te_quant": config.get("te_quant") or None,+ "weights_mib": _weights_mib_or_none(model),**_memory_stats(config.get("device") or ""),})diff --git a/studio/backend/tests/test_diffusion_fast_load.py b/studio/backend/tests/test_diffusion_fast_load.pyindex 70005007eff..ee5668db393 100644--- a/studio/backend/tests/test_diffusion_fast_load.py+++ b/studio/backend/tests/test_diffusion_fast_load.py@@ -360,6 +360,48 @@ def test_fast_upload_passes_tensor_subclasses_through(monkeypatch):assert module.weight.device.type == "cuda"+@torch_cuda+def test_fast_upload_stages_torchao_int8_weights_byte_for_byte(monkeypatch):+ """torchao ``Int8Tensor`` weights ride the ring; the device copy equals the stock ``.to``'s."""+ torch = pytest.importorskip("torch")+ int8 = pytest.importorskip("torchao.quantization.quantize_.workflows.int8.int8_tensor")+ from torchao.quantization.granularity import PerRow++ monkeypatch.setattr(fl, "_UPLOAD_MIN_TOTAL_BYTES", 0)+ monkeypatch.setattr(fl, "_UPLOAD_CHUNK_BYTES", 1 << 16)++ def _build():+ torch.manual_seed(0)+ m = torch.nn.Sequential(torch.nn.Linear(256, 512), torch.nn.Linear(512, 256)).to(+ torch.bfloat16+ )+ for lin in m:+ lin.weight = torch.nn.Parameter(+ int8.Int8Tensor.from_hp(lin.weight.detach(), PerRow()), requires_grad = False+ )+ return m++ ref = _build().to("cuda")+ got = _build()+ assert len(fl._subclass_candidates([got], set())) == 2+ copies = []+ ring = fl._ring_copy+ monkeypatch.setattr(fl, "_ring_copy", lambda *a, **k: copies.extend(ring(*a, **k)) or copies)+ with fl.fast_upload([got], "cuda") as staged:+ got.to("cuda")+ assert staged >= 6 # 2 biases + 2 x (qdata, scale)+ ring_ptrs = {c.data_ptr() for c in copies}+ assert all(lin.weight.qdata.data_ptr() in ring_ptrs for lin in got)+ for a, b in zip(ref, got):+ wa, wb = a.weight, b.weight+ assert type(wb) is type(wa) and wb.device == wa.device and wb.dtype == wa.dtype+ assert torch.equal(wb.qdata, wa.qdata) and torch.equal(wb.scale, wa.scale)+ assert wb.qdata.device.type == "cuda" and wb.block_size == wa.block_size+ assert wb.act_quant_kwargs == wa.act_quant_kwargs+ x = torch.randn(32, 256, dtype = torch.bfloat16, device = "cuda")+ assert torch.equal(got(x), ref(x))++def test_rotational_lookup_never_raises(tmp_path):path = tmp_path / "w.safetensors"path.write_bytes(b"z")diff --git a/studio/backend/tests/test_video_ltx25_prequant.py b/studio/backend/tests/test_video_ltx25_prequant.pyindex 98f4980a39d..ad3607487e7 100644--- a/studio/backend/tests/test_video_ltx25_prequant.py+++ b/studio/backend/tests/test_video_ltx25_prequant.py@@ -304,3 +304,51 @@ def test_until_published_the_plan_keeps_the_dense_shards(fam, _planning):REPO,[("LTX-2.5-INT8.safetensors", plain["LTX-2.5-INT8.safetensors"])],)+++ "requested, scheme, expected",+ [+ ("auto", "int8", "int8"),+ (None, "int8", "int8"),+ ("auto", "fp8", None),+ ("fp8", "fp8", "fp8"),+ ("int8", "int8", "int8"),+ ],+)+def test_auto_seeds_int8_where_bf16_fits_like_comfyui(+ fam, monkeypatch, requested, scheme, expected+):+ import types++ import core.inference.video as vid++ backend = vid.VideoBackend()+ backend._state = types.SimpleNamespace(pipe = None)+ monkeypatch.setattr(vid, "_video_auto_denoiser_scheme", lambda fam, **kw: scheme)+ monkeypatch.setattr(vid, "_pipeline_device_mib", lambda pipe, ordinal = None: 0)+ monkeypatch.setattr(vid, "_video_seed_stays_resident", lambda fam, **kw: True)+ monkeypatch.setattr(+ vid,+ "resolve_diffusion_device_target",+ lambda **_k: types.SimpleNamespace(device = "cuda", dtype = None, ordinal = None),+ )+ assert (+ backend._video_planned_auto_denoiser_scheme(+ fam, base = PACK, kind = "pipeline", transformer_quant = requested, speed_mode = None+ )+ == expected+ )+++def test_only_ltx25_int8_skips_the_resident_bf16_rule(fam):+ import core.inference.video as vid++ assert fam.profile.auto_prequant_when_resident == frozenset({"int8"})+ assert not vid._auto_seed_yields_to_resident_bf16("int8", "auto", fam)+ assert vid._auto_seed_yields_to_resident_bf16("fp8", "auto", fam)+ for repo in ("Lightricks/LTX-2", "Wan-AI/Wan2.2-TI2V-5B-Diffusers"):+ other = detect_video_family(repo)+ assert other.name != fam.name+ assert vid._auto_seed_yields_to_resident_bf16("int8", "auto", other)+ assert vid._auto_seed_yields_to_resident_bf16("int8", "auto")diff --git a/studio/backend/tests/test_video_ltx25_te_int8.py b/studio/backend/tests/test_video_ltx25_te_int8.pynew file mode 100644index 00000000000..9c24e1f74d5--- /dev/null+++ b/studio/backend/tests/test_video_ltx25_te_int8.py@@ -0,0 +1,270 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LTX-2.5's opt-in int8 ConvRot Gemma 4 encoder."""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch+from torch import nn++from core.inference import video_ltx25 as v25+from core.inference import video_ltx25_te_worker as worker+from core.inference.diffusion_convrot import build_convrot_hadamard, quantize_convrot_int8_weight+++def _comfy_kitchen_reference(weight: torch.Tensor, group: int) -> tuple[torch.Tensor, torch.Tensor]:+ """comfy-kitchen's eager ``quantize_int8_convrot_weight`` in float32."""+ h4 = torch.tensor(+ [[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype = torch.float32+ )+ h = h4+ while h.shape[0] < group:+ h = torch.kron(h, h4)+ h = h / group**0.5+ out_f, in_f = weight.shape+ rotated = (weight.float().reshape(out_f, in_f // group, group) @ h.T).reshape(out_f, in_f)+ scale = (rotated.abs().amax(dim = -1, keepdim = True) / 127.0).clamp(min = 1e-30)+ return (rotated / scale).round().clamp(-128, 127).to(torch.int8), scale+++def test_hadamard_is_comfy_kitchens_and_quantization_matches_it():+ torch.manual_seed(0)+ h4 = torch.tensor(+ [[1, 1, 1, -1], [1, 1, -1, 1], [1, -1, 1, 1], [-1, 1, 1, 1]], dtype = torch.float32+ )+ ref = torch.kron(torch.kron(torch.kron(h4, h4), h4), h4) / 16.0+ assert torch.equal(build_convrot_hadamard(256), ref)+ weight = (torch.randn(64, 512) * 0.02).to(torch.bfloat16)+ codes, scale = quantize_convrot_int8_weight(weight, 256)+ ref_codes, ref_scale = _comfy_kitchen_reference(weight, 256)+ assert codes.dtype == torch.int8 and scale.dtype == torch.float32 and scale.shape == (64, 1)+ assert torch.equal(codes, ref_codes)+ torch.testing.assert_close(scale, ref_scale, rtol = 1e-6, atol = 0)+++class _Attn(nn.Module):+ def __init__(self, d):+ super().__init__()+ self.q_proj = nn.Linear(d, d, bias = False)+ self.k_proj = nn.Linear(d, d // 2, bias = False)+ self.o_proj = nn.Linear(d, d, bias = False)+ self.q_norm = nn.LayerNorm(d)+++class _Mlp(nn.Module):+ def __init__(self, d):+ super().__init__()+ self.gate_proj = nn.Linear(d, 2 * d, bias = False)+ self.up_proj = nn.Linear(d, 2 * d, bias = False)+ self.down_proj = nn.Linear(2 * d, d, bias = False)+++class _Layer(nn.Module):+ def __init__(self, d):+ super().__init__()+ self.self_attn = _Attn(d)+ self.mlp = _Mlp(d)+++class _TinyGemma(nn.Module):+ def __init__(+ self,+ d = 256,+ layers = 2,+ ):+ super().__init__()+ self.model = nn.Module()+ self.model.language_model = nn.Module()+ self.model.language_model.embed_tokens = nn.Embedding(16, d)+ self.model.language_model.layers = nn.ModuleList(_Layer(d) for _ in range(layers))+++def test_worker_swaps_every_decoder_projection_and_keeps_the_math():+ torch.manual_seed(0)+ model = _TinyGemma().to(torch.bfloat16)+ x = torch.randn(3, 256, dtype = torch.bfloat16)+ linear = model.model.language_model.layers[0].mlp.gate_proj+ want = linear(x).float()+ assert worker.quantize_int8_convrot_(model, "cpu") == 12 # 2 layers x (q, k, o, gate, up, down)+ swapped = model.model.language_model.layers[0].mlp.gate_proj+ assert type(swapped).__name__ == "Int8ConvRotLinear"+ assert swapped.weight.dtype == torch.int8 and swapped.weight_scale.dtype == torch.float32+ assert not any(isinstance(m, nn.Linear) for m in model.model.language_model.layers.modules())+ assert model.model.language_model.embed_tokens.weight.dtype == torch.bfloat16+ got = swapped(x).float()+ rel = (got - want).norm() / want.norm()+ assert rel < 0.02, rel+ model.to("meta")+ assert swapped.weight.device.type == "meta" and swapped.weight.dtype == torch.int8+++def test_release_mapped_pages_only_drops_file_backed_pages():+ anon = torch.full((1 << 20,), 3.0)+ assert worker._release_mapped_pages(anon) is False+ assert float(anon.sum()) == 3.0 * (1 << 20)+ assert worker._release_mapped_pages(torch.empty(0)) is False+++ not __import__("sys").platform.startswith("linux"), reason = "MADV_DONTNEED is Linux only"+)+def test_host_quantize_reads_projections_out_of_the_map_without_keeping_them(tmp_path):+ """A host build quantizes from the memory map and releases pages; codes match an in-memory quantize."""+ from safetensors.torch import load_file, save_file++ torch.manual_seed(0)+ dense = _TinyGemma().to(torch.bfloat16)+ path = tmp_path / "model.safetensors"+ save_file({k: v.contiguous() for k, v in dense.state_dict().items()}, str(path))+ mapped = _TinyGemma().to(torch.bfloat16)+ mapped.load_state_dict(load_file(str(path)), assign = True)+ gate = mapped.model.language_model.layers[0].mlp.gate_proj.weight+ if not worker._file_mapping_holds(gate.data_ptr(), gate.data_ptr() + gate.numel() * 2):+ pytest.skip("this safetensors build copies tensors out of the file")+ before = gate.detach().clone()+ assert worker._release_mapped_pages(gate) is True+ assert torch.equal(gate, before) # a later read re-faults the file's own bytes+ embed_ptr = mapped.model.language_model.embed_tokens.weight.data_ptr()++ assert worker.quantize_int8_convrot_(mapped, "cpu", resident = False) == 12+ worker.quantize_int8_convrot_(dense, "cpu", resident = False)+ assert mapped.model.language_model.embed_tokens.weight.data_ptr() == embed_ptr+ for a, b in zip(mapped.model.language_model.layers, dense.model.language_model.layers):+ for name in ("q_proj", "k_proj", "o_proj"):+ assert torch.equal(getattr(a.self_attn, name).weight, getattr(b.self_attn, name).weight)+ assert torch.equal(+ getattr(a.self_attn, name).weight_scale, getattr(b.self_attn, name).weight_scale+ )+++def test_off_card_quantize_falls_back_to_the_host_on_accelerator_oom(monkeypatch):+ import core.inference.diffusion_convrot as convrot++ real = convrot.quantize_convrot_int8_weight+ seen = []++ def _quantize(weight, group):+ seen.append(weight.device.type)+ if weight.device.type != "cpu":+ raise torch.OutOfMemoryError("CUDA out of memory")+ return real(weight, group)++ monkeypatch.setattr(convrot, "quantize_convrot_int8_weight", _quantize)+ weight = torch.randn(8, 512, dtype = torch.bfloat16)+ # "meta" stands in for a full card: the copy succeeds, the quantization runs out of memory.+ codes, scale, work = worker._quantize_off_card(weight, torch.device("meta"))+ assert seen == ["meta", "cpu"] and work.type == "cpu"+ ref_codes, ref_scale = real(weight, worker.CONVROT_GROUP)+ assert torch.equal(codes, ref_codes) and torch.equal(scale, ref_scale)+ seen.clear()+ worker._quantize_off_card(weight, work)+ assert seen == ["cpu"]+++def test_host_quantize_copies_out_of_the_map_when_pages_cannot_be_released(monkeypatch):+ monkeypatch.setattr(worker, "_release_mapped_pages", lambda tensor: False)+ model = _TinyGemma().to(torch.bfloat16)+ embed_ptr = model.model.language_model.embed_tokens.weight.data_ptr()+ worker.quantize_int8_convrot_(model, "cpu", resident = False)+ assert model.model.language_model.embed_tokens.weight.data_ptr() != embed_ptr+++def test_worker_load_quantizes_only_when_asked(monkeypatch):+ import sys++ class _Cls:+ @classmethod+ def from_pretrained(cls, *args, **kwargs):+ return _TinyGemma()++ # A stand-in module: the real Gemma 4 lazy import can re-register sys.modules['transformers'], dropping patches.+ monkeypatch.setitem(+ sys.modules,+ "transformers",+ SimpleNamespace(+ __version__ = "5.10.2",+ Gemma4UnifiedForConditionalGeneration = _Cls,+ AutoTokenizer = SimpleNamespace(from_pretrained = lambda *a, **k: object()),+ ),+ )+ base = {"repo": "Lightricks/LTX-2.5-Diffusers", "device": "cpu", "dtype": "bfloat16"}+ _tok, model = worker._load(dict(base, te_quant = "int8"))+ assert (+ type(model.model.language_model.layers[1].self_attn.q_proj).__name__ == "Int8ConvRotLinear"+ )+ _tok, model = worker._load(dict(base, te_quant = None))+ assert isinstance(model.model.language_model.layers[1].self_attn.q_proj, nn.Linear)+ with pytest.raises(RuntimeError, match = "unsupported text encoder precision"):+ worker._load(dict(base, te_quant = "fp8"))+++def test_te_quant_tristate(monkeypatch):+ monkeypatch.delenv(v25.LTX25_TE_QUANT_ENV, raising = False)+ assert v25.ltx25_te_quant_scheme(None) is None+ assert v25.ltx25_te_quant_scheme("auto") is None+ assert v25.ltx25_te_quant_scheme("") is None+ assert v25.ltx25_te_quant_scheme("int8") == "int8"+ assert v25.ltx25_te_quant_scheme("none") is None+ assert v25.ltx25_te_quant_scheme("off") is None+ for scheme in ("fp8", "fp8_dynamic"):+ with pytest.raises(ValueError, match = "int8 .* or bf16"):+ v25.ltx25_te_quant_scheme(scheme)+ monkeypatch.setenv(v25.LTX25_TE_QUANT_ENV, "1")+ assert v25.ltx25_te_quant_scheme(None) == "int8"+ assert v25.ltx25_te_quant_scheme("off") is None+++def test_budget_prices_the_int8_encoder(monkeypatch):+ from core.inference.diffusion_te_prequant import te_prequant_budget_scale+ from core.inference.video_families import detect_video_family++ monkeypatch.delenv(v25.LTX25_TE_QUANT_ENV, raising = False)+ fam = detect_video_family("Lightricks/LTX-2.5-Diffusers")+ bf16_gb = fam.bf16_components_gb[1]+ kw = {"target": None, "base": "Lightricks/LTX-2.5-Diffusers"}+ scale = te_prequant_budget_scale(fam, te_quant_mode = "int8", **kw)+ assert scale == pytest.approx(v25.LTX25_TE_INT8_GB / bf16_gb)+ assert scale < 0.6+ assert te_prequant_budget_scale(fam, te_quant_mode = "none", **kw) == 1.0+ assert te_prequant_budget_scale(fam, te_quant_mode = None, **kw) == 1.0+ monkeypatch.setenv(v25.LTX25_TE_QUANT_ENV, "1")+ assert te_prequant_budget_scale(fam, te_quant_mode = None, **kw) == scale+++def test_client_carries_the_scheme_and_status_reports_it():+ kwargs = dict(+ repo = "Lightricks/LTX-2.5-Diffusers",+ hf_token = None,+ cache_dir = None,+ local_files_only = True,+ device = "cpu",+ dtype = "bfloat16",+ resident = False,+ venv_dir = "/nonexistent",+ )+ client = v25.Ltx25TextEncoderClient(te_quant = "int8", **kwargs)+ assert client.config["te_quant"] == "int8"+ outcome = v25.ltx25_te_outcome(client)+ assert outcome.mode == "int8" and "ConvRot" in outcome.reason+ assert v25.ltx25_te_outcome(v25.Ltx25TextEncoderClient(**kwargs)).mode is None+++def test_validate_load_request_accepts_int8_and_off_and_refuses_other_schemes():+ from core.inference.video import get_video_backend++ backend = get_video_backend()+ for mode in (None, "auto", "int8", "none", "off"):+ assert (+ backend.validate_load_request(+ "Lightricks/LTX-2.5-Diffusers", model_kind = "pipeline", text_encoder_quant = mode+ ).name+ == "ltx-2.5"+ )+ with pytest.raises(ValueError, match = "int8 .* or bf16"):+ backend.validate_load_request(+ "Lightricks/LTX-2.5-Diffusers", model_kind = "pipeline", text_encoder_quant = "fp8"+ )