Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio video: keep an explicit fast LTX-2.3 single-file load resident when it fits (#12345)
Daniel Hanba7ff6e13896
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex 14389f9c768..2c1cbfa45d3 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -1669,7 +1669,8 @@ def _pipeline_quant_uncompilable_reason(def _plan_proves_resident(plan: Any) -> bool:"""Resident AND fits; the planner also stays resident when it cannot read the card."""estimates = getattr(plan, "estimates", None) or {}- budget = estimates.get("safe_device_budget_mib")+ # An explicit fast proves its fit against the budget it was placed with.+ budget = estimates.get("resident_budget_mib", estimates.get("safe_device_budget_mib"))required = estimates.get("resident_required_mib")# Unified memory plans 'none' whatever the size (offload frees nothing there), so compare the two as well.return (diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.pyindex df347409d49..ef573ad7983 100644--- a/studio/backend/core/inference/diffusion_memory.py+++ b/studio/backend/core/inference/diffusion_memory.py@@ -806,6 +806,27 @@ def file_size_mib(path: Any) -> Optional[int]:return None+def safetensors_prefix_mib(path: Any, prefix: str) -> Optional[int]:+ """MiB of the ``prefix*`` tensors, from the safetensors header alone; None if unreadable or unmatched."""+ import json+ import struct++ try:+ with open(path, "rb") as fh:+ (header_len,) = struct.unpack("<Q", fh.read(8))+ if not 0 < header_len <= 256 * 1024 * 1024:+ return None+ header = json.loads(fh.read(header_len))+ total = 0+ for name, meta in header.items():+ if name != "__metadata__" and name.startswith(prefix):+ start, end = meta["data_offsets"]+ total += int(end) - int(start)+ except Exception: # noqa: BLE001 - not a readable safetensors header+ return None+ return -(-total // (1024 * 1024)) if total > 0 else None++def estimate_gguf_resident_mib(storage_mib: Optional[int]) -> Optional[int]:"""Approximate the RESIDENT device size of a GGUF transformer under ``GGUFQuantizationConfig``.@@ -886,6 +907,16 @@ def _safe_device_budget_mib(memory: DeviceMemory) -> Optional[int]:return max(0, int(memory.free_mib) - _reserve_mib(memory.memory_kind, base))+def _fast_device_budget_mib(memory: DeviceMemory) -> Optional[int]:+ """Free memory minus half the standard reserve (min 2 GiB): an explicit ``fast`` offloads only when resident+ would not fit, not to keep ``auto``'s headroom for other tenants."""+ if memory.free_mib is None:+ return None+ base = memory.total_mib or memory.free_mib+ reserve = max(2048, _reserve_mib(memory.memory_kind, base) // 2)+ return max(0, int(memory.free_mib) - reserve)++def plan_keeps_transformer_resident(plan: Any) -> bool:"""Whether ``plan`` never moves the denoiser after placement: torchao survives placement, not per-forward hooks."""policy = getattr(plan, "offload_policy", OFFLOAD_NONE)@@ -1165,9 +1196,14 @@ def plan_diffusion_memory(reasons.append(f"{device_memory.backend}: CPU offload unavailable; staying resident")elif mode == MEMORY_MODE_FAST:policy = OFFLOAD_NONE- if budget is not None and required is not None and required > budget:+ fast_budget = _fast_device_budget_mib(device_memory)+ estimates["resident_budget_mib"] = fast_budget+ if fast_budget is not None and required is not None and required > fast_budget:policy, stream_text_encoders, stream_transformer = _offload_tier()- reasons.append("fast requested but weights do not fit resident; offloading")+ reasons.append(+ f"fast requested but weights do not fit resident ({required} MiB needed, "+ f"{fast_budget} MiB free after the reserve); offloading"+ )if not stream_transformer:reasons.append(_RESIDENT_TRANSFORMER_REASON)elif stream_text_encoders:diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex 99c97b0682e..a53ab0158e6 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -87,6 +87,8 @@ from .diffusion_device import (resolve_selected_cuda_ordinal,)from .diffusion_memory import (+ MEMORY_MODE_FAST,+ OFFLOAD_NONE,apply_memory_plan,estimate_gguf_resident_mib,estimate_safetensors_dense_mib,@@ -98,6 +100,7 @@ from .diffusion_memory import (reclaim_host_memory,reclaim_offload_host_memory,release_pinned_host_memory,+ safetensors_prefix_mib,settled_snapshot_device_memory,)from .diffusion_torchao_patches import install_torchao_int_mm_patch@@ -330,6 +333,17 @@ def _ltx23_prequant_pick(return ltx23_prequant_eligible(checkpoint_filename, checkpoint_repo)+def _memory_mode_resolved(memory_mode: Any, plan: Any, offload_policy: str) -> tuple:+ """A ``fast`` that had to offload is a fallback valued at the engaged policy ("FAST -> GROUP", not "FAST -> FAST")."""+ if plan.requested_mode == MEMORY_MODE_FAST and offload_policy != OFFLOAD_NONE:+ return (memory_mode, offload_policy, "; ".join(plan.reasons), RESOLVED_FELL_BACK)+ return (+ memory_mode,+ plan.requested_mode,+ f"planned '{plan.offload_policy}' offload from the family size table",+ )++def _ltx23_prequant_serves(fam: Any,model_kind: str,@@ -4772,6 +4786,12 @@ class VideoBackend:transformer_mib = estimate_gguf_resident_mib(size_mib)else:transformer_mib = estimate_safetensors_dense_mib(size_mib)+ if fam.name == "ltx-2" and transformer_mib is not None:+ # The file also bundles VAE / audio VAE / vocoder, already priced as companions. The connectors under+ # this prefix stay in: the table undercounts them (~2.9 of 6 GB), and dropping them plans below the peak.+ dit_mib = safetensors_prefix_mib(str(checkpoint_path), "model.diffusion_model.")+ if dit_mib is not None:+ transformer_mib = min(transformer_mib, dit_mib)if transformer_mib is not None:transformer_mib = int(transformer_mib * dtype_scale)# Price the plan at the hosted fp8 DiT: the bf16 file size would plan an offload and skip the seed.@@ -5564,11 +5584,7 @@ class VideoBackend:resolved = build_resolved_record({- "memory_mode": (- memory_mode,- plan.requested_mode,- f"planned '{plan.offload_policy}' offload from the family size table",- ),+ "memory_mode": _memory_mode_resolved(memory_mode, plan, offload_policy),"speed_mode": (speed_mode,effective_speed,diff --git a/studio/backend/tests/test_video_ltx23_memory_plan.py b/studio/backend/tests/test_video_ltx23_memory_plan.pynew file mode 100644index 00000000000..ccec99454fc--- /dev/null+++ b/studio/backend/tests/test_video_ltx23_memory_plan.py@@ -0,0 +1,251 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""An explicit ``fast`` LTX-2.3 single-file load stays resident when it fits, whatever the speed mode."""++import json+import struct+import sys+import types++import pytest++from core.inference.diffusion_memory import (+ MEMORY_MODE_AUTO,+ MEMORY_MODE_FAST,+ OFFLOAD_GROUP,+ OFFLOAD_MODEL,+ OFFLOAD_NONE,+ DeviceMemory,+ plan_diffusion_memory,+)++MIB = 1024 * 1024+B200_TOTAL_MIB = 182_633+# What torch reported with another tenant holding ~91 GiB of the card.+TENANT_FREE_MIB = 89_106+# The live LTX-2.3 single-file inputs (ltx-2 table companions 24.4 + 5.5 GB, 121-frame 768x512 headroom).+LTX23_DIT_MIB = 40_073+LTX23_FILE_MIB = 44_011+COMPANION_MIB = 28_514+RUNTIME_MIB = 5_729+++def _target():+ return types.SimpleNamespace(device = "cuda", supports_model_cpu_offload = True)+++def _b200(free_mib):+ return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, B200_TOTAL_MIB)+++def _plan(+ free_mib,+ transformer_mib,+ mode = MEMORY_MODE_FAST,+):+ return plan_diffusion_memory(+ target = _target(),+ device_memory = _b200(free_mib),+ model_dense_mib = transformer_mib + COMPANION_MIB,+ runtime_headroom_mib = RUNTIME_MIB,+ companion_dense_mib = COMPANION_MIB,+ requested_mode = mode,+ )+++def _write_header(path, tensors):+ header, offset = {"__metadata__": {"format": "pt"}}, 0+ for name, mib in tensors.items():+ header[name] = {+ "dtype": "BF16",+ "shape": [mib * MIB // 2],+ "data_offsets": [offset, offset + mib * MIB],+ }+ offset += mib * MIB+ raw = json.dumps(header).encode()+ path.write_bytes(struct.pack("<Q", len(raw)) + raw)+++def test_fast_stays_resident_when_the_dit_fits_free_memory():+ plan = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)+ assert plan.offload_policy == OFFLOAD_NONE+ assert plan.reasons == ("fast requested; weights resident on device",)+++def test_fast_still_offloads_when_resident_would_not_fit():+ plan = _plan(60 * 1024, LTX23_DIT_MIB)+ assert plan.offload_policy == OFFLOAD_GROUP+ assert "do not fit resident" in plan.reasons[0]+ assert _plan(20 * 1024, LTX23_DIT_MIB).offload_policy == OFFLOAD_MODEL+++def test_fast_reserve_never_drops_below_two_gib():+ """Half the reserve on a small card would be 1 GiB; the floor keeps the old 2 GiB."""+ small = DeviceMemory("cuda", "cuda", "discrete_vram", 12_000, 12_000)+ fits = plan_diffusion_memory(+ target = _target(),+ device_memory = small,+ model_dense_mib = 12_000 - 2048 - 1000 - 1000,+ runtime_headroom_mib = 1000,+ base_overhead_mib = 1000,+ requested_mode = MEMORY_MODE_FAST,+ )+ over = plan_diffusion_memory(+ target = _target(),+ device_memory = small,+ model_dense_mib = 12_000 - 2048 - 1000 - 1000 + 1,+ runtime_headroom_mib = 1000,+ base_overhead_mib = 1000,+ requested_mode = MEMORY_MODE_FAST,+ )+ assert fits.offload_policy == OFFLOAD_NONE+ assert over.offload_policy != OFFLOAD_NONE+++def test_auto_keeps_its_own_headroom():+ assert _plan(TENANT_FREE_MIB, LTX23_DIT_MIB, MEMORY_MODE_AUTO).offload_policy != OFFLOAD_NONE+++def test_prefix_mib_reads_only_the_dit_tensors(tmp_path):+ from core.inference.diffusion_memory import safetensors_prefix_mib++ path = tmp_path / "ltx.safetensors"+ _write_header(+ path,+ {+ "model.diffusion_model.a.weight": 30,+ "model.diffusion_model.b.weight": 10,+ "vae.decoder.weight": 5,+ "text_embedding_projection.weight": 2,+ },+ )+ assert safetensors_prefix_mib(path, "model.diffusion_model.") == 40+ assert safetensors_prefix_mib(path, "absent.") is None+ junk = tmp_path / "junk.safetensors"+ junk.write_bytes(b"\xff" * 64)+ assert safetensors_prefix_mib(junk, "model.diffusion_model.") is None+ assert safetensors_prefix_mib(tmp_path / "missing.safetensors", "x") is None+++class _Planned(Exception):+ pass+++def _plan_inputs_for_single_file_load(monkeypatch, tmp_path, speed_mode, free_mib):+ """Drive ``load_pipeline`` for the LTX-2.3 single file up to its first memory plan; return that plan's inputs."""+ import core.inference.diffusion_te_prequant as te_prequant+ import core.inference.video as vid++ ckpt = tmp_path / "ltx-2.3-22b-distilled.safetensors"+ _write_header(+ ckpt,+ {+ "model.diffusion_model.blocks.weight": LTX23_DIT_MIB,+ "vae.decoder.weight": 1385,+ "audio_vae.weight": 102,+ "vocoder.weight": 246,+ "text_embedding_projection.weight": 2205,+ },+ )+ target = types.SimpleNamespace(+ device = "cuda",+ dtype = None,+ torch_device = "cuda",+ ordinal = None,+ supports_model_cpu_offload = True,+ )+ backend = vid.VideoBackend()+ monkeypatch.setattr(backend, "_device_target", lambda *a, **k: target)+ monkeypatch.setattr(backend, "_resolve_checkpoint_path", lambda *a, **k: ckpt)+ monkeypatch.setattr(vid, "file_size_mib", lambda _p: LTX23_FILE_MIB)+ monkeypatch.setattr(vid, "settled_snapshot_device_memory", lambda _t: _b200(free_mib))+ monkeypatch.setattr(te_prequant, "te_prequant_budget_scale", lambda *a, **k: 1.0)+ # The plan is drawn before any diffusers class is touched; CPU CI installs no diffusers.+ diffusers = types.ModuleType("diffusers")+ diffusers.__version__ = "0.40.0"+ diffusers.LTX2Pipeline = type("LTX2Pipeline", (), {})+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)+ seen: list = []+ real = plan_diffusion_memory++ def capture(**kw):+ plan = real(**kw)+ seen.append(+ (+ kw["model_dense_mib"],+ kw["companion_dense_mib"],+ kw["requested_mode"],+ plan.offload_policy,+ )+ )+ raise _Planned()++ monkeypatch.setattr(vid, "plan_diffusion_memory", capture)+ with pytest.raises(_Planned):+ backend.load_pipeline(+ "Lightricks/LTX-2.3",+ local_files_only = True,+ gguf_filename = ckpt.name,+ model_kind = "single_file",+ memory_mode = MEMORY_MODE_FAST,+ speed_mode = speed_mode,+ )+ return seen[0]++[email protected]("speed_mode", [None, "eager", "max"])+def test_ltx23_single_file_prices_the_dit_once_and_stays_resident(+ monkeypatch, tmp_path, speed_mode+):+ dense, companions, mode, policy = _plan_inputs_for_single_file_load(+ monkeypatch, tmp_path, speed_mode, TENANT_FREE_MIB+ )+ assert companions == COMPANION_MIB+ assert dense == LTX23_DIT_MIB + COMPANION_MIB+ assert mode == MEMORY_MODE_FAST+ assert policy == OFFLOAD_NONE+++def test_ltx23_plan_does_not_depend_on_the_speed_mode(monkeypatch, tmp_path):+ for free_mib in (TENANT_FREE_MIB, 60 * 1024, B200_TOTAL_MIB):+ plans = {+ speed: _plan_inputs_for_single_file_load(monkeypatch, tmp_path, speed, free_mib)+ for speed in (None, "eager")+ }+ assert plans[None] == plans["eager"], free_mib+++def test_fast_that_offloaded_is_recorded_as_a_fallback_naming_the_policy():+ from core.inference.diffusion_auto_policy import build_resolved_record+ from core.inference.video import _memory_mode_resolved++ fell = _plan(60 * 1024, LTX23_DIT_MIB)+ entry = build_resolved_record(+ {"memory_mode": _memory_mode_resolved("fast", fell, fell.offload_policy)}+ )["memory_mode"]+ assert (entry["status"], entry["requested"], entry["value"]) == (+ "fell_back",+ "fast",+ OFFLOAD_GROUP,+ )+ assert "do not fit resident" in entry["reason"]++ kept = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)+ entry = build_resolved_record(+ {"memory_mode": _memory_mode_resolved("fast", kept, kept.offload_policy)}+ )["memory_mode"]+ assert (entry["status"], entry["value"]) == ("applied", "fast")+++def test_resident_proof_uses_the_budget_fast_was_placed_with():+ """Auto precision keeps bf16 only on a proven fit; a fast plan placed resident under its own budget is one."""+ from core.inference.diffusion import _plan_proves_resident++ fast = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)+ assert fast.offload_policy == OFFLOAD_NONE+ assert fast.estimates["resident_required_mib"] > fast.estimates["safe_device_budget_mib"]+ assert _plan_proves_resident(fast)+ auto = _plan(B200_TOTAL_MIB, LTX23_DIT_MIB, MEMORY_MODE_AUTO)+ assert "resident_budget_mib" not in auto.estimates+ assert _plan_proves_resident(auto)