Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio video: keep an explicit fast LTX-2.3 single-file load resident when it fits (#12345)

Daniel Hanba7ff6e13896
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 14389f9c768..2c1cbfa45d3 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -1669,7 +1669,8 @@ def _pipeline_quant_uncompilable_reason(
def _plan_proves_resident(plan: Any) -> bool:
"""Resident AND fits; the planner also stays resident when it cannot read the card."""
estimates = getattr(plan, "estimates", None) or {}
- budget = estimates.get("safe_device_budget_mib")
+ # An explicit fast proves its fit against the budget it was placed with.
+ budget = estimates.get("resident_budget_mib", estimates.get("safe_device_budget_mib"))
required = estimates.get("resident_required_mib")
# Unified memory plans 'none' whatever the size (offload frees nothing there), so compare the two as well.
return (
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index df347409d49..ef573ad7983 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -806,6 +806,27 @@ def file_size_mib(path: Any) -> Optional[int]:
return None
+def safetensors_prefix_mib(path: Any, prefix: str) -> Optional[int]:
+ """MiB of the ``prefix*`` tensors, from the safetensors header alone; None if unreadable or unmatched."""
+ import json
+ import struct
+
+ try:
+ with open(path, "rb") as fh:
+ (header_len,) = struct.unpack("<Q", fh.read(8))
+ if not 0 < header_len <= 256 * 1024 * 1024:
+ return None
+ header = json.loads(fh.read(header_len))
+ total = 0
+ for name, meta in header.items():
+ if name != "__metadata__" and name.startswith(prefix):
+ start, end = meta["data_offsets"]
+ total += int(end) - int(start)
+ except Exception: # noqa: BLE001 - not a readable safetensors header
+ return None
+ return -(-total // (1024 * 1024)) if total > 0 else None
+
+
def estimate_gguf_resident_mib(storage_mib: Optional[int]) -> Optional[int]:
"""Approximate the RESIDENT device size of a GGUF transformer under ``GGUFQuantizationConfig``.
@@ -886,6 +907,16 @@ def _safe_device_budget_mib(memory: DeviceMemory) -> Optional[int]:
return max(0, int(memory.free_mib) - _reserve_mib(memory.memory_kind, base))
+def _fast_device_budget_mib(memory: DeviceMemory) -> Optional[int]:
+ """Free memory minus half the standard reserve (min 2 GiB): an explicit ``fast`` offloads only when resident
+ would not fit, not to keep ``auto``'s headroom for other tenants."""
+ if memory.free_mib is None:
+ return None
+ base = memory.total_mib or memory.free_mib
+ reserve = max(2048, _reserve_mib(memory.memory_kind, base) // 2)
+ return max(0, int(memory.free_mib) - reserve)
+
+
def plan_keeps_transformer_resident(plan: Any) -> bool:
"""Whether ``plan`` never moves the denoiser after placement: torchao survives placement, not per-forward hooks."""
policy = getattr(plan, "offload_policy", OFFLOAD_NONE)
@@ -1165,9 +1196,14 @@ def plan_diffusion_memory(
reasons.append(f"{device_memory.backend}: CPU offload unavailable; staying resident")
elif mode == MEMORY_MODE_FAST:
policy = OFFLOAD_NONE
- if budget is not None and required is not None and required > budget:
+ fast_budget = _fast_device_budget_mib(device_memory)
+ estimates["resident_budget_mib"] = fast_budget
+ if fast_budget is not None and required is not None and required > fast_budget:
policy, stream_text_encoders, stream_transformer = _offload_tier()
- reasons.append("fast requested but weights do not fit resident; offloading")
+ reasons.append(
+ f"fast requested but weights do not fit resident ({required} MiB needed, "
+ f"{fast_budget} MiB free after the reserve); offloading"
+ )
if not stream_transformer:
reasons.append(_RESIDENT_TRANSFORMER_REASON)
elif stream_text_encoders:
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index 99c97b0682e..a53ab0158e6 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -87,6 +87,8 @@ from .diffusion_device import (
resolve_selected_cuda_ordinal,
)
from .diffusion_memory import (
+ MEMORY_MODE_FAST,
+ OFFLOAD_NONE,
apply_memory_plan,
estimate_gguf_resident_mib,
estimate_safetensors_dense_mib,
@@ -98,6 +100,7 @@ from .diffusion_memory import (
reclaim_host_memory,
reclaim_offload_host_memory,
release_pinned_host_memory,
+ safetensors_prefix_mib,
settled_snapshot_device_memory,
)
from .diffusion_torchao_patches import install_torchao_int_mm_patch
@@ -330,6 +333,17 @@ def _ltx23_prequant_pick(
return ltx23_prequant_eligible(checkpoint_filename, checkpoint_repo)
+def _memory_mode_resolved(memory_mode: Any, plan: Any, offload_policy: str) -> tuple:
+ """A ``fast`` that had to offload is a fallback valued at the engaged policy ("FAST -> GROUP", not "FAST -> FAST")."""
+ if plan.requested_mode == MEMORY_MODE_FAST and offload_policy != OFFLOAD_NONE:
+ return (memory_mode, offload_policy, "; ".join(plan.reasons), RESOLVED_FELL_BACK)
+ return (
+ memory_mode,
+ plan.requested_mode,
+ f"planned '{plan.offload_policy}' offload from the family size table",
+ )
+
+
def _ltx23_prequant_serves(
fam: Any,
model_kind: str,
@@ -4772,6 +4786,12 @@ class VideoBackend:
transformer_mib = estimate_gguf_resident_mib(size_mib)
else:
transformer_mib = estimate_safetensors_dense_mib(size_mib)
+ if fam.name == "ltx-2" and transformer_mib is not None:
+ # The file also bundles VAE / audio VAE / vocoder, already priced as companions. The connectors under
+ # this prefix stay in: the table undercounts them (~2.9 of 6 GB), and dropping them plans below the peak.
+ dit_mib = safetensors_prefix_mib(str(checkpoint_path), "model.diffusion_model.")
+ if dit_mib is not None:
+ transformer_mib = min(transformer_mib, dit_mib)
if transformer_mib is not None:
transformer_mib = int(transformer_mib * dtype_scale)
# Price the plan at the hosted fp8 DiT: the bf16 file size would plan an offload and skip the seed.
@@ -5564,11 +5584,7 @@ class VideoBackend:
resolved = build_resolved_record(
{
- "memory_mode": (
- memory_mode,
- plan.requested_mode,
- f"planned '{plan.offload_policy}' offload from the family size table",
- ),
+ "memory_mode": _memory_mode_resolved(memory_mode, plan, offload_policy),
"speed_mode": (
speed_mode,
effective_speed,
diff --git a/studio/backend/tests/test_video_ltx23_memory_plan.py b/studio/backend/tests/test_video_ltx23_memory_plan.py
new file mode 100644
index 00000000000..ccec99454fc
--- /dev/null
+++ b/studio/backend/tests/test_video_ltx23_memory_plan.py
@@ -0,0 +1,251 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An explicit ``fast`` LTX-2.3 single-file load stays resident when it fits, whatever the speed mode."""
+
+import json
+import struct
+import sys
+import types
+
+import pytest
+
+from core.inference.diffusion_memory import (
+ MEMORY_MODE_AUTO,
+ MEMORY_MODE_FAST,
+ OFFLOAD_GROUP,
+ OFFLOAD_MODEL,
+ OFFLOAD_NONE,
+ DeviceMemory,
+ plan_diffusion_memory,
+)
+
+MIB = 1024 * 1024
+B200_TOTAL_MIB = 182_633
+# What torch reported with another tenant holding ~91 GiB of the card.
+TENANT_FREE_MIB = 89_106
+# The live LTX-2.3 single-file inputs (ltx-2 table companions 24.4 + 5.5 GB, 121-frame 768x512 headroom).
+LTX23_DIT_MIB = 40_073
+LTX23_FILE_MIB = 44_011
+COMPANION_MIB = 28_514
+RUNTIME_MIB = 5_729
+
+
+def _target():
+ return types.SimpleNamespace(device = "cuda", supports_model_cpu_offload = True)
+
+
+def _b200(free_mib):
+ return DeviceMemory("cuda", "cuda", "discrete_vram", free_mib, B200_TOTAL_MIB)
+
+
+def _plan(
+ free_mib,
+ transformer_mib,
+ mode = MEMORY_MODE_FAST,
+):
+ return plan_diffusion_memory(
+ target = _target(),
+ device_memory = _b200(free_mib),
+ model_dense_mib = transformer_mib + COMPANION_MIB,
+ runtime_headroom_mib = RUNTIME_MIB,
+ companion_dense_mib = COMPANION_MIB,
+ requested_mode = mode,
+ )
+
+
+def _write_header(path, tensors):
+ header, offset = {"__metadata__": {"format": "pt"}}, 0
+ for name, mib in tensors.items():
+ header[name] = {
+ "dtype": "BF16",
+ "shape": [mib * MIB // 2],
+ "data_offsets": [offset, offset + mib * MIB],
+ }
+ offset += mib * MIB
+ raw = json.dumps(header).encode()
+ path.write_bytes(struct.pack("<Q", len(raw)) + raw)
+
+
+def test_fast_stays_resident_when_the_dit_fits_free_memory():
+ plan = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)
+ assert plan.offload_policy == OFFLOAD_NONE
+ assert plan.reasons == ("fast requested; weights resident on device",)
+
+
+def test_fast_still_offloads_when_resident_would_not_fit():
+ plan = _plan(60 * 1024, LTX23_DIT_MIB)
+ assert plan.offload_policy == OFFLOAD_GROUP
+ assert "do not fit resident" in plan.reasons[0]
+ assert _plan(20 * 1024, LTX23_DIT_MIB).offload_policy == OFFLOAD_MODEL
+
+
+def test_fast_reserve_never_drops_below_two_gib():
+ """Half the reserve on a small card would be 1 GiB; the floor keeps the old 2 GiB."""
+ small = DeviceMemory("cuda", "cuda", "discrete_vram", 12_000, 12_000)
+ fits = plan_diffusion_memory(
+ target = _target(),
+ device_memory = small,
+ model_dense_mib = 12_000 - 2048 - 1000 - 1000,
+ runtime_headroom_mib = 1000,
+ base_overhead_mib = 1000,
+ requested_mode = MEMORY_MODE_FAST,
+ )
+ over = plan_diffusion_memory(
+ target = _target(),
+ device_memory = small,
+ model_dense_mib = 12_000 - 2048 - 1000 - 1000 + 1,
+ runtime_headroom_mib = 1000,
+ base_overhead_mib = 1000,
+ requested_mode = MEMORY_MODE_FAST,
+ )
+ assert fits.offload_policy == OFFLOAD_NONE
+ assert over.offload_policy != OFFLOAD_NONE
+
+
+def test_auto_keeps_its_own_headroom():
+ assert _plan(TENANT_FREE_MIB, LTX23_DIT_MIB, MEMORY_MODE_AUTO).offload_policy != OFFLOAD_NONE
+
+
+def test_prefix_mib_reads_only_the_dit_tensors(tmp_path):
+ from core.inference.diffusion_memory import safetensors_prefix_mib
+
+ path = tmp_path / "ltx.safetensors"
+ _write_header(
+ path,
+ {
+ "model.diffusion_model.a.weight": 30,
+ "model.diffusion_model.b.weight": 10,
+ "vae.decoder.weight": 5,
+ "text_embedding_projection.weight": 2,
+ },
+ )
+ assert safetensors_prefix_mib(path, "model.diffusion_model.") == 40
+ assert safetensors_prefix_mib(path, "absent.") is None
+ junk = tmp_path / "junk.safetensors"
+ junk.write_bytes(b"\xff" * 64)
+ assert safetensors_prefix_mib(junk, "model.diffusion_model.") is None
+ assert safetensors_prefix_mib(tmp_path / "missing.safetensors", "x") is None
+
+
+class _Planned(Exception):
+ pass
+
+
+def _plan_inputs_for_single_file_load(monkeypatch, tmp_path, speed_mode, free_mib):
+ """Drive ``load_pipeline`` for the LTX-2.3 single file up to its first memory plan; return that plan's inputs."""
+ import core.inference.diffusion_te_prequant as te_prequant
+ import core.inference.video as vid
+
+ ckpt = tmp_path / "ltx-2.3-22b-distilled.safetensors"
+ _write_header(
+ ckpt,
+ {
+ "model.diffusion_model.blocks.weight": LTX23_DIT_MIB,
+ "vae.decoder.weight": 1385,
+ "audio_vae.weight": 102,
+ "vocoder.weight": 246,
+ "text_embedding_projection.weight": 2205,
+ },
+ )
+ target = types.SimpleNamespace(
+ device = "cuda",
+ dtype = None,
+ torch_device = "cuda",
+ ordinal = None,
+ supports_model_cpu_offload = True,
+ )
+ backend = vid.VideoBackend()
+ monkeypatch.setattr(backend, "_device_target", lambda *a, **k: target)
+ monkeypatch.setattr(backend, "_resolve_checkpoint_path", lambda *a, **k: ckpt)
+ monkeypatch.setattr(vid, "file_size_mib", lambda _p: LTX23_FILE_MIB)
+ monkeypatch.setattr(vid, "settled_snapshot_device_memory", lambda _t: _b200(free_mib))
+ monkeypatch.setattr(te_prequant, "te_prequant_budget_scale", lambda *a, **k: 1.0)
+ # The plan is drawn before any diffusers class is touched; CPU CI installs no diffusers.
+ diffusers = types.ModuleType("diffusers")
+ diffusers.__version__ = "0.40.0"
+ diffusers.LTX2Pipeline = type("LTX2Pipeline", (), {})
+ monkeypatch.setitem(sys.modules, "diffusers", diffusers)
+ seen: list = []
+ real = plan_diffusion_memory
+
+ def capture(**kw):
+ plan = real(**kw)
+ seen.append(
+ (
+ kw["model_dense_mib"],
+ kw["companion_dense_mib"],
+ kw["requested_mode"],
+ plan.offload_policy,
+ )
+ )
+ raise _Planned()
+
+ monkeypatch.setattr(vid, "plan_diffusion_memory", capture)
+ with pytest.raises(_Planned):
+ backend.load_pipeline(
+ "Lightricks/LTX-2.3",
+ local_files_only = True,
+ gguf_filename = ckpt.name,
+ model_kind = "single_file",
+ memory_mode = MEMORY_MODE_FAST,
+ speed_mode = speed_mode,
+ )
+ return seen[0]
+
+
[email protected]("speed_mode", [None, "eager", "max"])
+def test_ltx23_single_file_prices_the_dit_once_and_stays_resident(
+ monkeypatch, tmp_path, speed_mode
+):
+ dense, companions, mode, policy = _plan_inputs_for_single_file_load(
+ monkeypatch, tmp_path, speed_mode, TENANT_FREE_MIB
+ )
+ assert companions == COMPANION_MIB
+ assert dense == LTX23_DIT_MIB + COMPANION_MIB
+ assert mode == MEMORY_MODE_FAST
+ assert policy == OFFLOAD_NONE
+
+
+def test_ltx23_plan_does_not_depend_on_the_speed_mode(monkeypatch, tmp_path):
+ for free_mib in (TENANT_FREE_MIB, 60 * 1024, B200_TOTAL_MIB):
+ plans = {
+ speed: _plan_inputs_for_single_file_load(monkeypatch, tmp_path, speed, free_mib)
+ for speed in (None, "eager")
+ }
+ assert plans[None] == plans["eager"], free_mib
+
+
+def test_fast_that_offloaded_is_recorded_as_a_fallback_naming_the_policy():
+ from core.inference.diffusion_auto_policy import build_resolved_record
+ from core.inference.video import _memory_mode_resolved
+
+ fell = _plan(60 * 1024, LTX23_DIT_MIB)
+ entry = build_resolved_record(
+ {"memory_mode": _memory_mode_resolved("fast", fell, fell.offload_policy)}
+ )["memory_mode"]
+ assert (entry["status"], entry["requested"], entry["value"]) == (
+ "fell_back",
+ "fast",
+ OFFLOAD_GROUP,
+ )
+ assert "do not fit resident" in entry["reason"]
+
+ kept = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)
+ entry = build_resolved_record(
+ {"memory_mode": _memory_mode_resolved("fast", kept, kept.offload_policy)}
+ )["memory_mode"]
+ assert (entry["status"], entry["value"]) == ("applied", "fast")
+
+
+def test_resident_proof_uses_the_budget_fast_was_placed_with():
+ """Auto precision keeps bf16 only on a proven fit; a fast plan placed resident under its own budget is one."""
+ from core.inference.diffusion import _plan_proves_resident
+
+ fast = _plan(TENANT_FREE_MIB, LTX23_DIT_MIB)
+ assert fast.offload_policy == OFFLOAD_NONE
+ assert fast.estimates["resident_required_mib"] > fast.estimates["safe_device_budget_mib"]
+ assert _plan_proves_resident(fast)
+ auto = _plan(B200_TOTAL_MIB, LTX23_DIT_MIB, MEMORY_MODE_AUTO)
+ assert "resident_budget_mib" not in auto.estimates
+ assert _plan_proves_resident(auto)