Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio LoRA fuse: charge the host RAM guard what the pinned backup really holds (#17191)
Daniel Han4eeffee8ae61
diff --git a/studio/backend/core/inference/diffusion_lora_fuse.py b/studio/backend/core/inference/diffusion_lora_fuse.pyindex 4ae7ace166d..40689555a42 100644--- a/studio/backend/core/inference/diffusion_lora_fuse.py+++ b/studio/backend/core/inference/diffusion_lora_fuse.py@@ -19,10 +19,12 @@ from typing import Any, OptionalLORA_FUSE_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE"BACKUP_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_BACKUP"HOST_GUARD_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_HOST_GUARD"+POW2_PACK_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK"_DEVICE_BACKUP_MIN_FREE = 8 << 30_FUSED_ATTR = "_unsloth_lora_fused"_ALIGN = 64_CHUNK = 64 << 20 # a power of two: no rounding waste in the pinned host allocator+_PINNED_CHUNK = 512 << 20def enabled() -> bool:@@ -113,23 +115,15 @@ def fuse(device = plan[0][2].deviceif any(w.device != device for _l, _a, w in plan):return False- # Packed into power-of-two chunks: the pinned host allocator rounds every block up to a power of two, so one- # slab of the whole backup would hold up to twice its size.- places, sizes, used = [], [], _CHUNK- for _layer, _active, weight in plan:- nbytes = -(-weight.numel() * weight.element_size() // _ALIGN) * _ALIGN- if nbytes > _CHUNK:- places.append((len(sizes), 0))- sizes.append(nbytes)- used = _CHUNK- continue- if used + nbytes > _CHUNK:- sizes.append(_CHUNK)- used = 0- places.append((len(sizes) - 1, used))- used += nbytes+ nbytes_list = [-(-w.numel() * w.element_size() // _ALIGN) * _ALIGN for _l, _a, w in plan]+ places, sizes = _layout(nbytes_list, pow2 = False)total = sum(sizes)on_device = _backup_on_device(device, total)+ pin = False if on_device else _host_backup_pinned(device)+ if pin and _pow2_pack_enabled():+ # pinned allocator rounds every block up to a power of two: charge what it really holds+ places, sizes = _layout(nbytes_list, pow2 = True)+ total = sum(sizes)if not on_device and not _host_backup_fits(total):if logger is not None:logger.info(@@ -141,7 +135,6 @@ def fuse(if on_device:chunks = [torch.empty(n, dtype = torch.uint8, device = device) for n in sizes]else:- pin = _host_backup_pinned(device)chunks = [torch.empty(n, dtype = torch.uint8, pin_memory = pin) for n in sizes]except Exception: # noqa: BLE001 - no room for the backup: stay unfusedreturn False@@ -177,6 +170,40 @@ def fuse(return True+def _pow2_pack_enabled() -> bool:+ return (os.environ.get(POW2_PACK_ENV) or "").strip().lower() not in ("0", "off", "false", "no")+++def _layout(nbytes_list: list, pow2: bool) -> tuple:+ """(chunk, offset) per weight and chunk sizes. ``pow2``: largest-first into power-of-two chunks (pinned memory)."""+ places, sizes = [None] * len(nbytes_list), []+ if not pow2:+ used = _CHUNK+ for i, nbytes in enumerate(nbytes_list):+ if nbytes > _CHUNK:+ places[i] = (len(sizes), 0)+ sizes.append(nbytes)+ used = _CHUNK+ continue+ if used + nbytes > _CHUNK:+ sizes.append(_CHUNK)+ used = 0+ places[i] = (len(sizes) - 1, used)+ used += nbytes+ return places, sizes+ used = []+ for i in sorted(range(len(nbytes_list)), key = lambda j: -nbytes_list[j]):+ nbytes = nbytes_list[i]+ ci = next((c for c in range(len(sizes)) if used[c] + nbytes <= sizes[c]), None)+ if ci is None:+ ci = len(sizes)+ sizes.append(max(_PINNED_CHUNK, 1 << (nbytes - 1).bit_length()))+ used.append(0)+ places[i] = (ci, used[ci])+ used[ci] += nbytes+ return places, [1 << (u - 1).bit_length() for u in used]++def _restore(entries: list) -> None:import torchwith torch.no_grad():diff --git a/studio/backend/tests/test_diffusion_lora_fuse.py b/studio/backend/tests/test_diffusion_lora_fuse.pyindex 98485bac04c..1d1ab280c20 100644--- a/studio/backend/tests/test_diffusion_lora_fuse.py+++ b/studio/backend/tests/test_diffusion_lora_fuse.py@@ -300,3 +300,72 @@ def test_the_host_backup_is_pageable_on_an_integrated_gpu(monkeypatch, integrateassert lf._host_backup_pinned(torch.device("cpu")) is Falsemonkeypatch.setenv(lf.HOST_GUARD_ENV, "0")assert lf._host_backup_pinned(cuda) is True+++def _pinned_backup_charge(monkeypatch, shapes):+ from core.inference import diffusion_lora_fuse as lf++ weights = [torch.empty(s, dtype = torch.bfloat16, device = "meta") for s in shapes]+ monkeypatch.setattr(lf, "_plan", lambda denoisers: [(None, ["px"], w) for w in weights])+ monkeypatch.setattr(lf, "_backup_on_device", lambda device, nbytes: False)+ monkeypatch.setattr(lf, "_host_backup_pinned", lambda device: True)+ charged, pinned = [], []+ monkeypatch.setattr(lf, "_host_backup_fits", lambda nbytes: charged.append(nbytes) or True)+ real_empty = torch.empty++ def empty(+ *args,+ pin_memory = False,+ **kwargs,+ ):+ if pin_memory:+ pinned.append(args[0])+ return real_empty(1, dtype = torch.uint8)+ return real_empty(*args, **kwargs)++ monkeypatch.setattr(torch, "empty", empty)+ try:+ assert lf.fuse(types.SimpleNamespace(), [None]) is False # stops at the stub backup+ finally:+ monkeypatch.setattr(torch, "empty", real_empty)+ # CUDA's caching host allocator rounds every pinned block up to a power of two+ return charged[0], sum(1 << (n - 1).bit_length() for n in pinned)+++_FLUX_LIKE = [(3072, 12288)] * 24 + [(3072, 21504), (15360, 3072)] * 4 + [(3072, 3072)] * 8+++def test_the_host_guard_is_charged_what_the_pinned_allocator_holds(monkeypatch):+ from core.inference import diffusion_lora_fuse as lf++ monkeypatch.delenv("UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK", raising = False)+ charged, held = _pinned_backup_charge(monkeypatch, _FLUX_LIKE)+ assert charged >= held+ exact = sum(-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE)+ assert held <= exact * 1.15+++def test_power_of_two_layout_places_every_weight_inside_its_chunk():+ from core.inference import diffusion_lora_fuse as lf++ nbytes = [-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE]+ places, sizes = lf._layout(nbytes, pow2 = True)+ assert all(s & (s - 1) == 0 for s in sizes)+ spans = {}+ for n, (ci, off) in zip(nbytes, places):+ assert off % lf._ALIGN == 0 and off + n <= sizes[ci]+ spans.setdefault(ci, []).append((off, off + n))+ for chunk in spans.values():+ chunk.sort()+ assert all(a[1] <= b[0] for a, b in zip(chunk, chunk[1:]))+++def test_power_of_two_packing_kill_switch_keeps_the_exact_slabs(monkeypatch):+ from core.inference import diffusion_lora_fuse as lf++ monkeypatch.setenv("UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK", "0")+ assert lf.POW2_PACK_ENV == "UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK"+ charged, _held = _pinned_backup_charge(monkeypatch, _FLUX_LIKE)+ assert charged == sum(+ lf._layout([-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE], pow2 = False)[1]+ )