Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio LoRA fuse: charge the host RAM guard what the pinned backup really holds (#17191)

Daniel Han4eeffee8ae61
diff --git a/studio/backend/core/inference/diffusion_lora_fuse.py b/studio/backend/core/inference/diffusion_lora_fuse.py
index 4ae7ace166d..40689555a42 100644
--- a/studio/backend/core/inference/diffusion_lora_fuse.py
+++ b/studio/backend/core/inference/diffusion_lora_fuse.py
@@ -19,10 +19,12 @@ from typing import Any, Optional
LORA_FUSE_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE"
BACKUP_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_BACKUP"
HOST_GUARD_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_HOST_GUARD"
+POW2_PACK_ENV = "UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK"
_DEVICE_BACKUP_MIN_FREE = 8 << 30
_FUSED_ATTR = "_unsloth_lora_fused"
_ALIGN = 64
_CHUNK = 64 << 20 # a power of two: no rounding waste in the pinned host allocator
+_PINNED_CHUNK = 512 << 20
def enabled() -> bool:
@@ -113,23 +115,15 @@ def fuse(
device = plan[0][2].device
if any(w.device != device for _l, _a, w in plan):
return False
- # Packed into power-of-two chunks: the pinned host allocator rounds every block up to a power of two, so one
- # slab of the whole backup would hold up to twice its size.
- places, sizes, used = [], [], _CHUNK
- for _layer, _active, weight in plan:
- nbytes = -(-weight.numel() * weight.element_size() // _ALIGN) * _ALIGN
- if nbytes > _CHUNK:
- places.append((len(sizes), 0))
- sizes.append(nbytes)
- used = _CHUNK
- continue
- if used + nbytes > _CHUNK:
- sizes.append(_CHUNK)
- used = 0
- places.append((len(sizes) - 1, used))
- used += nbytes
+ nbytes_list = [-(-w.numel() * w.element_size() // _ALIGN) * _ALIGN for _l, _a, w in plan]
+ places, sizes = _layout(nbytes_list, pow2 = False)
total = sum(sizes)
on_device = _backup_on_device(device, total)
+ pin = False if on_device else _host_backup_pinned(device)
+ if pin and _pow2_pack_enabled():
+ # pinned allocator rounds every block up to a power of two: charge what it really holds
+ places, sizes = _layout(nbytes_list, pow2 = True)
+ total = sum(sizes)
if not on_device and not _host_backup_fits(total):
if logger is not None:
logger.info(
@@ -141,7 +135,6 @@ def fuse(
if on_device:
chunks = [torch.empty(n, dtype = torch.uint8, device = device) for n in sizes]
else:
- pin = _host_backup_pinned(device)
chunks = [torch.empty(n, dtype = torch.uint8, pin_memory = pin) for n in sizes]
except Exception: # noqa: BLE001 - no room for the backup: stay unfused
return False
@@ -177,6 +170,40 @@ def fuse(
return True
+def _pow2_pack_enabled() -> bool:
+ return (os.environ.get(POW2_PACK_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
+
+
+def _layout(nbytes_list: list, pow2: bool) -> tuple:
+ """(chunk, offset) per weight and chunk sizes. ``pow2``: largest-first into power-of-two chunks (pinned memory)."""
+ places, sizes = [None] * len(nbytes_list), []
+ if not pow2:
+ used = _CHUNK
+ for i, nbytes in enumerate(nbytes_list):
+ if nbytes > _CHUNK:
+ places[i] = (len(sizes), 0)
+ sizes.append(nbytes)
+ used = _CHUNK
+ continue
+ if used + nbytes > _CHUNK:
+ sizes.append(_CHUNK)
+ used = 0
+ places[i] = (len(sizes) - 1, used)
+ used += nbytes
+ return places, sizes
+ used = []
+ for i in sorted(range(len(nbytes_list)), key = lambda j: -nbytes_list[j]):
+ nbytes = nbytes_list[i]
+ ci = next((c for c in range(len(sizes)) if used[c] + nbytes <= sizes[c]), None)
+ if ci is None:
+ ci = len(sizes)
+ sizes.append(max(_PINNED_CHUNK, 1 << (nbytes - 1).bit_length()))
+ used.append(0)
+ places[i] = (ci, used[ci])
+ used[ci] += nbytes
+ return places, [1 << (u - 1).bit_length() for u in used]
+
+
def _restore(entries: list) -> None:
import torch
with torch.no_grad():
diff --git a/studio/backend/tests/test_diffusion_lora_fuse.py b/studio/backend/tests/test_diffusion_lora_fuse.py
index 98485bac04c..1d1ab280c20 100644
--- a/studio/backend/tests/test_diffusion_lora_fuse.py
+++ b/studio/backend/tests/test_diffusion_lora_fuse.py
@@ -300,3 +300,72 @@ def test_the_host_backup_is_pageable_on_an_integrated_gpu(monkeypatch, integrate
assert lf._host_backup_pinned(torch.device("cpu")) is False
monkeypatch.setenv(lf.HOST_GUARD_ENV, "0")
assert lf._host_backup_pinned(cuda) is True
+
+
+def _pinned_backup_charge(monkeypatch, shapes):
+ from core.inference import diffusion_lora_fuse as lf
+
+ weights = [torch.empty(s, dtype = torch.bfloat16, device = "meta") for s in shapes]
+ monkeypatch.setattr(lf, "_plan", lambda denoisers: [(None, ["px"], w) for w in weights])
+ monkeypatch.setattr(lf, "_backup_on_device", lambda device, nbytes: False)
+ monkeypatch.setattr(lf, "_host_backup_pinned", lambda device: True)
+ charged, pinned = [], []
+ monkeypatch.setattr(lf, "_host_backup_fits", lambda nbytes: charged.append(nbytes) or True)
+ real_empty = torch.empty
+
+ def empty(
+ *args,
+ pin_memory = False,
+ **kwargs,
+ ):
+ if pin_memory:
+ pinned.append(args[0])
+ return real_empty(1, dtype = torch.uint8)
+ return real_empty(*args, **kwargs)
+
+ monkeypatch.setattr(torch, "empty", empty)
+ try:
+ assert lf.fuse(types.SimpleNamespace(), [None]) is False # stops at the stub backup
+ finally:
+ monkeypatch.setattr(torch, "empty", real_empty)
+ # CUDA's caching host allocator rounds every pinned block up to a power of two
+ return charged[0], sum(1 << (n - 1).bit_length() for n in pinned)
+
+
+_FLUX_LIKE = [(3072, 12288)] * 24 + [(3072, 21504), (15360, 3072)] * 4 + [(3072, 3072)] * 8
+
+
+def test_the_host_guard_is_charged_what_the_pinned_allocator_holds(monkeypatch):
+ from core.inference import diffusion_lora_fuse as lf
+
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK", raising = False)
+ charged, held = _pinned_backup_charge(monkeypatch, _FLUX_LIKE)
+ assert charged >= held
+ exact = sum(-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE)
+ assert held <= exact * 1.15
+
+
+def test_power_of_two_layout_places_every_weight_inside_its_chunk():
+ from core.inference import diffusion_lora_fuse as lf
+
+ nbytes = [-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE]
+ places, sizes = lf._layout(nbytes, pow2 = True)
+ assert all(s & (s - 1) == 0 for s in sizes)
+ spans = {}
+ for n, (ci, off) in zip(nbytes, places):
+ assert off % lf._ALIGN == 0 and off + n <= sizes[ci]
+ spans.setdefault(ci, []).append((off, off + n))
+ for chunk in spans.values():
+ chunk.sort()
+ assert all(a[1] <= b[0] for a, b in zip(chunk, chunk[1:]))
+
+
+def test_power_of_two_packing_kill_switch_keeps_the_exact_slabs(monkeypatch):
+ from core.inference import diffusion_lora_fuse as lf
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK", "0")
+ assert lf.POW2_PACK_ENV == "UNSLOTH_DIFFUSION_LORA_FUSE_POW2_PACK"
+ charged, _held = _pinned_backup_charge(monkeypatch, _FLUX_LIKE)
+ assert charged == sum(
+ lf._layout([-(-r * c * 2 // 64) * 64 for r, c in _FLUX_LIKE], pow2 = False)[1]
+ )