Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Mask from text runs SAM 3.1 in bf16 unless the card is short of memory (#14736)

Daniel Han9cd247d678f0
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.py
index 9b1ee40fb15..e60b098973c 100644
--- a/studio/backend/core/inference/diffusion_segment.py
+++ b/studio/backend/core/inference/diffusion_segment.py
@@ -21,10 +21,11 @@ when it is already downloaded and SAM 3.1 is not. ``UNSLOTH_SAM3_PATH`` points a
directory instead.
SAM 3.1 also ships pre-quantized detector files (FP8, INT8, INT8-ConvRot; diffusion_segment_quant). On a CUDA card
-``UNSLOTH_SAM3_PRECISION`` (auto / bf16 / fp8 / int8) picks one; auto loads only a file that passed the accuracy gate
-(INT8, the ConvRot build unless ``UNSLOTH_SAM3_INT8_CONVROT=0``). The download fetches that one weights file with the
-configs and tokenizer, and bf16 when the file is not published; the CPU and Apple's MPS run bf16 weights (a quantized
-file that is the only one on disk is dequantized there).
+``UNSLOTH_SAM3_PRECISION`` (auto / bf16 / fp8 / int8) picks one. auto runs bf16, which is faster, while the card has
+room for it, and only when it is short of memory loads a file that passed the accuracy gate (INT8, the ConvRot build
+unless ``UNSLOTH_SAM3_INT8_CONVROT=0``). The download fetches that one weights file with the configs and tokenizer,
+and bf16 when the file is not published; the CPU and Apple's MPS run bf16 weights (a quantized file that is the only
+one on disk is dequantized there).
The model is auxiliary to the image model, like the FLUX.3 prompt upsampler: loaded on the first request, kept for
a short keep-alive, unloaded right after a call when the accelerator is tight, and dropped whenever the GPU arbiter
@@ -174,6 +175,9 @@ _GPU_NEED_MIB = 4096
# never denoises next to it. The prompt upsampler's tight rule.
_TIGHT_FREE_MIB = 6144
_TIGHT_FREE_FRACTION = 0.20
+# bf16's measured drop in mem_get_info free after a Find (reserve + workspace, above its allocated peak), so the bf16
+# pick is not read as tight by _after_call and unloaded. auto goes 8-bit only if free minus this reads tight.
+_BF16_RESIDENT_MIB = 2560
class SegmenterUnavailableError(RuntimeError):
@@ -643,10 +647,12 @@ def plan_weights(
*,
variant: Optional[SegmenterVariant] = None,
use_cache: bool = True,
+ memory_tight: bool = False,
) -> WeightsPlan:
- """The weights a load on ``device`` uses: the precision UNSLOTH_SAM3_PRECISION asks for (auto: the gated order
- in diffusion_segment_quant) when its file is on disk and the device runs it, else bf16. bf16 with no bf16 file
- on disk dequantizes a pre-quantized one the selection may load (never one it rejected)."""
+ """The weights a load on ``device`` uses: the precision UNSLOTH_SAM3_PRECISION asks for (auto: bf16, or the gated
+ order in diffusion_segment_quant when ``memory_tight``) when its file is on disk and the device runs it, else
+ bf16. bf16 with no bf16 file on disk dequantizes a pre-quantized one the selection may load (never one it
+ rejected)."""
from core.inference.diffusion_segment_quant import PRECISION_BF16, resolve_precision
variant = variant or active_variant()
@@ -655,7 +661,9 @@ def plan_weights(
for artifact, folder in local:
if _loadable(variant, artifact):
usable.setdefault(artifact.precision, (artifact, folder))
- precision, why = resolve_precision(device, dtype, available = tuple(usable))
+ precision, why = resolve_precision(
+ device, dtype, available = tuple(usable), memory_tight = memory_tight
+ )
if precision != PRECISION_BF16:
artifact, folder = usable[precision]
return WeightsPlan(precision, folder, str(Path(folder) / artifact.filename), artifact, why)
@@ -686,7 +694,12 @@ def _host_device() -> str:
return "cpu"
-def download_plan(variant: SegmenterVariant, device: Optional[str] = None) -> dict:
+def download_plan(
+ variant: SegmenterVariant,
+ device: Optional[str] = None,
+ *,
+ memory_tight: Optional[bool] = None,
+) -> dict:
"""What a download of ``variant`` fetches on this host: the spec id, the precision it runs, why, and the bytes.
A pre-quantized file only when the variant publishes them (a pinned or "main" prequant revision) and the host
would run it; its spec falls back to bf16 when the file is not on the Hub yet."""
@@ -700,7 +713,10 @@ def download_plan(variant: SegmenterVariant, device: Optional[str] = None) -> di
_cached_snapshot(prequant_spec_id(variant, artifact)), artifact
):
offered.setdefault(artifact.precision, artifact)
- precision, why = resolve_precision(device or _host_device(), available = tuple(offered))
+ device = device or _host_device()
+ if memory_tight is None:
+ memory_tight = auto_memory_tight(device)
+ precision, why = resolve_precision(device, available = tuple(offered), memory_tight = memory_tight)
if precision == PRECISION_BF16:
return {
"spec": variant.id,
@@ -784,7 +800,14 @@ def _precision_summary(
elif not supported:
body.update(precision = None, precision_reason = None, weights_file = None)
elif path is not None:
- plan = plan_weights(path, _host_device(), variant = variant, use_cache = not override)
+ device = _host_device()
+ plan = plan_weights(
+ path,
+ device,
+ variant = variant,
+ use_cache = not override,
+ memory_tight = auto_memory_tight(device),
+ )
body.update(
precision = plan.precision,
precision_reason = plan.reason,
@@ -854,6 +877,24 @@ def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:
return free_mib < floor
+def auto_memory_tight(device: str) -> bool:
+ """The CUDA card ``device`` would read as tight with the bf16 segmenter on it. False off CUDA, when unreadable, and
+ before this process initialised CUDA (never create a context just to plan)."""
+ if not device.startswith("cuda"):
+ return False
+ try:
+ import torch
+ if not torch.cuda.is_initialized():
+ return False
+ ordinal = int(device.split(":", 1)[1]) if ":" in device else torch.cuda.current_device()
+ except Exception: # noqa: BLE001
+ return False
+ free_mib, total_mib = _cuda_free_mib(ordinal)
+ if free_mib is None:
+ return False
+ return vram_is_tight(free_mib - _BF16_RESIDENT_MIB, total_mib)
+
+
def _other_owner_on(ordinal: Optional[int]) -> Optional[str]:
"""The GPU arbiter's resident other than the image model on CUDA card ``ordinal`` (one with unknown GPUs counts
for every card; ``ordinal`` None is Apple's single Metal device, which every resident shares), else None. Called under the run lock after the handover fence: a later handover waits for the
@@ -1170,6 +1211,8 @@ class MaskSegmenter:
self.weights_file: Optional[str] = None
# Why the last pre-quantized load fell back to bf16, if one did.
self.last_prequant_failure: Optional[str] = None
+ # Reused while loaded: a reading taken with the model resident would flip auto's choice and reload it.
+ self._memory_tight: Optional[bool] = None
# Planning reads the Hub cache only for the active variant's own snapshot, not an explicit folder.
self._use_cache = True
self._timer: Optional[threading.Timer] = None
@@ -1208,7 +1251,13 @@ class MaskSegmenter:
# -- lifecycle -------------------------------------------------------------------------------------------------
def _load(self, path: str, device: str, dtype: Any) -> None:
- plan = plan_weights(path, device, dtype, use_cache = self._use_cache)
+ if self._model is not None and self.device == device and self._memory_tight is not None:
+ memory_tight = self._memory_tight
+ else:
+ memory_tight = auto_memory_tight(device)
+ plan = plan_weights(
+ path, device, dtype, use_cache = self._use_cache, memory_tight = memory_tight
+ )
key = (plan.model_dir, plan.weights, plan.dequantize)
if (
self._model is not None
@@ -1238,6 +1287,7 @@ class MaskSegmenter:
self._model, self._tokenizer = model, tokenizer
self._path, self.device, self._dtype = path, device, dtype
self._weights_key = key
+ self._memory_tight = memory_tight
self.precision, self.precision_reason = plan.precision, plan.reason
self.weights_file = (
os.path.basename(plan.weights) if plan.weights else SEGMENTER_WEIGHTS_FILE
@@ -1484,6 +1534,7 @@ class MaskSegmenter:
self._weights_key = None
self.device = None
self._dtype = None
+ self._memory_tight = None
self.precision = self.precision_reason = self.weights_file = None
_free_device_cache(device)
logger.info("images.segment: unloaded from %s (%s)", device, reason)
diff --git a/studio/backend/core/inference/diffusion_segment_quant.py b/studio/backend/core/inference/diffusion_segment_quant.py
index 78f1c8cb55a..9e3f5328bad 100644
--- a/studio/backend/core/inference/diffusion_segment_quant.py
+++ b/studio/backend/core/inference/diffusion_segment_quant.py
@@ -43,7 +43,8 @@ ENV_PRECISION = "UNSLOTH_SAM3_PRECISION"
# int8 loads the ConvRot artifact unless this is 0 (then the plain INT8 one).
ENV_INT8_CONVROT = "UNSLOTH_SAM3_INT8_CONVROT"
-# What ``auto`` may pick on a CUDA card, best first; it also needs that file's GATE_PASSED. Empty: auto stays bf16.
+# What ``auto`` may pick on a CUDA card short of memory, best first (needs GATE_PASSED); empty: always bf16. With room
+# auto runs bf16: at this size the 8-bit files save ~0.5 GB but are ~2x slower per Find (activation quant cost).
AUTO_ORDER: tuple[str, ...] = (PRECISION_INT8,)
# Minimum compute capability per scheme: fp8 scaled_mm needs Ada / Hopper (sm_89+), int8 _int_mm Ampere (sm_80+).
@@ -471,12 +472,16 @@ def resolve_precision(
dtype: Any = None,
*,
available: tuple[str, ...] = QUANT_PRECISIONS,
+ memory_tight: bool = False,
) -> tuple[str, str]:
"""``(precision, why)`` for a run on ``device``. ``available`` is the quantized arms the variant ships (and, at
- load, has on disk). bf16 when asked for, when nothing quantized can run here, or when auto has no gated arm."""
+ load, has on disk). bf16 when asked for, when nothing quantized can run here, when auto has no gated arm, or for
+ auto on CUDA unless ``memory_tight``. An explicit fp8 / int8 is honoured either way."""
requested = requested_precision()
if requested == PRECISION_BF16:
return PRECISION_BF16, f"{ENV_PRECISION}=bf16"
+ if requested == PRECISION_AUTO and not memory_tight and device.startswith("cuda"):
+ return PRECISION_BF16, "auto: bf16 fits and is faster"
candidates = AUTO_ORDER if requested == PRECISION_AUTO else (requested,)
reasons = []
for precision in candidates:
diff --git a/studio/backend/tests/test_diffusion_segment_quant.py b/studio/backend/tests/test_diffusion_segment_quant.py
index 9c8f212dcb0..f42085f3a5d 100644
--- a/studio/backend/tests/test_diffusion_segment_quant.py
+++ b/studio/backend/tests/test_diffusion_segment_quant.py
@@ -54,20 +54,118 @@ def test_requested_precision(monkeypatch, value, expected):
assert sq.requested_precision() == expected
-def test_auto_follows_the_gated_order_on_cuda(monkeypatch):
+def test_auto_follows_the_gated_order_on_a_tight_cuda_card(monkeypatch):
import torch
_cuda(monkeypatch, (9, 0))
assert sq.AUTO_ORDER, "the gate passed at least one quantized arm"
- assert sq.resolve_precision("cuda:0", torch.bfloat16)[0] == sq.AUTO_ORDER[0]
+ tight = dict(memory_tight = True)
+ assert sq.resolve_precision("cuda:0", torch.bfloat16, **tight)[0] == sq.AUTO_ORDER[0]
# Only what is on disk (or published) counts.
- assert sq.resolve_precision("cuda:0", available = ())[0] == "bf16"
- assert sq.resolve_precision("cuda:0", available = (sq.AUTO_ORDER[-1],))[0] == sq.AUTO_ORDER[-1]
+ assert sq.resolve_precision("cuda:0", available = (), **tight)[0] == "bf16"
+ assert (
+ sq.resolve_precision("cuda:0", available = (sq.AUTO_ORDER[-1],), **tight)[0]
+ == sq.AUTO_ORDER[-1]
+ )
monkeypatch.setattr(sq, "AUTO_ORDER", ())
- precision, why = sq.resolve_precision("cuda:0")
+ precision, why = sq.resolve_precision("cuda:0", **tight)
assert precision == "bf16" and "gate" in why
+def test_auto_runs_bf16_while_the_card_has_room(monkeypatch):
+ """The 8-bit files are about 2.2x slower per Find than bf16 (model card, B200), so auto only trades speed for
+ memory when the card is short of it. An explicit int8 / fp8 is honoured either way."""
+ import torch
+
+ _cuda(monkeypatch, (9, 0))
+ for args in ((), (torch.bfloat16,)):
+ precision, why = sq.resolve_precision("cuda:0", *args)
+ assert precision == "bf16" and "auto" in why
+ assert sq.resolve_precision("cuda:0", *args, memory_tight = False)[0] == "bf16"
+ for value in ("int8", "fp8"):
+ monkeypatch.setenv(sq.ENV_PRECISION, value)
+ assert sq.resolve_precision("cuda:0", torch.bfloat16)[0] == value
+ assert sq.resolve_precision("cuda:0", torch.bfloat16, memory_tight = True)[0] == value
+ monkeypatch.setenv(sq.ENV_PRECISION, "bf16")
+ assert sq.resolve_precision("cuda:0", torch.bfloat16, memory_tight = True)[0] == "bf16"
+
+
+def test_auto_memory_tight_reads_the_card_only_once_cuda_is_up(monkeypatch):
+ import torch
+
+ reads = []
+
+ def free(ordinal):
+ reads.append(ordinal)
+ return free_total
+
+ monkeypatch.setattr(seg, "_cuda_free_mib", free)
+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: False)
+ free_total = (1024, 80_000)
+ # No CUDA context in this process yet: nothing of ours is on a card, and planning must not create one.
+ assert seg.auto_memory_tight("cuda:0") is False and reads == []
+ assert seg.auto_memory_tight("cpu") is False and seg.auto_memory_tight("mps") is False
+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: True)
+ # 80 GB card: tight below 20% (16000 MiB) free once bf16's 2048 MiB are placed.
+ free_total = (60_000, 80_000)
+ assert seg.auto_memory_tight("cuda:1") is False and reads == [1]
+ free_total = (16_000 + seg._BF16_RESIDENT_MIB - 1, 80_000)
+ assert seg.auto_memory_tight("cuda:1") is True
+ free_total = (16_000 + seg._BF16_RESIDENT_MIB, 80_000)
+ assert seg.auto_memory_tight("cuda:1") is False
+ # Small card: the 6 GiB floor.
+ free_total = (seg._TIGHT_FREE_MIB + seg._BF16_RESIDENT_MIB - 1, 12_000)
+ assert seg.auto_memory_tight("cuda:0") is True
+ free_total = (None, None)
+ assert seg.auto_memory_tight("cuda:0") is False
+
+
+def test_a_bf16_pick_is_not_unloaded_as_tight_after_its_first_find(monkeypatch):
+ # mem_get_info free drop with bf16 resident after a Find, A100 + torch 2.11 (dequantized INT8 file, the larger).
+ import torch
+
+ measured_drop_mib = 2282
+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: True)
+ for total in (12_000, 24_000, 40_000, 80_000):
+ floor = max(seg._TIGHT_FREE_MIB, int(total * seg._TIGHT_FREE_FRACTION))
+ free = floor + seg._BF16_RESIDENT_MIB
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal, f = free, t = total: (f, t))
+ assert seg.auto_memory_tight("cuda:0") is False
+ assert not seg.vram_is_tight(free - measured_drop_mib, total), total
+
+
+def test_a_loaded_segmenter_keeps_its_memory_reading(monkeypatch):
+ """Loaded on a card, the free memory read on the next call no longer counts the segmenter itself: the choice made
+ at load is reused, so a bf16 model is not swapped for int8 (or back) on every call."""
+ seen = []
+ plan = seg.WeightsPlan("bf16", "/m")
+ monkeypatch.setattr(
+ seg, "plan_weights", lambda *a, memory_tight, **k: seen.append(memory_tight) or plan
+ )
+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ segmenter._model, segmenter._path, segmenter.device, segmenter._dtype = (
+ object(),
+ "/m",
+ "cuda:0",
+ "bf16",
+ )
+ segmenter._weights_key, segmenter._memory_tight = ("/m", None, False), False
+ segmenter._load("/m", "cuda:0", "bf16")
+ assert seen == [False]
+ # Another device (or nothing loaded) reads the card again.
+ segmenter._model = None
+ segmenter._memory_tight = None
+ monkeypatch.setattr(segmenter, "_load_weights", lambda plan, *a: (object(), plan))
+ import transformers
+
+ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: object())
+ segmenter._load("/m", "cuda:0", "bf16")
+ assert seen == [False, True] and segmenter._memory_tight is True
+ segmenter._unload_locked("test")
+ assert segmenter._memory_tight is None
+
+
def test_quantized_kernels_are_cuda_only(monkeypatch):
import torch
@@ -175,7 +273,10 @@ def test_download_plan_fetches_one_weights_file(monkeypatch):
seg.SAM31_SIZE_BYTES,
)
variant = _published(monkeypatch)
- plan = seg.download_plan(variant, "cuda:0")
+ # A card with room fetches bf16 (the faster path); a tight one the smaller gated file.
+ assert seg.download_plan(variant, "cuda:0")["spec"] == "sam3.1"
+ assert seg.download_plan(variant, "cuda:0", memory_tight = False)["spec"] == "sam3.1"
+ plan = seg.download_plan(variant, "cuda:0", memory_tight = True)
artifact = seg.variant_artifact(variant, sq.AUTO_ORDER[0])
assert (
plan["spec"] == seg.prequant_spec_id(variant, artifact)
@@ -184,10 +285,13 @@ def test_download_plan_fetches_one_weights_file(monkeypatch):
assert plan["file"] == artifact.filename
assert plan["size_bytes"] == artifact.size_bytes + seg.SAM31_CONFIG_BYTES < seg.SAM31_SIZE_BYTES
# CPU / Mac hosts fetch bf16.
- assert seg.download_plan(variant, "cpu")["spec"] == "sam3.1"
- assert seg.download_plan(variant, "mps")["spec"] == "sam3.1"
+ assert seg.download_plan(variant, "cpu", memory_tight = True)["spec"] == "sam3.1"
+ assert seg.download_plan(variant, "mps", memory_tight = True)["spec"] == "sam3.1"
+ # An explicit int8 fetches the int8 file whatever the memory.
+ monkeypatch.setenv(sq.ENV_PRECISION, "int8")
+ assert seg.download_plan(variant, "cuda:0")["spec"] == seg.prequant_spec_id(variant, artifact)
monkeypatch.setenv(sq.ENV_PRECISION, "bf16")
- assert seg.download_plan(variant, "cuda:0")["spec"] == "sam3.1"
+ assert seg.download_plan(variant, "cuda:0", memory_tight = True)["spec"] == "sam3.1"
def test_start_download_uses_the_plan(monkeypatch):
@@ -204,9 +308,12 @@ def test_start_download_uses_the_plan(monkeypatch):
monkeypatch.setattr(seg, "_get_download_state", lambda: _State())
seg.start_download(None, "sam3.1")
- assert started == [
- seg.prequant_spec_id(variant, seg.variant_artifact(variant, sq.AUTO_ORDER[0]))
- ]
+ assert started == ["sam3.1"]
+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)
+ seg.start_download(None, "sam3.1")
+ assert started[-1] == seg.prequant_spec_id(
+ variant, seg.variant_artifact(variant, sq.AUTO_ORDER[0])
+ )
# A spec id passes through untouched.
seg.start_download(None, "sam3.1@fp8")
assert started[-1] == "sam3.1@fp8"
@@ -321,7 +428,12 @@ def test_plan_weights(monkeypatch, tmp_path):
sam31 = seg.SEGMENTER_VARIANTS["sam3.1"]
every = [a.filename for a in sam31.prequant]
full = _folder(tmp_path / "full", "model.safetensors", *every)
+ # auto with room: the bf16 file, though the quantized ones are on disk.
plan = seg.plan_weights(full, "cuda:0", torch.bfloat16, variant = sam31, use_cache = False)
+ assert (plan.precision, plan.weights, plan.dequantize) == ("bf16", None, False)
+ plan = seg.plan_weights(
+ full, "cuda:0", torch.bfloat16, variant = sam31, use_cache = False, memory_tight = True
+ )
first = seg.variant_artifact(sam31, sq.AUTO_ORDER[0])
assert (plan.precision, plan.weights, plan.dequantize) == (
sq.AUTO_ORDER[0],
@@ -361,7 +473,10 @@ def test_only_a_quantized_file_on_a_cpu_is_dequantized(monkeypatch, tmp_path):
and plan.dequantize
and plan.weights.endswith("INT8-ConvRot.safetensors")
)
+ # A card with room runs it dequantized (bf16 speed); a tight one runs it quantized.
plan = seg.plan_weights(only, "cuda:0", variant = sam31, use_cache = False)
+ assert plan.precision == "bf16" and plan.dequantize
+ plan = seg.plan_weights(only, "cuda:0", variant = sam31, use_cache = False, memory_tight = True)
assert plan.precision == "int8" and not plan.dequantize
@@ -475,6 +590,9 @@ def test_status_reports_the_precision_and_its_size(monkeypatch):
assert body["precision_requested"] == "auto"
variant = _published(monkeypatch)
body = seg.status()
+ assert (body["precision"], body["weights_file"]) == ("bf16", "model.safetensors")
+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)
+ body = seg.status()
artifact = seg.variant_artifact(variant, sq.AUTO_ORDER[0])
assert (body["precision"], body["weights_file"]) == (sq.AUTO_ORDER[0], artifact.filename)
assert body["size_bytes"] == artifact.size_bytes + seg.SAM31_CONFIG_BYTES
@@ -553,7 +671,8 @@ def test_auto_skips_a_file_that_failed_the_gate(monkeypatch, tmp_path):
sam31 = seg.SEGMENTER_VARIANTS["sam3.1"]
fp8_only = _folder(tmp_path, "model.safetensors", "sam3.1-FP8.safetensors")
monkeypatch.setattr(sq, "AUTO_ORDER", ("fp8",))
- assert seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False).precision == "bf16"
+ plan = seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False, memory_tight = True)
+ assert plan.precision == "bf16"
monkeypatch.setenv(sq.ENV_PRECISION, "fp8")
assert seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False).precision == "fp8"