Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Mask from text runs SAM 3.1 in bf16 unless the card is short of memory (#14736)
Daniel Han9cd247d678f0
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.pyindex 9b1ee40fb15..e60b098973c 100644--- a/studio/backend/core/inference/diffusion_segment.py+++ b/studio/backend/core/inference/diffusion_segment.py@@ -21,10 +21,11 @@ when it is already downloaded and SAM 3.1 is not. ``UNSLOTH_SAM3_PATH`` points adirectory instead.SAM 3.1 also ships pre-quantized detector files (FP8, INT8, INT8-ConvRot; diffusion_segment_quant). On a CUDA card-``UNSLOTH_SAM3_PRECISION`` (auto / bf16 / fp8 / int8) picks one; auto loads only a file that passed the accuracy gate-(INT8, the ConvRot build unless ``UNSLOTH_SAM3_INT8_CONVROT=0``). The download fetches that one weights file with the-configs and tokenizer, and bf16 when the file is not published; the CPU and Apple's MPS run bf16 weights (a quantized-file that is the only one on disk is dequantized there).+``UNSLOTH_SAM3_PRECISION`` (auto / bf16 / fp8 / int8) picks one. auto runs bf16, which is faster, while the card has+room for it, and only when it is short of memory loads a file that passed the accuracy gate (INT8, the ConvRot build+unless ``UNSLOTH_SAM3_INT8_CONVROT=0``). The download fetches that one weights file with the configs and tokenizer,+and bf16 when the file is not published; the CPU and Apple's MPS run bf16 weights (a quantized file that is the only+one on disk is dequantized there).The model is auxiliary to the image model, like the FLUX.3 prompt upsampler: loaded on the first request, kept fora short keep-alive, unloaded right after a call when the accelerator is tight, and dropped whenever the GPU arbiter@@ -174,6 +175,9 @@ _GPU_NEED_MIB = 4096# never denoises next to it. The prompt upsampler's tight rule._TIGHT_FREE_MIB = 6144_TIGHT_FREE_FRACTION = 0.20+# bf16's measured drop in mem_get_info free after a Find (reserve + workspace, above its allocated peak), so the bf16+# pick is not read as tight by _after_call and unloaded. auto goes 8-bit only if free minus this reads tight.+_BF16_RESIDENT_MIB = 2560class SegmenterUnavailableError(RuntimeError):@@ -643,10 +647,12 @@ def plan_weights(*,variant: Optional[SegmenterVariant] = None,use_cache: bool = True,+ memory_tight: bool = False,) -> WeightsPlan:- """The weights a load on ``device`` uses: the precision UNSLOTH_SAM3_PRECISION asks for (auto: the gated order- in diffusion_segment_quant) when its file is on disk and the device runs it, else bf16. bf16 with no bf16 file- on disk dequantizes a pre-quantized one the selection may load (never one it rejected)."""+ """The weights a load on ``device`` uses: the precision UNSLOTH_SAM3_PRECISION asks for (auto: bf16, or the gated+ order in diffusion_segment_quant when ``memory_tight``) when its file is on disk and the device runs it, else+ bf16. bf16 with no bf16 file on disk dequantizes a pre-quantized one the selection may load (never one it+ rejected)."""from core.inference.diffusion_segment_quant import PRECISION_BF16, resolve_precisionvariant = variant or active_variant()@@ -655,7 +661,9 @@ def plan_weights(for artifact, folder in local:if _loadable(variant, artifact):usable.setdefault(artifact.precision, (artifact, folder))- precision, why = resolve_precision(device, dtype, available = tuple(usable))+ precision, why = resolve_precision(+ device, dtype, available = tuple(usable), memory_tight = memory_tight+ )if precision != PRECISION_BF16:artifact, folder = usable[precision]return WeightsPlan(precision, folder, str(Path(folder) / artifact.filename), artifact, why)@@ -686,7 +694,12 @@ def _host_device() -> str:return "cpu"-def download_plan(variant: SegmenterVariant, device: Optional[str] = None) -> dict:+def download_plan(+ variant: SegmenterVariant,+ device: Optional[str] = None,+ *,+ memory_tight: Optional[bool] = None,+) -> dict:"""What a download of ``variant`` fetches on this host: the spec id, the precision it runs, why, and the bytes.A pre-quantized file only when the variant publishes them (a pinned or "main" prequant revision) and the hostwould run it; its spec falls back to bf16 when the file is not on the Hub yet."""@@ -700,7 +713,10 @@ def download_plan(variant: SegmenterVariant, device: Optional[str] = None) -> di_cached_snapshot(prequant_spec_id(variant, artifact)), artifact):offered.setdefault(artifact.precision, artifact)- precision, why = resolve_precision(device or _host_device(), available = tuple(offered))+ device = device or _host_device()+ if memory_tight is None:+ memory_tight = auto_memory_tight(device)+ precision, why = resolve_precision(device, available = tuple(offered), memory_tight = memory_tight)if precision == PRECISION_BF16:return {"spec": variant.id,@@ -784,7 +800,14 @@ def _precision_summary(elif not supported:body.update(precision = None, precision_reason = None, weights_file = None)elif path is not None:- plan = plan_weights(path, _host_device(), variant = variant, use_cache = not override)+ device = _host_device()+ plan = plan_weights(+ path,+ device,+ variant = variant,+ use_cache = not override,+ memory_tight = auto_memory_tight(device),+ )body.update(precision = plan.precision,precision_reason = plan.reason,@@ -854,6 +877,24 @@ def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:return free_mib < floor+def auto_memory_tight(device: str) -> bool:+ """The CUDA card ``device`` would read as tight with the bf16 segmenter on it. False off CUDA, when unreadable, and+ before this process initialised CUDA (never create a context just to plan)."""+ if not device.startswith("cuda"):+ return False+ try:+ import torch+ if not torch.cuda.is_initialized():+ return False+ ordinal = int(device.split(":", 1)[1]) if ":" in device else torch.cuda.current_device()+ except Exception: # noqa: BLE001+ return False+ free_mib, total_mib = _cuda_free_mib(ordinal)+ if free_mib is None:+ return False+ return vram_is_tight(free_mib - _BF16_RESIDENT_MIB, total_mib)++def _other_owner_on(ordinal: Optional[int]) -> Optional[str]:"""The GPU arbiter's resident other than the image model on CUDA card ``ordinal`` (one with unknown GPUs countsfor every card; ``ordinal`` None is Apple's single Metal device, which every resident shares), else None. Called under the run lock after the handover fence: a later handover waits for the@@ -1170,6 +1211,8 @@ class MaskSegmenter:self.weights_file: Optional[str] = None# Why the last pre-quantized load fell back to bf16, if one did.self.last_prequant_failure: Optional[str] = None+ # Reused while loaded: a reading taken with the model resident would flip auto's choice and reload it.+ self._memory_tight: Optional[bool] = None# Planning reads the Hub cache only for the active variant's own snapshot, not an explicit folder.self._use_cache = Trueself._timer: Optional[threading.Timer] = None@@ -1208,7 +1251,13 @@ class MaskSegmenter:# -- lifecycle -------------------------------------------------------------------------------------------------def _load(self, path: str, device: str, dtype: Any) -> None:- plan = plan_weights(path, device, dtype, use_cache = self._use_cache)+ if self._model is not None and self.device == device and self._memory_tight is not None:+ memory_tight = self._memory_tight+ else:+ memory_tight = auto_memory_tight(device)+ plan = plan_weights(+ path, device, dtype, use_cache = self._use_cache, memory_tight = memory_tight+ )key = (plan.model_dir, plan.weights, plan.dequantize)if (self._model is not None@@ -1238,6 +1287,7 @@ class MaskSegmenter:self._model, self._tokenizer = model, tokenizerself._path, self.device, self._dtype = path, device, dtypeself._weights_key = key+ self._memory_tight = memory_tightself.precision, self.precision_reason = plan.precision, plan.reasonself.weights_file = (os.path.basename(plan.weights) if plan.weights else SEGMENTER_WEIGHTS_FILE@@ -1484,6 +1534,7 @@ class MaskSegmenter:self._weights_key = Noneself.device = Noneself._dtype = None+ self._memory_tight = Noneself.precision = self.precision_reason = self.weights_file = None_free_device_cache(device)logger.info("images.segment: unloaded from %s (%s)", device, reason)diff --git a/studio/backend/core/inference/diffusion_segment_quant.py b/studio/backend/core/inference/diffusion_segment_quant.pyindex 78f1c8cb55a..9e3f5328bad 100644--- a/studio/backend/core/inference/diffusion_segment_quant.py+++ b/studio/backend/core/inference/diffusion_segment_quant.py@@ -43,7 +43,8 @@ ENV_PRECISION = "UNSLOTH_SAM3_PRECISION"# int8 loads the ConvRot artifact unless this is 0 (then the plain INT8 one).ENV_INT8_CONVROT = "UNSLOTH_SAM3_INT8_CONVROT"-# What ``auto`` may pick on a CUDA card, best first; it also needs that file's GATE_PASSED. Empty: auto stays bf16.+# What ``auto`` may pick on a CUDA card short of memory, best first (needs GATE_PASSED); empty: always bf16. With room+# auto runs bf16: at this size the 8-bit files save ~0.5 GB but are ~2x slower per Find (activation quant cost).AUTO_ORDER: tuple[str, ...] = (PRECISION_INT8,)# Minimum compute capability per scheme: fp8 scaled_mm needs Ada / Hopper (sm_89+), int8 _int_mm Ampere (sm_80+).@@ -471,12 +472,16 @@ def resolve_precision(dtype: Any = None,*,available: tuple[str, ...] = QUANT_PRECISIONS,+ memory_tight: bool = False,) -> tuple[str, str]:"""``(precision, why)`` for a run on ``device``. ``available`` is the quantized arms the variant ships (and, at- load, has on disk). bf16 when asked for, when nothing quantized can run here, or when auto has no gated arm."""+ load, has on disk). bf16 when asked for, when nothing quantized can run here, when auto has no gated arm, or for+ auto on CUDA unless ``memory_tight``. An explicit fp8 / int8 is honoured either way."""requested = requested_precision()if requested == PRECISION_BF16:return PRECISION_BF16, f"{ENV_PRECISION}=bf16"+ if requested == PRECISION_AUTO and not memory_tight and device.startswith("cuda"):+ return PRECISION_BF16, "auto: bf16 fits and is faster"candidates = AUTO_ORDER if requested == PRECISION_AUTO else (requested,)reasons = []for precision in candidates:diff --git a/studio/backend/tests/test_diffusion_segment_quant.py b/studio/backend/tests/test_diffusion_segment_quant.pyindex 9c8f212dcb0..f42085f3a5d 100644--- a/studio/backend/tests/test_diffusion_segment_quant.py+++ b/studio/backend/tests/test_diffusion_segment_quant.py@@ -54,20 +54,118 @@ def test_requested_precision(monkeypatch, value, expected):assert sq.requested_precision() == expected-def test_auto_follows_the_gated_order_on_cuda(monkeypatch):+def test_auto_follows_the_gated_order_on_a_tight_cuda_card(monkeypatch):import torch_cuda(monkeypatch, (9, 0))assert sq.AUTO_ORDER, "the gate passed at least one quantized arm"- assert sq.resolve_precision("cuda:0", torch.bfloat16)[0] == sq.AUTO_ORDER[0]+ tight = dict(memory_tight = True)+ assert sq.resolve_precision("cuda:0", torch.bfloat16, **tight)[0] == sq.AUTO_ORDER[0]# Only what is on disk (or published) counts.- assert sq.resolve_precision("cuda:0", available = ())[0] == "bf16"- assert sq.resolve_precision("cuda:0", available = (sq.AUTO_ORDER[-1],))[0] == sq.AUTO_ORDER[-1]+ assert sq.resolve_precision("cuda:0", available = (), **tight)[0] == "bf16"+ assert (+ sq.resolve_precision("cuda:0", available = (sq.AUTO_ORDER[-1],), **tight)[0]+ == sq.AUTO_ORDER[-1]+ )monkeypatch.setattr(sq, "AUTO_ORDER", ())- precision, why = sq.resolve_precision("cuda:0")+ precision, why = sq.resolve_precision("cuda:0", **tight)assert precision == "bf16" and "gate" in why+def test_auto_runs_bf16_while_the_card_has_room(monkeypatch):+ """The 8-bit files are about 2.2x slower per Find than bf16 (model card, B200), so auto only trades speed for+ memory when the card is short of it. An explicit int8 / fp8 is honoured either way."""+ import torch++ _cuda(monkeypatch, (9, 0))+ for args in ((), (torch.bfloat16,)):+ precision, why = sq.resolve_precision("cuda:0", *args)+ assert precision == "bf16" and "auto" in why+ assert sq.resolve_precision("cuda:0", *args, memory_tight = False)[0] == "bf16"+ for value in ("int8", "fp8"):+ monkeypatch.setenv(sq.ENV_PRECISION, value)+ assert sq.resolve_precision("cuda:0", torch.bfloat16)[0] == value+ assert sq.resolve_precision("cuda:0", torch.bfloat16, memory_tight = True)[0] == value+ monkeypatch.setenv(sq.ENV_PRECISION, "bf16")+ assert sq.resolve_precision("cuda:0", torch.bfloat16, memory_tight = True)[0] == "bf16"+++def test_auto_memory_tight_reads_the_card_only_once_cuda_is_up(monkeypatch):+ import torch++ reads = []++ def free(ordinal):+ reads.append(ordinal)+ return free_total++ monkeypatch.setattr(seg, "_cuda_free_mib", free)+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: False)+ free_total = (1024, 80_000)+ # No CUDA context in this process yet: nothing of ours is on a card, and planning must not create one.+ assert seg.auto_memory_tight("cuda:0") is False and reads == []+ assert seg.auto_memory_tight("cpu") is False and seg.auto_memory_tight("mps") is False+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: True)+ # 80 GB card: tight below 20% (16000 MiB) free once bf16's 2048 MiB are placed.+ free_total = (60_000, 80_000)+ assert seg.auto_memory_tight("cuda:1") is False and reads == [1]+ free_total = (16_000 + seg._BF16_RESIDENT_MIB - 1, 80_000)+ assert seg.auto_memory_tight("cuda:1") is True+ free_total = (16_000 + seg._BF16_RESIDENT_MIB, 80_000)+ assert seg.auto_memory_tight("cuda:1") is False+ # Small card: the 6 GiB floor.+ free_total = (seg._TIGHT_FREE_MIB + seg._BF16_RESIDENT_MIB - 1, 12_000)+ assert seg.auto_memory_tight("cuda:0") is True+ free_total = (None, None)+ assert seg.auto_memory_tight("cuda:0") is False+++def test_a_bf16_pick_is_not_unloaded_as_tight_after_its_first_find(monkeypatch):+ # mem_get_info free drop with bf16 resident after a Find, A100 + torch 2.11 (dequantized INT8 file, the larger).+ import torch++ measured_drop_mib = 2282+ monkeypatch.setattr(torch.cuda, "is_initialized", lambda: True)+ for total in (12_000, 24_000, 40_000, 80_000):+ floor = max(seg._TIGHT_FREE_MIB, int(total * seg._TIGHT_FREE_FRACTION))+ free = floor + seg._BF16_RESIDENT_MIB+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal, f = free, t = total: (f, t))+ assert seg.auto_memory_tight("cuda:0") is False+ assert not seg.vram_is_tight(free - measured_drop_mib, total), total+++def test_a_loaded_segmenter_keeps_its_memory_reading(monkeypatch):+ """Loaded on a card, the free memory read on the next call no longer counts the segmenter itself: the choice made+ at load is reused, so a bf16 model is not swapped for int8 (or back) on every call."""+ seen = []+ plan = seg.WeightsPlan("bf16", "/m")+ monkeypatch.setattr(+ seg, "plan_weights", lambda *a, memory_tight, **k: seen.append(memory_tight) or plan+ )+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ segmenter._model, segmenter._path, segmenter.device, segmenter._dtype = (+ object(),+ "/m",+ "cuda:0",+ "bf16",+ )+ segmenter._weights_key, segmenter._memory_tight = ("/m", None, False), False+ segmenter._load("/m", "cuda:0", "bf16")+ assert seen == [False]+ # Another device (or nothing loaded) reads the card again.+ segmenter._model = None+ segmenter._memory_tight = None+ monkeypatch.setattr(segmenter, "_load_weights", lambda plan, *a: (object(), plan))+ import transformers++ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: object())+ segmenter._load("/m", "cuda:0", "bf16")+ assert seen == [False, True] and segmenter._memory_tight is True+ segmenter._unload_locked("test")+ assert segmenter._memory_tight is None++def test_quantized_kernels_are_cuda_only(monkeypatch):import torch@@ -175,7 +273,10 @@ def test_download_plan_fetches_one_weights_file(monkeypatch):seg.SAM31_SIZE_BYTES,)variant = _published(monkeypatch)- plan = seg.download_plan(variant, "cuda:0")+ # A card with room fetches bf16 (the faster path); a tight one the smaller gated file.+ assert seg.download_plan(variant, "cuda:0")["spec"] == "sam3.1"+ assert seg.download_plan(variant, "cuda:0", memory_tight = False)["spec"] == "sam3.1"+ plan = seg.download_plan(variant, "cuda:0", memory_tight = True)artifact = seg.variant_artifact(variant, sq.AUTO_ORDER[0])assert (plan["spec"] == seg.prequant_spec_id(variant, artifact)@@ -184,10 +285,13 @@ def test_download_plan_fetches_one_weights_file(monkeypatch):assert plan["file"] == artifact.filenameassert plan["size_bytes"] == artifact.size_bytes + seg.SAM31_CONFIG_BYTES < seg.SAM31_SIZE_BYTES# CPU / Mac hosts fetch bf16.- assert seg.download_plan(variant, "cpu")["spec"] == "sam3.1"- assert seg.download_plan(variant, "mps")["spec"] == "sam3.1"+ assert seg.download_plan(variant, "cpu", memory_tight = True)["spec"] == "sam3.1"+ assert seg.download_plan(variant, "mps", memory_tight = True)["spec"] == "sam3.1"+ # An explicit int8 fetches the int8 file whatever the memory.+ monkeypatch.setenv(sq.ENV_PRECISION, "int8")+ assert seg.download_plan(variant, "cuda:0")["spec"] == seg.prequant_spec_id(variant, artifact)monkeypatch.setenv(sq.ENV_PRECISION, "bf16")- assert seg.download_plan(variant, "cuda:0")["spec"] == "sam3.1"+ assert seg.download_plan(variant, "cuda:0", memory_tight = True)["spec"] == "sam3.1"def test_start_download_uses_the_plan(monkeypatch):@@ -204,9 +308,12 @@ def test_start_download_uses_the_plan(monkeypatch):monkeypatch.setattr(seg, "_get_download_state", lambda: _State())seg.start_download(None, "sam3.1")- assert started == [- seg.prequant_spec_id(variant, seg.variant_artifact(variant, sq.AUTO_ORDER[0]))- ]+ assert started == ["sam3.1"]+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)+ seg.start_download(None, "sam3.1")+ assert started[-1] == seg.prequant_spec_id(+ variant, seg.variant_artifact(variant, sq.AUTO_ORDER[0])+ )# A spec id passes through untouched.seg.start_download(None, "sam3.1@fp8")assert started[-1] == "sam3.1@fp8"@@ -321,7 +428,12 @@ def test_plan_weights(monkeypatch, tmp_path):sam31 = seg.SEGMENTER_VARIANTS["sam3.1"]every = [a.filename for a in sam31.prequant]full = _folder(tmp_path / "full", "model.safetensors", *every)+ # auto with room: the bf16 file, though the quantized ones are on disk.plan = seg.plan_weights(full, "cuda:0", torch.bfloat16, variant = sam31, use_cache = False)+ assert (plan.precision, plan.weights, plan.dequantize) == ("bf16", None, False)+ plan = seg.plan_weights(+ full, "cuda:0", torch.bfloat16, variant = sam31, use_cache = False, memory_tight = True+ )first = seg.variant_artifact(sam31, sq.AUTO_ORDER[0])assert (plan.precision, plan.weights, plan.dequantize) == (sq.AUTO_ORDER[0],@@ -361,7 +473,10 @@ def test_only_a_quantized_file_on_a_cpu_is_dequantized(monkeypatch, tmp_path):and plan.dequantizeand plan.weights.endswith("INT8-ConvRot.safetensors"))+ # A card with room runs it dequantized (bf16 speed); a tight one runs it quantized.plan = seg.plan_weights(only, "cuda:0", variant = sam31, use_cache = False)+ assert plan.precision == "bf16" and plan.dequantize+ plan = seg.plan_weights(only, "cuda:0", variant = sam31, use_cache = False, memory_tight = True)assert plan.precision == "int8" and not plan.dequantize@@ -475,6 +590,9 @@ def test_status_reports_the_precision_and_its_size(monkeypatch):assert body["precision_requested"] == "auto"variant = _published(monkeypatch)body = seg.status()+ assert (body["precision"], body["weights_file"]) == ("bf16", "model.safetensors")+ monkeypatch.setattr(seg, "auto_memory_tight", lambda device: True)+ body = seg.status()artifact = seg.variant_artifact(variant, sq.AUTO_ORDER[0])assert (body["precision"], body["weights_file"]) == (sq.AUTO_ORDER[0], artifact.filename)assert body["size_bytes"] == artifact.size_bytes + seg.SAM31_CONFIG_BYTES@@ -553,7 +671,8 @@ def test_auto_skips_a_file_that_failed_the_gate(monkeypatch, tmp_path):sam31 = seg.SEGMENTER_VARIANTS["sam3.1"]fp8_only = _folder(tmp_path, "model.safetensors", "sam3.1-FP8.safetensors")monkeypatch.setattr(sq, "AUTO_ORDER", ("fp8",))- assert seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False).precision == "bf16"+ plan = seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False, memory_tight = True)+ assert plan.precision == "bf16"monkeypatch.setenv(sq.ENV_PRECISION, "fp8")assert seg.plan_weights(fp8_only, "cuda:0", variant = sam31, use_cache = False).precision == "fp8"