Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make inpaint and localized-edit masks from text with SAM 3 (#14235)
Daniel Hanbff0a25b4bfc
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.pynew file mode 100644index 00000000000..9e1e26b4395--- /dev/null+++ b/studio/backend/core/inference/diffusion_segment.py@@ -0,0 +1,1031 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Text-prompted masks for the Images page: SAM 3 (Segment Anything with Concepts) turns "sofa, red lamp, pillow:2"+into a white-on-black mask the inpaint and localized-edit painters load and keep editable.++Behaviour follows ComfyUI's ``SAM3_Detect`` node and its "Image Segmentation (SAM3)" blueprint+(comfy_extras/nodes_sam3.py, comfy/text_encoders/sam3_clip.py): comma-separated concepts, ``:N`` caps a concept's+detections (default 1), the image is stretched to 1008x1008 with bilinear resampling, a detection is kept when its+score > 0.5, the best ``N`` per concept are kept, and the union of their masks (mask logits upsampled bilinearly to+the source size, > 0) is the result. Two departures: the score is sigmoid(logit) * sigmoid(presence), as in Meta's+SAM 3 and transformers' post-processing (ComfyUI drops the presence term, and then a concept that is not in the+picture, "elephant" in a living room, masks the sofa at 0.72; with it every absent concept scores below 0.02 and no+present one changes), and ComfyUI's optional crop-and-refine pass through the tracker's SAM decoder is not run.++Weights are the official transformers-format ``facebook/sam3`` checkpoint (gated: the license is accepted on Hugging+Face). SAM 3.1 ships only an original-format multiplex checkpoint with no transformers integration, so it is not+loaded here; ``UNSLOTH_SAM3_PATH`` points at any local transformers-format SAM 3 directory instead.++The model is auxiliary to the image model, like the FLUX.3 prompt upsampler: loaded on the first request, kept for+a short keep-alive, unloaded right after a call when the accelerator is tight, and dropped whenever the GPU arbiter+hands the GPU to a load, evicts the image model, or training starts. It runs on the CPU while training holds the GPU+or when the accelerator has no room.+"""++from __future__ import annotations++import base64+import io+import json+import math+import os+import re+import sys+import threading+import time+from contextlib import contextmanager, suppress+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Iterator, Optional++from loggers import get_logger++logger = get_logger(__name__)++# ── weights ─────────────────────────────────────────────────────────────────────────────────────────────────────────+SEGMENTER_ID = "sam3"+SEGMENTER_REPO = "facebook/sam3"+SEGMENTER_LICENSE_URL = "https://huggingface.co/facebook/sam3"+SEGMENTER_WEIGHTS_FILE = "model.safetensors"+# Only what Sam3Model and its tokenizer read: the repo also holds sam3.pt, the same weights in the original format.+SEGMENTER_FILES = (+ "config.json",+ "processor_config.json",+ "tokenizer.json",+ "tokenizer_config.json",+ "special_tokens_map.json",+ "vocab.json",+ "merges.txt",+ SEGMENTER_WEIGHTS_FILE,+)+# fp32 safetensors as published (half of it once loaded in bf16 / fp16), plus the tokenizer and configs.+SEGMENTER_SIZE_BYTES = 3_439_938_512 + 5_060_000+ENV_PATH_OVERRIDE = "UNSLOTH_SAM3_PATH"+ENV_KEEP_ALIVE = "UNSLOTH_SAM3_KEEP_ALIVE"++# ── ComfyUI SAM3_Detect defaults ────────────────────────────────────────────────────────────────────────────────────+IMAGE_SIZE = 1008+DEFAULT_THRESHOLD = 0.5+MAX_CONCEPTS = 16+# The detector proposes 200 queries per concept.+MAX_DETECTIONS = 200+TEXT_MAX_LENGTH = 32++# ── memory ──────────────────────────────────────────────────────────────────────────────────────────────────────────+KEEP_ALIVE_SECONDS = 120.0+# bf16 weights (~1.7 GB) plus the 1008x1008 ViT activations and the allocator's slack.+_GPU_NEED_MIB = 4096+# After a call, below this much free (or this fraction of the card) the model leaves at once, so the image model+# never denoises next to it. The prompt upsampler's tight rule.+_TIGHT_FREE_MIB = 6144+_TIGHT_FREE_FRACTION = 0.20+++class SegmenterUnavailableError(RuntimeError):+ """This install cannot run SAM 3 (transformers without it)."""+++class SegmenterNotDownloadedError(RuntimeError):+ """The weights are not in the cache; the UI offers the download."""+++class SegmenterBusyError(RuntimeError):+ """The image model is rendering or loading; the GPU is not free for the segmenter."""+++class SegmenterReleaseTimeoutError(RuntimeError):+ """A GPU handover waited out a call still running on the accelerator: the card is not free yet."""+++# ── prompt syntax ───────────────────────────────────────────────────────────────────────────────────────────────────+@dataclass(frozen = True)+class Concept:+ text: str+ max_detections: int+++_MAX_SUFFIX = re.compile(r"^(.+?)\s*:\s*([\d.]+)\s*$")+++def parse_prompt(text: Optional[str]) -> list[Concept]:+ """ComfyUI's ``_parse_prompts`` (comfy/text_encoders/sam3_clip.py): drop parentheses, split on commas, and read a+ trailing ``:N`` as that concept's maximum detections (rounded, at least 1; 1 when absent). Duplicates are kept,+ as ComfyUI keeps them. Capped at MAX_CONCEPTS concepts and MAX_DETECTIONS per concept."""+ text = (text or "").replace("(", "").replace(")", "")+ parts = [p.strip() for p in text.split(",") if p.strip()]+ concepts: list[Concept] = []+ for part in parts:+ match = _MAX_SUFFIX.match(part)+ if match:+ label = match.group(1).strip()+ try:+ count = max(1, round(float(match.group(2))))+ except (ValueError, OverflowError): # "1.2.3", or hundreds of digits (float -> inf)+ label, count = part, 1+ else:+ label, count = part, 1+ if label:+ concepts.append(Concept(label, min(count, MAX_DETECTIONS)))+ if len(concepts) > MAX_CONCEPTS:+ raise ValueError(f"At most {MAX_CONCEPTS} concepts per mask (got {len(concepts)}).")+ return concepts+++# ── availability and files ──────────────────────────────────────────────────────────────────────────────────────────+def transformers_supports_sam3() -> bool:+ """Whether the installed transformers ships SAM 3 (5.x), read from the package layout: the status route must not+ import the model code, and an attribute probe on transformers' lazy module during another thread's import of it+ can read as missing."""+ try:+ import importlib.util++ # A --no-torch install ships transformers without torch: SAM 3 has the code there and still cannot run.+ if importlib.util.find_spec("torch") is None:+ return False+ return importlib.util.find_spec("transformers.models.sam3.modeling_sam3") is not None+ except Exception: # noqa: BLE001+ return False+++def _override_dir() -> Optional[str]:+ """The override directory when it holds what Sam3Model and its tokenizer load (a config, the weights, single or+ sharded, and a tokenizer), else None: a partial or unrelated folder must not read as downloaded."""+ value = (os.environ.get(ENV_PATH_OVERRIDE) or "").strip()+ if not value:+ return None+ path = Path(value).expanduser()+ has = lambda *names: any((path / name).is_file() for name in names) # noqa: E731+ complete = (+ has("config.json")+ and (has(SEGMENTER_WEIGHTS_FILE) or _shards_complete(path / (SEGMENTER_WEIGHTS_FILE + ".index.json")))+ # AutoTokenizer (CLIPTokenizer for sam3) loads tokenizer.json, else needs vocab.json and merges.txt together.+ and (has("tokenizer.json") or (has("vocab.json") and has("merges.txt")))+ )+ return str(path) if complete else None+++def _shards_complete(index: Path) -> bool:+ """A sharded checkpoint's index is present and every shard its weight_map names is beside it."""+ try:+ weight_map = json.loads(index.read_text(encoding = "utf-8")).get("weight_map")+ except Exception: # noqa: BLE001 -- missing or unreadable: not a checkpoint+ return False+ if not isinstance(weight_map, dict) or not weight_map:+ return False+ shards = set(weight_map.values())+ return all(isinstance(name, str) and (index.parent / name).is_file() for name in shards)+++def _cached_snapshot(+ model_id: str = SEGMENTER_ID,+ *,+ hub_cache: Optional[Path] = None,+ revision: Optional[str] = None,+) -> Optional[str]:+ """The snapshot directory holding every SEGMENTER_FILES at one revision (the recorded one first), else None."""+ try:+ from huggingface_hub import hf_hub_download++ from core.inference.stt_sidecar import (+ _active_hf_hub_cache,+ _fallback_revisions,+ _read_revision_record,+ )++ root = hub_cache if hub_cache is not None else _active_hf_hub_cache()+ if revision is not None:+ revisions = [revision]+ else:+ recorded = _read_revision_record(SEGMENTER_REPO)+ revisions = ([recorded] if recorded else []) + [+ rev+ for rev in _fallback_revisions(SEGMENTER_REPO, hub_cache = root)+ if rev != recorded+ ]+ for rev in revisions:+ paths = []+ for filename in SEGMENTER_FILES:+ try:+ paths.append(+ hf_hub_download(+ repo_id = SEGMENTER_REPO,+ filename = filename,+ revision = rev,+ local_files_only = True,+ cache_dir = str(root),+ )+ )+ except Exception: # noqa: BLE001 -- missing at this revision+ paths = []+ break+ if paths:+ return str(Path(paths[0]).parent)+ except Exception: # noqa: BLE001 -- an unreadable cache reads as not downloaded+ return None+ return None+++def model_dir() -> Optional[str]:+ """The env override when set (a typo'd override is not silently replaced by the cache), else the cached repo."""+ if (os.environ.get(ENV_PATH_OVERRIDE) or "").strip():+ return _override_dir()+ return _cached_snapshot()+++@dataclass(frozen = True)+class _SegmenterSpec:+ repo: str+ files: tuple+ label: str+ model_file: str = SEGMENTER_WEIGHTS_FILE+ mmproj_file: str = "config.json"+++SEGMENTER_SPEC = _SegmenterSpec(repo = SEGMENTER_REPO, files = SEGMENTER_FILES, label = "SAM 3 segmenter")+++def _resolve_segmenter_id(model_id: Optional[str]) -> str:+ if (model_id or SEGMENTER_ID) != SEGMENTER_ID:+ raise ValueError(f"Unknown segmenter '{model_id}'.")+ return SEGMENTER_ID+++def describe_download_error(_model_id: str, error: object) -> Optional[str]:+ """A gated-repo refusal names the license to accept; anything else keeps the generic message."""+ name = type(error).__name__+ text = str(error)+ if (+ name == "GatedRepoError"+ or "gated" in text.lower()+ or "403" in text+ or "401" in text+ or "awaiting a review" in text+ ):+ return (+ f"{SEGMENTER_REPO} is gated on Hugging Face. Accept the SAM 3 license at {SEGMENTER_LICENSE_URL} "+ "with your account, add a Hugging Face token in Settings, then download again."+ )+ return None+++_download_state_lock = threading.Lock()+_download_state = None+++def _cached_download_paths(+ model_id: str,+ *,+ hub_cache: Optional[Path] = None,+ revision: Optional[str] = None,+):+ path = _cached_snapshot(model_id, hub_cache = hub_cache, revision = revision)+ return None if path is None else (path, path)+++def _get_download_state():+ global _download_state+ with _download_state_lock:+ if _download_state is None:+ from core.inference.stt_mtmd_sidecar import _MtmdDownloadState+ _download_state = _MtmdDownloadState(+ specs = {SEGMENTER_ID: SEGMENTER_SPEC},+ cached_paths = _cached_download_paths,+ resolve = _resolve_segmenter_id,+ label = "SAM 3 segmenter",+ describe_error = describe_download_error,+ )+ return _download_state+++def start_download(hf_token: Optional[str] = None) -> str:+ return _get_download_state().start(SEGMENTER_ID, hf_token)+++def download_status() -> dict:+ return _get_download_state().status()+++def cancel_download(download_id: Optional[str] = None) -> bool:+ return _get_download_state().cancel(SEGMENTER_ID, download_id)+++def status() -> dict:+ """The status route body: can it run, is it downloaded, is it loaded (and where), and the download progress."""+ supported = transformers_supports_sam3()+ path = model_dir() if supported else None+ segmenter = _segmenter+ return {+ "supported": supported,+ "reason": None+ if supported+ else "Mask from text needs PyTorch and transformers 5 (Python 3.10 or newer); this install lacks one of them.",+ "downloaded": path is not None,+ "loaded": bool(segmenter is not None and segmenter.loaded),+ "device": segmenter.device if segmenter is not None and segmenter.loaded else None,+ "repo": SEGMENTER_REPO,+ "gated": True,+ "license_url": SEGMENTER_LICENSE_URL,+ "size_bytes": SEGMENTER_SIZE_BYTES,+ "override": bool((os.environ.get(ENV_PATH_OVERRIDE) or "").strip()),+ "download": download_status(),+ }+++# ── device choice ───────────────────────────────────────────────────────────────────────────────────────────────────+def _resident_cuda_ordinal() -> Optional[int]:+ """The CUDA card the resident image model sits on, else None."""+ try:+ from core.inference.diffusion_engine_router import get_active_diffusion_engine+ ordinal = getattr(+ getattr(get_active_diffusion_engine(), "_state", None), "placed_ordinal", None+ )+ return None if ordinal is None else int(ordinal)+ except Exception: # noqa: BLE001+ return None+++def _cuda_free_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:+ try:+ import torch+ free, total = torch.cuda.mem_get_info(ordinal)+ return int(free // (1024 * 1024)), int(total // (1024 * 1024))+ except Exception: # noqa: BLE001+ return None, None+++def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:+ if free_mib is None:+ return False+ floor = _TIGHT_FREE_MIB+ if total_mib:+ floor = max(floor, int(total_mib * _TIGHT_FREE_FRACTION))+ return free_mib < floor+++def _other_owner_on(ordinal: Optional[int]) -> Optional[str]:+ """The GPU arbiter's resident other than the image model on CUDA card ``ordinal`` (one with unknown GPUs counts+ for every card; ``ordinal`` None is Apple's single Metal device, which every resident shares), else None. Called under the run lock after the handover fence: a later handover waits for the+ call (or aborts) instead."""+ try:+ from core.inference import gpu_arbiter++ held = gpu_arbiter.residents()+ physical = None if ordinal is None else gpu_arbiter.physical_gpu_ids([ordinal])+ except Exception: # noqa: BLE001 -- no arbiter: nothing else is placed through it+ return None+ for owner, gpus in held.items():+ if owner == gpu_arbiter.DIFFUSION:+ continue+ if gpus is None or physical is None or gpus & physical:+ return owner+ return None+++def choose_device(allow_gpu: bool = True) -> tuple[str, Any]:+ """(torch device string, dtype): the image model's CUDA card (else the current one) in bf16 (fp16 where bf16 is+ not supported) when it has room for the segmenter, Apple's MPS in fp32, else the CPU in fp32."""+ import torch++ if allow_gpu and torch.cuda.is_available():+ ordinal = _resident_cuda_ordinal()+ if ordinal is None or ordinal >= torch.cuda.device_count():+ ordinal = torch.cuda.current_device()+ other = _other_owner_on(ordinal)+ if other is not None:+ # A chat / video / other model owns the card: it grows its working memory without knowing about SAM 3.+ logger.info("images.segment: cuda:%s is held by %s, running on the CPU", ordinal, other)+ return "cpu", torch.float32+ free_mib, _total = _cuda_free_mib(ordinal)+ if free_mib is None or free_mib >= _GPU_NEED_MIB:+ bf16 = False+ with suppress(Exception):+ # is_bf16_supported reads the CURRENT card; the segmenter runs on ``ordinal``.+ with torch.cuda.device(ordinal):+ bf16 = bool(torch.cuda.is_bf16_supported())+ return f"cuda:{ordinal}", torch.bfloat16 if bf16 else torch.float16+ logger.info("images.segment: %s MiB free on cuda:%s, running on the CPU", free_mib, ordinal)+ if (+ allow_gpu+ and getattr(getattr(torch, "backends", None), "mps", None) is not None+ and torch.backends.mps.is_available()+ ):+ other = _other_owner_on(None)+ if other is None:+ return "mps", torch.float32+ logger.info("images.segment: Metal is held by %s, running on the CPU", other)+ return "cpu", torch.float32+++# ── inference ───────────────────────────────────────────────────────────────────────────────────────────────────────+@dataclass+class Instance:+ concept: str+ score: float+ box: tuple[float, float, float, float]+++@dataclass+class SegmentResult:+ mask_png: bytes+ width: int+ height: int+ instances: list[Instance] = field(default_factory = list)+ concepts: list[dict] = field(default_factory = list)+ device: str = "cpu"+ elapsed_ms: float = 0.0++ def to_json(self) -> dict:+ return {+ "mask": "data:image/png;base64," + base64.b64encode(self.mask_png).decode("ascii"),+ "width": self.width,+ "height": self.height,+ "instances": [+ {+ "prompt": i.concept,+ "score": round(i.score, 4),+ "box": [round(v, 1) for v in i.box],+ }+ for i in self.instances+ ],+ "prompts": self.concepts,+ "device": self.device,+ "elapsed_ms": round(self.elapsed_ms, 1),+ }+++def model_image_size(model: Any) -> int:+ """The square input the checkpoint was built for (1008 for SAM 3)."""+ try:+ return int(model.config.vision_config.backbone_config.image_size)+ except Exception: # noqa: BLE001+ return IMAGE_SIZE+++def preprocess(+ image: Any,+ device: str,+ dtype: Any,+ size: int = IMAGE_SIZE,+) -> Any:+ """ComfyUI's input: RGB in [0, 1] stretched to 1008x1008 with bilinear interpolation (no crop, no antialias),+ then SAM 3's normalisation (mean 0.5, std 0.5)."""+ import numpy as np+ import torch+ import torch.nn.functional as F++ array = np.array(image.convert("RGB"), dtype = np.uint8)+ tensor = torch.from_numpy(array).to(device).permute(2, 0, 1).unsqueeze(0).float() / 255.0+ tensor = F.interpolate(tensor, size = (size, size), mode = "bilinear", align_corners = False)+ return ((tensor - 0.5) / 0.5).to(dtype)+++def select_detections(+ pred_logits: Any,+ pred_boxes: Any,+ pred_masks: Any,+ *,+ max_detections: int,+ threshold: float,+ presence_logit: Any = None,+) -> tuple[Any, Any, Any]:+ """ComfyUI's keep rule for one concept: score > threshold, best first, at most ``max_detections``, with the score+ sigmoid(logit) * sigmoid(presence) when the presence logit is given. Takes one image's (Q,), (Q, 4) normalised+ xyxy and (Q, h, w) logits; returns the kept scores, boxes, masks."""+ probs = pred_logits.float().sigmoid()+ if presence_logit is not None:+ probs = probs * presence_logit.float().sigmoid().reshape(-1)[0]+ keep = probs > threshold+ probs, boxes, masks = probs[keep], pred_boxes[keep], pred_masks[keep]+ order = probs.argsort(descending = True)[:max_detections]+ return probs[order], boxes[order], masks[order]+++def union_mask(masks: Any, height: int, width: int) -> Any:+ """(N, h, w) mask logits -> one (height, width) bool mask: bilinear upsample, > 0, any."""+ import torch+ import torch.nn.functional as F++ if masks is None or masks.shape[0] == 0:+ return torch.zeros(+ height, width, dtype = torch.bool, device = None if masks is None else masks.device+ )+ union = torch.zeros(height, width, dtype = torch.bool, device = masks.device)+ # One instance at a time keeps a 4K source to a single full-size float plane.+ for logits in masks:+ up = F.interpolate(+ logits[None, None].float(), size = (height, width), mode = "bilinear", align_corners = False+ )[0, 0]+ union |= up > 0+ return union+++def mask_to_png(mask: Any) -> bytes:+ from PIL import Image++ array = (mask.to("cpu").numpy().astype("uint8")) * 255+ buffer = io.BytesIO()+ Image.fromarray(array, mode = "L").save(buffer, format = "PNG", optimize = True)+ return buffer.getvalue()+++def _free_device_cache(device: Optional[str]) -> None:+ """Collect what nothing references any more, then hand ``device``'s cached blocks back to the driver."""+ import gc++ gc.collect()+ if device and device.startswith("cuda"):+ with suppress(Exception):+ import torch++ with torch.cuda.device(device):+ torch.cuda.empty_cache()+ elif device == "mps":+ with suppress(Exception):+ import torch++ torch.mps.empty_cache()+++class MaskSegmenter:+ """One lazily loaded Sam3Model + its tokenizer. ``_run_lock`` serialises loads, calls and unloads."""++ def __init__(self, keep_alive_seconds: Optional[float] = None) -> None:+ if keep_alive_seconds is None:+ try:+ keep_alive_seconds = float(os.environ.get(ENV_KEEP_ALIVE) or KEEP_ALIVE_SECONDS)+ except ValueError:+ keep_alive_seconds = KEEP_ALIVE_SECONDS+ self.keep_alive_seconds = max(0.0, keep_alive_seconds)+ self._run_lock = threading.Lock()+ self._state_lock = threading.Lock()+ self._model = None+ self._tokenizer = None+ self._path: Optional[str] = None+ self.device: Optional[str] = None+ self._dtype = None+ self._timer: Optional[threading.Timer] = None+ self._release_pending = False+ self._release_epoch = 0+ # Bumped by every GPU handover's release: a call that queued before one never runs on the accelerator.+ self._handover_epoch = 0+ # The device of the call in flight: "" until it is chosen, None when no call runs.+ self._call_device: Optional[str] = None+ # Set (under _state_lock) once the call in flight has read its release flag: a release that times out after+ # that is not honoured by the call, so it waits out the call's short tail itself.+ self._call_settled = False++ @property+ def loaded(self) -> bool:+ return self._model is not None++ def holds_vram(self) -> bool:+ return self._model is not None and bool(self.device) and self.device != "cpu"++ # -- lifecycle -------------------------------------------------------------------------------------------------+ def _load(self, path: str, device: str, dtype: Any) -> None:+ if (+ self._model is not None+ and self._path == path+ and self.device == device+ and self._dtype == dtype+ ):+ return+ if self._model is not None:+ self._unload_locked("reload")+ from transformers import AutoTokenizer, Sam3Model++ started = time.perf_counter()+ # The tokenizer first: a broken one fails before any weights are placed on the device.+ tokenizer = AutoTokenizer.from_pretrained(path)+ model = None+ try:+ if device.startswith("cuda"):+ # Straight onto the card: about a third faster than loading on the CPU and moving it.+ model = Sam3Model.from_pretrained(path, dtype = dtype, device_map = device)+ else:+ model = Sam3Model.from_pretrained(path, dtype = dtype)+ model.to(device)+ model.eval()+ except BaseException:+ # Weights placed before the failure are referenced nowhere else: drop them and their cached blocks.+ model = None+ _free_device_cache(device)+ raise+ self._model, self._tokenizer = model, tokenizer+ self._path, self.device, self._dtype = path, device, dtype+ logger.info(+ "images.segment: loaded %s on %s (%s) in %.1fs",+ path,+ device,+ str(dtype).replace("torch.", ""),+ time.perf_counter() - started,+ )++ def _unload_locked(self, reason: str) -> bool:+ with self._state_lock:+ if self._timer is not None:+ self._timer.cancel()+ self._timer = None+ if self._model is None:+ return False+ device = self.device+ self._model = None+ self._tokenizer = None+ self._path = None+ self.device = None+ self._dtype = None+ _free_device_cache(device)+ logger.info("images.segment: unloaded from %s (%s)", device, reason)+ return True+++ def _call_on_accelerator(self) -> bool:+ """A call is in flight on the GPU / MPS, or has not chosen its device yet."""+ device = self._call_device+ return device is not None and device != "cpu"++ def release(+ self,+ reason: str = "release",+ *,+ wait: bool = False,+ timeout: float = 30.0,+ handover: bool = False,+ ) -> bool:+ """Unload now if idle. A call in flight unloads when it ends (``wait`` waits up to ``timeout`` for it+ instead). True when the weights left now. ``handover`` (the GPU is changing hands) raises+ SegmenterReleaseTimeoutError when the wait ran out on a call still on the accelerator."""+ with self._state_lock:+ self._release_epoch += 1+ if handover:+ self._handover_epoch += 1+ acquired = (+ self._run_lock.acquire(timeout = timeout)+ if wait+ else self._run_lock.acquire(blocking = False)+ )+ if not acquired:+ with self._state_lock:+ self._release_pending = True+ # Read with the flag: a call that has not settled yet sees the flag and unloads when it ends.+ honoured = self._call_device is not None and not self._call_settled+ on_accelerator = self._call_on_accelerator()+ if wait and not honoured:+ # The call ended (or is ending) without seeing the flag: its tail is short, so wait it out and unload+ # here rather than leave the weights resident for the next owner.+ acquired = self._run_lock.acquire(timeout = _SETTLE_TIMEOUT_SECONDS)+ if not acquired:+ if handover and on_accelerator:+ raise SegmenterReleaseTimeoutError(+ "A mask from text is still running on the GPU; GPU ownership cannot be transferred yet."+ )+ return False+ try:+ with self._state_lock:+ self._release_pending = False+ return self._unload_locked(reason)+ finally:+ self._run_lock.release()++ def _arm_keep_alive(self) -> None:+ with self._state_lock:+ if self._timer is not None:+ self._timer.cancel()+ epoch = self._release_epoch+ timer = threading.Timer(self.keep_alive_seconds, self._expire, args = (epoch,))+ timer.daemon = True+ self._timer = timer+ timer.start()++ def _expire(self, epoch: int) -> None:+ if not self._run_lock.acquire(blocking = False):+ return # in use: the call re-arms the keep-alive when it ends+ try:+ with self._state_lock:+ if epoch != self._release_epoch and self._model is None:+ return+ self._unload_locked("keep-alive expired")+ finally:+ self._run_lock.release()++ def _after_call(self) -> None:+ """Called with ``_run_lock`` held: honour a release that arrived mid-call, drop a GPU copy when the card is+ tight, else keep it for the keep-alive."""+ with self._state_lock:+ pending, self._release_pending = self._release_pending, False+ self._call_settled = True+ if pending:+ self._unload_locked("released during a call")+ return+ if self.device and self.device.startswith("cuda"):+ free_mib, total_mib = _cuda_free_mib(int(self.device.split(":", 1)[1]))+ if vram_is_tight(free_mib, total_mib):+ self._unload_locked(f"tight: {free_mib} of {total_mib} MiB free")+ return+ if self.keep_alive_seconds <= 0:+ self._unload_locked("no keep-alive")+ return+ self._arm_keep_alive()++ # -- inference -------------------------------------------------------------------------------------------------+ def segment(+ self,+ image: Any,+ prompt: str,+ *,+ threshold: float = DEFAULT_THRESHOLD,+ allow_gpu: bool = True,+ path: Optional[str] = None,+ ) -> SegmentResult:+ concepts = parse_prompt(prompt)+ if not concepts:+ raise ValueError("Describe what to mask, for example: sofa, red lamp, pillow:2")+ if not 0.0 <= float(threshold) <= 1.0:+ raise ValueError("threshold must be between 0 and 1.")+ if not transformers_supports_sam3():+ raise SegmenterUnavailableError(+ "This install's transformers has no SAM 3. It needs transformers 5 (Python 3.10 or newer)."+ )+ path = path or model_dir()+ if path is None:+ raise SegmenterNotDownloadedError(+ f"The SAM 3 segmenter is not downloaded. Download {SEGMENTER_REPO} first."+ )+ queued_at = self._handover_epoch+ with self._run_lock:+ if allow_gpu and (self._handover_epoch != queued_at or _gpu_handover_in_progress()):+ # The GPU changed hands while this call waited, or is changing hands now: the new owner may still be+ # loading onto the card.+ raise SegmenterBusyError("The GPU is changing hands.")+ with self._state_lock:+ self._call_device, self._call_settled = "", False+ try:+ return self._segment_locked(image, concepts, threshold, path, allow_gpu)+ finally:+ with self._state_lock:+ self._call_device, self._call_settled = None, False++ def _segment_locked(+ self, image: Any, concepts: list[Concept], threshold: float, path: str, allow_gpu: bool+ ) -> SegmentResult:+ started = time.perf_counter()+ device, dtype = choose_device(allow_gpu)+ self._call_device = device+ oom = False+ try:+ result = self._segment_on(image, concepts, threshold, path, device, dtype)+ except Exception as exc:+ if not _is_oom(exc) or device == "cpu":+ self._after_call()+ raise+ oom = True+ if oom:+ # Out of the except block: the OOM's traceback (and the frames pinning the failed attempt's tensors) is+ # gone, so the unload and the cache release below actually free the card before the call reads CPU-only.+ logger.warning("images.segment: out of memory on %s, retrying on the CPU", device)+ self._unload_locked("out of memory")+ _free_device_cache(device) # also when the OOM hit mid-load, before any model was kept+ import torch++ device, dtype = "cpu", torch.float32+ self._call_device = device+ try:+ result = self._segment_on(image, concepts, threshold, path, device, dtype)+ except Exception:+ # The CPU copy and any release asked for mid-call are settled here too, not left resident.+ self._after_call()+ raise+ result.elapsed_ms = (time.perf_counter() - started) * 1000.0+ self._after_call()+ return result++ def _segment_on(+ self,+ image: Any,+ concepts: list[Concept],+ threshold: float,+ path: str,+ device: str,+ dtype: Any,+ ) -> SegmentResult:+ import torch++ self._load(path, device, dtype)+ model, tokenizer = self._model, self._tokenizer+ width, height = image.size+ scale = torch.tensor([width, height, width, height], dtype = torch.float32)+ union = torch.zeros(height, width, dtype = torch.bool, device = device)+ instances: list[Instance] = []+ summary: list[dict] = []+ with torch.inference_mode():+ pixels = preprocess(image, device, dtype, model_image_size(model))+ vision = model.get_vision_features(pixel_values = pixels)+ # Duplicates are encoded once; each concept keeps its own cap, as ComfyUI runs them one by one.+ texts = list(dict.fromkeys(c.text for c in concepts))+ tokens = tokenizer(+ texts,+ padding = "max_length",+ max_length = TEXT_MAX_LENGTH,+ truncation = True,+ return_tensors = "pt",+ ).to(device)+ # Only each text's kept detections (best first, up to its largest cap) outlive its forward, never the raw+ # output's 200 mask planes; a repeat with a smaller cap takes the best of them, as its own run would.+ caps: dict[str, int] = {}+ for concept in concepts:+ caps[concept.text] = max(caps.get(concept.text, 0), concept.max_detections)+ # Per text only its kept scores and boxes outlive the forward: the union of a text's best-``cap`` masks+ # covers every occurrence's best-``k`` (k <= cap), so the masks are OR-ed in at once and dropped.+ kept: dict[str, tuple[Any, Any]] = {}+ for concept in concepts:+ if concept.text not in kept:+ index = texts.index(concept.text)+ out = model(+ vision_embeds = vision,+ input_ids = tokens["input_ids"][index : index + 1],+ attention_mask = tokens["attention_mask"][index : index + 1],+ )+ presence = getattr(out, "presence_logits", None)+ scores, boxes, masks = select_detections(+ out.pred_logits[0],+ out.pred_boxes[0],+ out.pred_masks[0],+ max_detections = caps[concept.text],+ threshold = threshold,+ presence_logit = None if presence is None else presence[0],+ )+ del out, presence+ union |= union_mask(masks, height, width)+ del masks+ kept[concept.text] = (scores, boxes)+ scores, boxes = (t[: concept.max_detections] for t in kept[concept.text])+ boxes_px = (boxes.float().cpu() * scale).tolist()+ for score, box in zip(scores.tolist(), boxes_px):+ x1, y1, x2, y2 = (float(v) for v in box)+ instances.append(+ Instance(+ concept.text,+ float(score),+ (+ max(0.0, x1),+ max(0.0, y1),+ min(float(width), x2),+ min(float(height), y2),+ ),+ )+ )+ summary.append(+ {+ "text": concept.text,+ "max_detections": concept.max_detections,+ "found": int(scores.shape[0]),+ }+ )+ del kept, vision, pixels+ return SegmentResult(+ mask_png = mask_to_png(union),+ width = width,+ height = height,+ instances = instances,+ concepts = summary,+ device = device,+ )+++# How long a timed-out release waits for a call that already settled (unload / keep-alive, then the lock) to let go.+_SETTLE_TIMEOUT_SECONDS = 5.0+++def _gpu_handover_in_progress() -> bool:+ """The arbiter is handing the GPU to a new owner (its load still registering). Never imports the arbiter."""+ arbiter = sys.modules.get("core.inference.gpu_arbiter")+ probe = getattr(arbiter, "handover_in_progress", None)+ try:+ return bool(probe()) if callable(probe) else False+ except Exception: # noqa: BLE001+ return False+++def _is_oom(exc: BaseException) -> bool:+ name = type(exc).__name__+ return name in ("OutOfMemoryError",) or "out of memory" in str(exc).lower()+++_segmenter: Optional[MaskSegmenter] = None+_segmenter_lock = threading.Lock()+++def get_segmenter() -> MaskSegmenter:+ global _segmenter+ with _segmenter_lock:+ if _segmenter is None:+ _segmenter = MaskSegmenter()+ return _segmenter+++def release_segmenter(+ reason: str = "release",+ *,+ wait: bool = False,+ handover: bool = False,+) -> bool:+ """Drop the weights if they were ever loaded. Never creates the segmenter. ``handover``: see+ MaskSegmenter.release."""+ segmenter = _segmenter+ return segmenter is not None and segmenter.release(reason, wait = wait, handover = handover)+++_renders_pending = 0+_renders_lock = threading.Lock()+++@contextmanager+def render_fence() -> Iterator[None]:+ """Held by an image render from before it releases the segmenter until it ends: no Find starts meanwhile."""+ global _renders_pending+ with _renders_lock:+ _renders_pending += 1+ try:+ yield+ finally:+ with _renders_lock:+ _renders_pending -= 1+++def _take_render_slot() -> tuple[Any, str]:+ """(the active image engine's generation lock, acquired) or (None, why not). ``generating`` while a render or+ a model transition holds it, ``loading`` while a load is in flight; an engine with no lock (or none readable)+ runs nothing on torch, so the call proceeds with no slot. Never blocks."""+ try:+ from core.inference.diffusion_engine_router import get_active_diffusion_engine+ engine = get_active_diffusion_engine()+ except Exception: # noqa: BLE001 -- no engine: nothing to collide with+ return None, ""+ loading = getattr(engine, "_loading", None)+ if loading is not None and not getattr(loading, "error", None):+ return None, "loading"+ lock = getattr(engine, "_generate_lock", None)+ if lock is None:+ return None, ""+ if not lock.acquire(blocking = False):+ return None, "generating"+ loading = getattr(engine, "_loading", None)+ if loading is not None and not getattr(loading, "error", None):+ lock.release()+ return None, "loading"+ return lock, ""+++def segment_image(+ image_b64: str,+ prompt: str,+ *,+ threshold: float = DEFAULT_THRESHOLD,+ allow_gpu: bool = True,+) -> dict:+ """The segment route's work: decode the upload (EXIF-upright, as inpaint decodes it), refuse while the image+ model renders or loads, run SAM 3, answer the route body."""+ from core.inference.diffusion import decode_b64_image++ if not parse_prompt(prompt):+ raise ValueError("Describe what to mask, for example: sofa, red lamp, pillow:2")+ image = decode_b64_image(image_b64, mode = "RGB")+ slot, why = _take_render_slot()+ if not why and _renders_pending:+ # A render released SAM 3 and is on its way to the slot: a Find taking it first would reload SAM 3 under it.+ if slot is not None:+ slot.release()+ slot, why = None, "generating"+ if why:+ raise SegmenterBusyError(+ "The image model is generating."+ if why == "generating"+ else "An image model is loading."+ )+ try:+ return (+ get_segmenter()+ .segment(image, prompt, threshold = threshold, allow_gpu = allow_gpu)+ .to_json()+ )+ finally:+ if slot is not None:+ slot.release()+++__all__ = [+ "Concept",+ "DEFAULT_THRESHOLD",+ "MaskSegmenter",+ "SEGMENTER_REPO",+ "SegmenterBusyError",+ "SegmenterNotDownloadedError",+ "SegmenterReleaseTimeoutError",+ "SegmenterUnavailableError",+ "get_segmenter",+ "parse_prompt",+ "release_segmenter",+ "segment_image",+ "status",+]diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex 9ad0c200a6b..b5a5471738f 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -19,8 +19,9 @@ from __future__ import annotationsimport sysimport threading+from contextlib import contextmanager, nullcontextfrom contextvars import ContextVar-from typing import Any, Callable, Iterable, Optional+from typing import Any, Callable, Iterable, Iterator, Optionalfrom loggers import get_logger@@ -191,6 +192,7 @@ def _evict_diffusion() -> None:_cancel_prompt_upsampler_warm_start()get_active_diffusion_engine().unload()_stop_prompt_upsampler()+ _release_mask_segmenter("image model evicted", handover = True)def _cancel_prompt_upsampler_warm_start() -> None:@@ -215,6 +217,59 @@ def _stop_prompt_upsampler() -> None:logger.warning("gpu_arbiter: could not stop the FLUX.3 prompt upsampler: %s", exc)+class SegmenterStillRunningError(RuntimeError):+ """A mask from text still runs on the GPU after the release's wait: the GPU cannot change hands yet.+ ``blocks_training`` makes the core trainer abort its start on it rather than log and continue."""++ blocks_training = True+++def _release_mask_segmenter(reason: str, *, handover: bool = False) -> None:+ """The Images text-to-mask segmenter (SAM 3) is auxiliary: it leaves whenever the GPU changes hands, an image model+ is evicted or released, or training starts. Only if Images ever loaded it; a call in flight finishes first.+ ``handover``: a call still on the GPU after the wait aborts the transfer, as an engine that did not stop does."""+ module = sys.modules.get("core.inference.diffusion_segment")+ if module is None:+ return+ try:+ module.release_segmenter(reason, wait = True, handover = handover)+ except Exception as exc: # noqa: BLE001 - never block an acquire on anything else+ if handover and type(exc).__name__ == "SegmenterReleaseTimeoutError":+ raise SegmenterStillRunningError(+ "A mask from text is still running on the GPU; GPU ownership cannot be transferred."+ ) from exc+ logger.warning("gpu_arbiter: could not release the mask segmenter: %s", exc)+++_handovers = 0+++@contextmanager+def _handover_fence() -> Iterator[None]:+ global _handovers+ _handovers += 1+ try:+ yield+ finally:+ _handovers -= 1+++def handover_in_progress() -> bool:+ """An acquire is handing the GPU to a new owner (evicting, or its load registering). Lock-free read."""+ return _handovers > 0+++def release_mask_segmenter(reason: str) -> None:+ """Release the text-to-mask segmenter for a training start: an in-flight call finishes first, and one still on the+ GPU after the wait raises SegmenterStillRunningError so the start is refused instead of spawning beside it."""+ try:+ _release_mask_segmenter(reason, handover = True)+ except SegmenterStillRunningError as exc:+ raise SegmenterStillRunningError(+ "A mask from text is still running on the GPU; try again when it finishes."+ ) from exc++def _release_idle_video_servers(reason: str) -> None:try:from core.inference.video_minimax_h3 import release_h3_native_servers@@ -616,7 +671,9 @@ def acquire_for(from utils.account_context import current_account_idacting = account_id or current_account_id()- with _lock:+ # The segmenter's VRAM goes to this owner: a mask call arriving before its register returns stays off the card.+ handover = owner != DIFFUSION or register is not None or replacing+ with _lock, (_handover_fence() if handover else nullcontext()):if expected_current is not None and (_owner, _owner_epoch) != expected_current:raise OwnerChangedError("The resident GPU model changed; retry the load.")coreside = policy != "off" and owner in _KEPT_UNDER_ALWAYS@@ -672,6 +729,9 @@ def acquire_for(if owner != VIDEO and not (coreside and VIDEO in _residents):# A resident H3 sd-server sits outside every owner's teardown._release_idle_video_servers(f"GPU acquired for {owner}")+ if owner != DIFFUSION or register is not None or replacing:+ # A load (or another kind taking the GPU) gets the segmenter's VRAM; an image model's re-assert keeps it.+ _release_mask_segmenter(f"GPU acquired for {owner}", handover = True)# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.# A resident parked behind a later claimant is a re-assert, not a new load.rejoins = prev_owner != owner and owner in _residents and parked@@ -820,6 +880,9 @@ def release(owner: str) -> None:with _lock:if owner == DIFFUSION and owner in _view():_stop_prompt_upsampler()+ if owner == DIFFUSION:+ # Training releases DIFFUSION before it starts, resident or not: the segmenter goes too.+ _release_mask_segmenter("image GPU released")_drop_resident(owner)@@ -835,6 +898,7 @@ def release_if(owner: str, predicate: Callable[[], bool]) -> bool:return Falseif owner == DIFFUSION:_stop_prompt_upsampler()+ _release_mask_segmenter("image GPU released")_drop_resident(owner)return Truediff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.pyindex ab6e113a839..6b10ef4dfdf 100644--- a/studio/backend/core/inference/stt_mtmd_sidecar.py+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py@@ -317,7 +317,8 @@ class _MtmdDownloadState:"""Tracks the two-file download of one mtmd dictation model.``specs`` / ``cached_paths`` / ``resolve`` let another llama.cpp sidecar (the FLUX.3 prompt upsampler) reuse the- same pinned two-file download; None keeps the dictation catalogue, looked up at call time.+ same pinned two-file download; None keeps the dictation catalogue, looked up at call time. A spec with a ``files``+ tuple downloads those files instead of its model and projector (the Images text-to-mask segmenter)."""def __init__(@@ -326,11 +327,14 @@ class _MtmdDownloadState:cached_paths = None,resolve = None,label: str = "mtmd STT",+ describe_error = None,) -> None:self._specs = specsself._cached_paths_fn = cached_pathsself._resolve_fn = resolveself._label = label+ # (model_id, exception or worker stderr) -> a message for the UI, or None for the generic one.+ self._describe_error_fn = describe_errorself._lock = threading.Lock()self._thread: Optional[threading.Thread] = Noneself._process: Optional[subprocess.Popen] = None@@ -350,6 +354,14 @@ class _MtmdDownloadState:def _resolve(self, model_id: Optional[str]) -> str:return (self._resolve_fn or resolve_mtmd_model_id)(model_id)+ def _describe_error(self, model_id: str, error: object) -> Optional[str]:+ if self._describe_error_fn is None:+ return None+ try:+ return self._describe_error_fn(model_id, error)+ except Exception: # noqa: BLE001 -- the generic message still reaches the UI+ return None+def _cached_paths(self, model_id: str, **kwargs):return (self._cached_paths_fn or _cached_model_paths)(model_id, **kwargs)@@ -496,7 +508,7 @@ class _MtmdDownloadState:selected: list[_SelectedHubFile] = []revision: Optional[str] = None- for filename in (spec.model_file, spec.mmproj_file):+ for filename in getattr(spec, "files", None) or (spec.model_file, spec.mmproj_file):meta = get_hf_file_metadata(hf_hub_url(spec.repo, filename, revision = revision),token = normalize_token(hf_token),@@ -573,12 +585,20 @@ class _MtmdDownloadState:detail = stderr.decode("utf-8", "replace").strip()logger.warning("%s download failed for %s: %s", self._label, model_id, detail)with self._lock:- self._error = modelscope_missing(detail) or f"Download failed for '{model_id}'."+ self._error = (+ self._describe_error(model_id, detail)+ or modelscope_missing(detail)+ or f"Download failed for '{model_id}'."+ )except Exception as exc:with self._lock:if not self._cancelled:logger.warning("%s download failed for %s: %s", self._label, model_id, exc)- self._error = modelscope_missing(exc) or f"Download failed for '{model_id}'."+ self._error = (+ self._describe_error(model_id, exc)+ or modelscope_missing(exc)+ or f"Download failed for '{model_id}'."+ )finally:# Release first: it is the half that would wedge the repository.if registry is not None and owner is not None:diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex e324faacbaf..547e72a8ca7 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -5308,6 +5308,38 @@ class PromptUpsamplerDownloadRequest(BaseModel):hf_token: Optional[str] = Field(None, description = "Hugging Face token for the download")+class MaskSegmenterDownloadRequest(BaseModel):+ """Body of POST /images/segment/download (optional)."""++ hf_token: Optional[str] = Field(+ None, description = "Hugging Face token for the gated facebook/sam3 download"+ )+++class MaskSegmentRequest(BaseModel):+ """Make an inpaint / localized-edit mask from text with SAM 3."""++ image: str = Field(+ ...,+ min_length = 1,+ max_length = 32 * 1024 * 1024,+ description = "Base64/data-URL source image (as init_image on /images/generate)",+ )+ prompt: str = Field(+ ...,+ min_length = 1,+ max_length = 2000,+ description = "Comma-separated concepts; a trailing :N keeps up to N of that concept (default 1), "+ "e.g. 'sofa, red lamp, pillow:2'",+ )+ threshold: float = Field(+ 0.5,+ ge = 0.0,+ le = 1.0,+ description = "Detection score threshold (ComfyUI SAM3 Detect's default 0.5)",+ )++class PromptUpsamplerPreviewRequest(BaseModel):"""Enhance one prompt with the FLUX.3 prompt upsampler without generating."""diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 33dba74d875..d283b534462 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3796,6 +3796,8 @@ from models.inference import (DiffusionLoadRequest,DiffusionGenerateRequest,DiffusionGenerateResponse,+ MaskSegmentRequest,+ MaskSegmenterDownloadRequest,PromptUpsamplerDownloadRequest,PromptUpsamplerPreviewRequest,PromptUpsamplerPreviewResponse,@@ -51703,6 +51705,17 @@ async def _generate_diffusion_image_core(request: DiffusionGenerateRequest,) -> DiffusionGenerateResponse:"""The local-engine image run, shared by the route and the chat generate_image tool."""+ from core.inference.diffusion_segment import render_fence++ # From before the run releases the mask segmenter until it ends: a Find cannot reload SAM 3 in the gap before the+ # render takes the engine's generation slot.+ with render_fence():+ return await _generate_diffusion_image_run(request)+++async def _generate_diffusion_image_run(+ request: DiffusionGenerateRequest,+) -> DiffusionGenerateResponse:from core.inference import image_galleryfrom core.inference.diffusion_engine_router import get_active_diffusion_enginefrom core.inference.diffusion_families import (@@ -51738,6 +51751,8 @@ async def _generate_diffusion_image_core(# A warm start still launching holds the engine's generation slot: a run that is not using the upsampler does# not wait it out._cancel_upsampler_warm_start()+ # As the upsampler leaves before the DiT denoises, so does the mask segmenter (Find's SAM 3 on its keep-alive).+ await asyncio.to_thread(_release_mask_segmenter_for_render)result = Nonefor attempt in range(2):request = submitted@@ -52971,6 +52986,17 @@ def _cancel_upsampler_warm_start() -> None:logger.warning("flux3.prompt_upsampler: could not cancel a warm start: %s", exc)+def _release_mask_segmenter_for_render() -> None:+ """Drop the text-to-mask segmenter's weights before a render: a resident image model renders without an arbiter+ acquire, so no handover would unload it. Only if Images ever loaded it; a Find in flight finishes first."""+ module = sys.modules.get("core.inference.diffusion_segment")+ if module is not None:+ try:+ module.release_segmenter("image generation", wait = True)+ except Exception as exc: # noqa: BLE001 -- never fail a render on this+ logger.warning("images.segment: could not release before a render: %s", exc)++@studio_router.post("/images/prompt-upsampler/release")async def prompt_upsampler_release(current_subject: str = Depends(get_current_subject)):"""Enhance prompt was turned off: cancel a warm start still launching and stop an idle server instead of holding it@@ -53065,6 +53091,112 @@ async def prompt_upsampler_preview(return PromptUpsamplerPreviewResponse(enhanced_prompt = caption)+# ── Text-to-mask segmenter (SAM 3) ───────────────────────────────────────────────────────────────────────────────────+++@studio_router.get("/images/segment/status")+async def mask_segmenter_status(current_subject: str = Depends(get_current_subject)):+ """Whether SAM 3 can run here, is downloaded (gated facebook/sam3), is loaded and where, and the download+ progress (the dictation download's shape)."""+ from core.inference import diffusion_segment+ return JSONResponse(content = await asyncio.to_thread(diffusion_segment.status))+++@studio_router.post("/images/segment/download")+async def mask_segmenter_download(+ payload: Optional[MaskSegmenterDownloadRequest] = None,+ current_subject: str = Depends(get_current_subject),+ hf_token: HfTokenArg = Depends(get_request_hf_token),+):+ """Start the background download of facebook/sam3's transformers weights; progress on /status."""+ from core.inference import diffusion_segment+ from core.inference.stt_sidecar import SttDownloadCacheError, SttModelIdError++ account_access.require_installation_owner()+ if not diffusion_segment.transformers_supports_sam3():+ raise HTTPException(status_code = 501, detail = diffusion_segment.status()["reason"])+ if diffusion_segment.status().get("override"):+ raise HTTPException(+ status_code = 409,+ detail = "The segmenter weights come from UNSLOTH_SAM3_PATH; unset it to download facebook/sam3.",+ )+ token = account_access.account_hf_token(+ (payload.hf_token if payload is not None and payload.hf_token else None) or hf_token+ )+ try:+ download_id = await asyncio.to_thread(diffusion_segment.start_download, token)+ except (SttModelIdError, SttDownloadCacheError) as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ return JSONResponse(content = {**diffusion_segment.download_status(), "download_id": download_id})+++@studio_router.post("/images/segment/cancel")+async def mask_segmenter_cancel(current_subject: str = Depends(get_current_subject)):+ """Stop the segmenter download; partial files stay cached, so the next download resumes."""+ from core.inference import diffusion_segment++ account_access.require_installation_owner()+ cancelled = await asyncio.to_thread(diffusion_segment.cancel_download)+ return JSONResponse(content = {**diffusion_segment.download_status(), "cancelled": cancelled})+++@studio_router.post("/images/segment/release")+async def mask_segmenter_release(current_subject: str = Depends(get_current_subject)):+ """The mask painter closed: drop the segmenter now instead of holding it for the keep-alive. A call in flight+ unloads when it ends. Never an error."""+ from core.inference import diffusion_segment+ from auth import policy++ if policy.installation_has_managed_accounts():+ return {"released": False}+ released = await asyncio.to_thread(diffusion_segment.release_segmenter, "mask painter closed")+ return {"released": bool(released)}+++@studio_router.post("/images/segment")+@account_access.gpu_busy_route+async def mask_segment(+ request: MaskSegmentRequest, current_subject: str = Depends(get_current_subject)+):+ """A white-on-black mask (PNG data URL, source size) of what ``prompt`` names in ``image``, plus each kept+ instance's box and score. 409 when not downloaded or the image model is busy, 501 without SAM 3 support."""+ from core.inference import diffusion_segment as seg++ account_access.require_idle_other_accounts()+ # Training holds the GPU: the segmenter runs on the CPU rather than beside it. On the GPU the diffusion trainer's+ # admission is held for the call, so a start cannot free the GPU under it.+ allow_gpu = not _training_is_active()+ admission = _diffusion_training_admission() if allow_gpu else nullcontext()+ try:+ with account_access.media_generation("diffusion"), admission:+ body = await asyncio.to_thread(+ functools.partial(+ seg.segment_image,+ request.image,+ request.prompt,+ threshold = request.threshold,+ allow_gpu = allow_gpu,+ )+ )+ except seg.SegmenterNotDownloadedError as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ except seg.SegmenterUnavailableError as exc:+ raise HTTPException(status_code = 501, detail = str(exc))+ except seg.SegmenterBusyError as exc:+ raise HTTPException(status_code = 409, detail = f"{exc} Try again when it finishes.")+ except ValueError as exc:+ raise HTTPException(status_code = 400, detail = str(exc))+ except HTTPException:+ raise+ except Exception as exc: # noqa: BLE001+ if type(exc).__name__ == "TrainingActiveError":+ # A diffusion training start won the race for the admission.+ raise HTTPException(status_code = 409, detail = str(exc))+ logger.error("images.segment_failed: %s", exc, exc_info = True)+ raise HTTPException(status_code = 500, detail = "Making the mask failed.")+ return JSONResponse(content = body)++# ──────────────────────────────────────────────────────────────────────────# OpenAI-compatible images API (POST /v1/images/generations). The inference router is mounted at both /api/inference and /v1, so thisdiff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex fc84de953d2..c379da6a4bf 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -2437,6 +2437,8 @@ async def start_training(# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the diffusers backend# reports unloaded while the native engine still holds state.+ # The mask segmenter sits on torch's card whichever engine renders; one still running refuses the start.+ gpu_arbiter.release_mask_segmenter("training start")diffusion = get_active_diffusion_engine()if getattr(diffusion, "runs_off_torch_device", False) is True:logger.info("Keeping the Images model: it runs outside torch's GPUs")@@ -2447,6 +2449,8 @@ async def start_training()diffusion.unload()gpu_arbiter.release(gpu_arbiter.DIFFUSION)+ except gpu_arbiter.SegmenterStillRunningError:+ raiseexcept Exception as e:logger.warning("Could not unload diffusion model for training: %s", e)@@ -2502,6 +2506,7 @@ async def start_training(logger.warning("Inference/training memory coordination failed; proceeding: %s", e)# The hook runs only once start guards pass -> VRAM freed iff training starts.+ from core.inference.gpu_arbiter import SegmenterStillRunningErrorfrom routes.training_vram import ManagedEngineStillRunningfrom utils.transformers_version import SidecarSwapInProgress@@ -2518,6 +2523,7 @@ async def start_training(SidecarSwapInProgress,ExactResumeResourcesUnavailable,ManagedEngineStillRunning,+ SegmenterStillRunningError,) as exc:_reject_start_request(backend, reserved_start_request_id, str(exc))raise@@ -2575,7 +2581,11 @@ async def start_training(except SidecarSwapInProgress as exc:# Expected loss of the race against a sidecar install: a retryable 409, not an internal error.raise HTTPException(status_code = 409, detail = str(exc))- except (ExactResumeResourcesUnavailable, ManagedEngineStillRunning) as exc:+ except (+ ExactResumeResourcesUnavailable,+ ManagedEngineStillRunning,+ SegmenterStillRunningError,+ ) as exc:raise HTTPException(status_code = 409, detail = str(exc))if not success:@@ -3479,6 +3489,8 @@ def _free_gpu_for_diffusion_training() -> None:# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the resident sd-server still# holds the GPU, so unloading only the singleton is a no-op.+ # The mask segmenter sits on torch's card whichever engine renders; one still running refuses the start.+ gpu_arbiter.release_mask_segmenter("diffusion training start")diffusion = get_active_diffusion_engine()if getattr(diffusion, "runs_off_torch_device", False) is True:logger.info("Keeping the Images model: it runs outside torch's GPUs")@@ -3487,6 +3499,8 @@ def _free_gpu_for_diffusion_training() -> None:logger.info("Unloading resident Images pipeline to free GPU memory for training")diffusion.unload()gpu_arbiter.release(gpu_arbiter.DIFFUSION)+ except gpu_arbiter.SegmenterStillRunningError:+ raise # a RuntimeError: the start route answers 409except Exception as e: # noqa: BLE001logger.warning("Could not unload Images pipeline for diffusion training: %s", e)diff --git a/studio/backend/tests/test_diffusion_segment.py b/studio/backend/tests/test_diffusion_segment.pynew file mode 100644index 00000000000..7a3f911c572--- /dev/null+++ b/studio/backend/tests/test_diffusion_segment.py@@ -0,0 +1,1334 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Text-to-mask (SAM 3) for the Images painters: ComfyUI's prompt syntax, the keep rule, the union mask at the+source size, the lifecycle (keep-alive, release, GPU arbiter hooks) and the routes. A tiny random-weight Sam3Model+built from a shrunk config runs the real forward on the CPU; no download and no network."""++from __future__ import annotations++import base64+import io+import threading+import time+import types++import pytest++from core.inference import diffusion_segment as seg+++def _png_b64(+ width: int = 64,+ height: int = 48,+ color = (200, 30, 30),+) -> str:+ from PIL import Image++ buffer = io.BytesIO()+ Image.new("RGB", (width, height), color).save(buffer, format = "PNG")+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()+++def _decode_mask(data_url: str):+ from PIL import Image+ raw = base64.b64decode(data_url.split(",", 1)[1])+ return Image.open(io.BytesIO(raw))+++# ── prompt syntax (comfy/text_encoders/sam3_clip.py _parse_prompts) ─────────────────────────────────────────────────+ "text, expected",+ [+ ("Sofa:2", [("Sofa", 2)]),+ ("sofa", [("sofa", 1)]),+ (+ "Sofa:2, Red lamp, Window, Pillow:3",+ [("Sofa", 2), ("Red lamp", 1), ("Window", 1), ("Pillow", 3)],+ ),+ ("(red lamp), window", [("red lamp", 1), ("window", 1)]),+ ("a : 4 ", [("a", 4)]),+ ("a:2.6", [("a", 3)]),+ # Python's round(): half to even, as ComfyUI computes it.+ ("a:2.5, b:3.5", [("a", 2), ("b", 4)]),+ ("a:0", [("a", 1)]),+ (" , sofa,, ,", [("sofa", 1)]),+ ("note: red", [("note: red", 1)]),+ ("v:2.5.1", [("v:2.5.1", 1)]),+ ("grain:5000", [("grain", seg.MAX_DETECTIONS)]),+ ("", []),+ (None, []),+ ("sofa, sofa:3", [("sofa", 1), ("sofa", 3)]),+ ],+)+def test_parse_prompt_matches_comfyui(text, expected):+ assert [(c.text, c.max_detections) for c in seg.parse_prompt(text)] == expected+++def test_parse_prompt_overflowing_count_is_a_malformed_suffix():+ # float() of hundreds of digits is inf and round(inf) raises OverflowError: read it like "v:2.5.1".+ huge = "9" * 400+ assert [(c.text, c.max_detections) for c in seg.parse_prompt(f"object:{huge}, cat")] == [+ (f"object:{huge}", 1),+ ("cat", 1),+ ]+++def test_parse_prompt_caps_the_concept_count():+ with pytest.raises(ValueError, match = "At most"):+ seg.parse_prompt(",".join(f"thing{i}" for i in range(seg.MAX_CONCEPTS + 1)))+++# ── keep rule and mask ──────────────────────────────────────────────────────────────────────────────────────────────+def test_select_detections_thresholds_sorts_and_caps():+ import torch++ logits = torch.tensor([0.0, 3.0, -3.0, 2.0, 4.0]) # sigmoid: .5, .95, .05, .88, .98+ boxes = torch.arange(20, dtype = torch.float32).reshape(5, 4)+ masks = torch.arange(5, dtype = torch.float32)[:, None, None].expand(5, 2, 2).clone()+ scores, kept_boxes, kept_masks = seg.select_detections(+ logits, boxes, masks, max_detections = 2, threshold = 0.5+ )+ # 0.5 itself is not above the threshold; best first; at most 2.+ assert [round(v, 2) for v in scores.tolist()] == [0.98, 0.95]+ assert kept_boxes[:, 0].tolist() == [16.0, 4.0]+ assert kept_masks[:, 0, 0].tolist() == [4.0, 1.0]+++def test_presence_suppresses_an_absent_concept():+ import torch++ logits = torch.tensor([1.0, 0.2]) # 0.73 and 0.55 alone: both kept without presence+ boxes = torch.zeros(2, 4)+ masks = torch.zeros(2, 2, 2)+ scores, _, _ = seg.select_detections(logits, boxes, masks, max_detections = 5, threshold = 0.5)+ assert scores.shape[0] == 2+ absent = torch.tensor([-7.0]) # sigmoid ~ 0.001+ scores, _, _ = seg.select_detections(+ logits, boxes, masks, max_detections = 5, threshold = 0.5, presence_logit = absent+ )+ assert scores.shape[0] == 0+ present = torch.tensor([7.0])+ scores, _, _ = seg.select_detections(+ logits, boxes, masks, max_detections = 5, threshold = 0.5, presence_logit = present+ )+ assert scores.shape[0] == 2+++def test_union_mask_upsamples_to_the_source_size():+ import torch++ left = torch.full((1, 4, 4), -5.0)+ left[0, :, :2] = 5.0+ bottom = torch.full((1, 4, 4), -5.0)+ bottom[0, 3:, :] = 5.0+ union = seg.union_mask(torch.cat([left, bottom]), 30, 40)+ assert tuple(union.shape) == (30, 40)+ assert union[0, 0] and union[29, 39] and not union[0, 39]+ assert not seg.union_mask(torch.zeros(0, 4, 4), 7, 9).any()+ # Nothing kept: the empty union stays on the masks' device (a CUDA run ORs it into a CUDA union).+ empty = seg.union_mask(torch.zeros(0, 4, 4, device = "meta"), 7, 9)+ assert empty.device.type == "meta" and tuple(empty.shape) == (7, 9)+ png = seg.mask_to_png(union)+ from PIL import Image++ image = Image.open(io.BytesIO(png))+ assert image.mode == "L" and image.size == (40, 30)+ assert set(image.histogram()[1:255]) == {0}+++def test_preprocess_is_comfyui_bilinear_1008_normalised():+ import torch+ from PIL import Image++ image = Image.new("RGB", (37, 23), (255, 0, 128))+ tensor = seg.preprocess(image, "cpu", torch.float32)+ assert tuple(tensor.shape) == (1, 3, seg.IMAGE_SIZE, seg.IMAGE_SIZE)+ assert torch.allclose(tensor[0, 0], torch.ones(1)) and torch.allclose(+ tensor[0, 1], -torch.ones(1)+ )+++# ── the real forward on a tiny random Sam3Model ─────────────────────────────────────────────────────────────────────+def _tiny_config():+ from transformers import Sam3Config+ return Sam3Config(+ vision_config = {+ "backbone_config": {+ "hidden_size": 32,+ "intermediate_size": 64,+ "num_hidden_layers": 2,+ "num_attention_heads": 2,+ "window_size": 4,+ "global_attn_indexes": [1],+ "pretrain_image_size": 56,+ "image_size": 112,+ "patch_size": 14,+ },+ "fpn_hidden_size": 32,+ # 112 / 14 = 8 tokens a side, at the 4x, 2x and 1x levels.+ "backbone_feature_sizes": [[32, 32], [16, 16], [8, 8]],+ },+ text_config = {+ "vocab_size": 128,+ "hidden_size": 32,+ "intermediate_size": 64,+ "projection_dim": 32,+ "num_hidden_layers": 1,+ "num_attention_heads": 2,+ "max_position_embeddings": 32,+ },+ geometry_encoder_config = {+ "hidden_size": 32,+ "num_layers": 1,+ "num_attention_heads": 2,+ "intermediate_size": 64,+ },+ detr_encoder_config = {+ "hidden_size": 32,+ "num_layers": 1,+ "num_attention_heads": 2,+ "intermediate_size": 64,+ },+ detr_decoder_config = {+ "hidden_size": 32,+ "num_layers": 1,+ "num_queries": 12,+ "num_attention_heads": 2,+ "intermediate_size": 64,+ },+ mask_decoder_config = {+ "hidden_size": 32,+ "num_upsampling_stages": 3,+ "num_attention_heads": 2,+ },+ )+++def test_tiny_config_is_small_but_the_real_input_is_1008():+ """The tiny model runs at 112 px to keep the CPU tests fast; the size comes from the checkpoint's config."""+ from transformers import Sam3Config++ assert seg.IMAGE_SIZE == 1008+ assert Sam3Config().vision_config.backbone_config.image_size == seg.IMAGE_SIZE+++class _FakeTokenizer:+ """CLIP-shaped ids for the tiny vocab: one id per character, padded to max_length."""++ def __call__(+ self,+ texts,+ padding = None,+ max_length = 32,+ truncation = True,+ return_tensors = "pt",+ ):+ import torch++ ids = torch.zeros(len(texts), max_length, dtype = torch.long)+ mask = torch.zeros(len(texts), max_length, dtype = torch.long)+ for row, text in enumerate(texts):+ tokens = [1] + [2 + (ord(ch) % 100) for ch in text][: max_length - 2] + [127]+ ids[row, : len(tokens)] = torch.tensor(tokens)+ mask[row, : len(tokens)] = 1++ class _Batch(dict):+ def to(self, device):+ return _Batch({k: v.to(device) for k, v in self.items()})++ return _Batch(input_ids = ids, attention_mask = mask)+++def tiny_sam3(tmp_path, monkeypatch):+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "Sam3Model"):+ pytest.skip("transformers without SAM 3")+ import torch++ torch.manual_seed(0)+ model = transformers.Sam3Model(_tiny_config())+ model.save_pretrained(tmp_path)+ monkeypatch.setattr(+ transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: _FakeTokenizer()+ )+ return str(tmp_path)+++def test_tiny_model_segments_at_source_size(tiny_sam3):+ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ image = Image.new("RGB", (64, 40), (120, 80, 40))+ # A threshold of 0 keeps every query up to each concept's cap, so the caps are what is checked.+ result = segmenter.segment(+ image, "sofa, pillow:3, sofa", threshold = 0.0, allow_gpu = False, path = tiny_sam3+ )+ assert result.device == "cpu" and (result.width, result.height) == (64, 40)+ assert [c["found"] for c in result.concepts] == [1, 3, 1]+ assert [c["max_detections"] for c in result.concepts] == [1, 3, 1]+ assert [i.concept for i in result.instances] == ["sofa", "pillow", "pillow", "pillow", "sofa"]+ for inst in result.instances:+ x1, y1, x2, y2 = inst.box+ assert 0 <= x1 <= 64 and 0 <= y1 <= 40 and x2 <= 64 and y2 <= 40+ mask = Image.open(io.BytesIO(result.mask_png))+ assert mask.mode == "L" and mask.size == (64, 40)+ body = result.to_json()+ assert body["mask"].startswith("data:image/png;base64,")+ assert set(body) == {"mask", "width", "height", "instances", "prompts", "device", "elapsed_ms"}+ # No keep-alive: the weights left with the call.+ assert not segmenter.loaded+ # An unreachable threshold keeps nothing: an all-black mask, still at the source size.+ result = segmenter.segment(image, "sofa", threshold = 1.0, allow_gpu = False, path = tiny_sam3)+ assert result.instances == [] and Image.open(io.BytesIO(result.mask_png)).getbbox() is None+ assert torch.is_grad_enabled()+++def test_mask_planes_do_not_outlive_their_text(tiny_sam3, monkeypatch):+ import weakref++ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ real_load, real_select = segmenter._load, seg.select_detections+ planes: list = []++ def _select(*args, **kwargs):+ scores, boxes, masks = real_select(*args, **kwargs)+ planes.append(weakref.ref(masks))+ return scores, boxes, masks++ class _Watch:+ def __init__(self, model):+ self._model = model++ def __getattr__(self, name):+ return getattr(self._model, name)++ def __call__(self, **kwargs):+ # Every earlier text's kept mask planes are already OR-ed into the union and gone.+ assert all(ref() is None for ref in planes), "an earlier concept's mask planes are still held"+ return self._model(**kwargs)++ def _load(path, device, dtype):+ real_load(path, device, dtype)+ if not isinstance(segmenter._model, _Watch):+ segmenter._model = _Watch(segmenter._model)++ monkeypatch.setattr(seg, "select_detections", _select)+ monkeypatch.setattr(segmenter, "_load", _load)+ image = Image.new("RGB", (32, 32), (120, 80, 40))+ prompt = "cat:200, dog:200, cat:2, bird:200"+ result = segmenter.segment(image, prompt, threshold = 0.0, allow_gpu = False, path = tiny_sam3)+ assert len(planes) == 3+ # The union is what every occurrence's own best-k masks give: each text's largest cap covers its smaller ones.+ monkeypatch.setattr(seg, "select_detections", real_select)+ alone = [+ segmenter.segment(image, part, threshold = 0.0, allow_gpu = False, path = tiny_sam3)+ for part in ("cat:200", "dog:200", "bird:200")+ ]+ from PIL import ImageChops++ combined = Image.open(io.BytesIO(alone[0].mask_png))+ for other in alone[1:]:+ combined = ImageChops.lighter(combined, Image.open(io.BytesIO(other.mask_png)))+ assert ImageChops.difference(combined, Image.open(io.BytesIO(result.mask_png))).getbbox() is None+++def test_raw_outputs_do_not_outlive_their_concept(tiny_sam3, monkeypatch):+ import weakref++ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ real_load = segmenter._load+ outputs: list = []++ class _Watch:+ def __init__(self, model):+ self._model = model++ def __getattr__(self, name):+ return getattr(self._model, name)++ def __call__(self, **kwargs):+ # Every earlier concept's full output (all its mask planes) is gone before the next forward.+ assert all(ref() is None for ref in outputs), "a raw SAM 3 output is still held"+ out = self._model(**kwargs)+ outputs.append(weakref.ref(out))+ return out++ def _load(path, device, dtype):+ real_load(path, device, dtype)+ if not isinstance(segmenter._model, _Watch):+ segmenter._model = _Watch(segmenter._model)++ monkeypatch.setattr(segmenter, "_load", _load)+ image = Image.new("RGB", (32, 32), (120, 80, 40))+ result = segmenter.segment(+ image, "cat, dog, cat:3", threshold = 0.0, allow_gpu = False, path = tiny_sam3+ )+ assert len(outputs) == 2 # one forward per distinct text+ # The repeat keeps the best of the larger cap: cat:1 is cat:3's first detection.+ found = {(c["text"], c["max_detections"]): c["found"] for c in result.concepts}+ assert found[("cat", 1)] == 1 and found[("cat", 3)] == 3+ cats = [i for i in result.instances if i.concept == "cat"]+ assert cats[0] == cats[1] and cats[0].score >= cats[2].score >= cats[3].score+++def test_keep_alive_then_release(tiny_sam3):+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ image = Image.new("RGB", (32, 32))+ segmenter.segment(image, "cat", allow_gpu = False, path = tiny_sam3)+ assert segmenter.loaded and segmenter._timer is not None+ assert segmenter.release("test") is True+ assert not segmenter.loaded and segmenter._timer is None+ assert segmenter.release("again") is False+++def test_keep_alive_expiry_unloads(tiny_sam3):+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", allow_gpu = False, path = tiny_sam3)+ segmenter._timer.cancel()+ segmenter._expire(segmenter._release_epoch)+ assert not segmenter.loaded+++def test_release_during_a_call_unloads_when_it_ends(tiny_sam3, monkeypatch):+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ entered, proceed = threading.Event(), threading.Event()+ real = segmenter._segment_on++ def _slow(*args, **kwargs):+ entered.set()+ proceed.wait(10)+ return real(*args, **kwargs)++ monkeypatch.setattr(segmenter, "_segment_on", _slow)+ worker = threading.Thread(+ target = segmenter.segment,+ args = (Image.new("RGB", (16, 16)), "cat"),+ kwargs = {"allow_gpu": False, "path": tiny_sam3},+ )+ worker.start()+ assert entered.wait(10)+ # Non-blocking: the call is in flight, so the release is deferred, not refused.+ assert segmenter.release("arbiter") is False+ proceed.set()+ worker.join(30)+ assert not segmenter.loaded+++def test_tight_gpu_unloads_right_after_the_call(tiny_sam3, monkeypatch):+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", allow_gpu = False, path = tiny_sam3)+ assert segmenter.loaded+ # Pretend the copy sits on a card that is now tight.+ segmenter.device = "cuda:0"+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (1000, 80_000))+ with segmenter._run_lock:+ segmenter._after_call()+ assert not segmenter.loaded+++def test_vram_is_tight_rule():+ assert seg.vram_is_tight(1000, 24_000)+ assert seg.vram_is_tight(10_000, 80_000) # under 20% of the card+ assert not seg.vram_is_tight(30_000, 80_000)+ assert not seg.vram_is_tight(None, None)+++def test_choose_device_falls_back_to_cpu(monkeypatch):+ import torch++ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ if getattr(torch.backends, "mps", None) is not None:+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)+ assert seg.choose_device(True) == ("cpu", torch.float32)+ # Training (allow_gpu False) never takes a GPU.+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ assert seg.choose_device(False) == ("cpu", torch.float32)+++def test_choose_device_needs_room_on_the_card(monkeypatch):+ import contextlib++ import torch++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda: True)+ monkeypatch.setattr(torch.cuda, "device", lambda ordinal: contextlib.nullcontext())+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 1)+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (20_000, 24_000))+ assert seg.choose_device(True) == ("cuda:1", torch.bfloat16)+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (1_000, 24_000))+ if getattr(torch.backends, "mps", None) is not None:+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)+ assert seg.choose_device(True) == ("cpu", torch.float32)+++def test_choose_device_never_shares_a_card_another_owner_holds(monkeypatch):+ import contextlib++ import torch++ from core.inference import gpu_arbiter++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(torch.cuda, "device", lambda ordinal: contextlib.nullcontext())+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: True)+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (60_000, 80_000)) # plenty of room either way+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: None)+ monkeypatch.setattr(gpu_arbiter, "physical_gpu_ids", lambda visible: frozenset(visible))+ if getattr(torch.backends, "mps", None) is not None:+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)+ cases = [+ ({}, "cuda:0"), # nobody+ ({gpu_arbiter.DIFFUSION: frozenset({0})}, "cuda:0"), # the image model: SAM 3 runs beside it by design+ ({gpu_arbiter.CHAT: frozenset({0})}, "cpu"),+ ({gpu_arbiter.VIDEO: frozenset({0}), gpu_arbiter.DIFFUSION: frozenset({0})}, "cpu"),+ ({gpu_arbiter.CHAT: None}, "cpu"), # unknown GPUs: every card+ ({gpu_arbiter.CHAT: frozenset({1})}, "cuda:0"), # another card+ ]+ for held, expected in cases:+ monkeypatch.setattr(gpu_arbiter, "residents", lambda held = held: dict(held))+ device, dtype = seg.choose_device(True)+ assert device == expected, held+ assert dtype == (torch.bfloat16 if expected != "cpu" else torch.float32)+++def test_metal_is_not_shared_with_another_owner(monkeypatch):+ import torch++ from core.inference import gpu_arbiter++ if getattr(torch.backends, "mps", None) is None:+ pytest.skip("torch without an MPS backend")+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: True)+ # Apple has one device: any resident but the image model counts, whatever GPU ids it reports.+ for held, expected in (+ ({}, "mps"),+ ({gpu_arbiter.DIFFUSION: None}, "mps"),+ ({gpu_arbiter.CHAT: frozenset({0})}, "cpu"),+ ({gpu_arbiter.VIDEO: None, gpu_arbiter.DIFFUSION: None}, "cpu"),+ ({gpu_arbiter.CHAT: frozenset()}, "cpu"),+ ):+ monkeypatch.setattr(gpu_arbiter, "residents", lambda held = held: dict(held))+ assert seg.choose_device(True) == (expected, torch.float32), held+++def test_a_failed_load_places_no_weights(tmp_path, monkeypatch):+ import torch+ import transformers++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ placed, freed = [], []++ def _broken_tokenizer(*a, **k):+ raise OSError("no tokenizer files")++ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", _broken_tokenizer)+ monkeypatch.setattr(transformers.Sam3Model, "from_pretrained", lambda *a, **k: placed.append(a) or object())+ with pytest.raises(OSError):+ segmenter._load(str(tmp_path), "cuda:0", torch.bfloat16)+ assert placed == [] and not segmenter.loaded # the tokenizer fails before any weights reach the card++ # Weights placed, then the load fails: they are dropped and the device cache handed back.+ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: object())++ class _Half:+ def eval(self):+ raise RuntimeError("bad checkpoint")++ monkeypatch.setattr(transformers.Sam3Model, "from_pretrained", lambda *a, **k: _Half())+ monkeypatch.setattr(seg, "_free_device_cache", lambda device: freed.append(device))+ with pytest.raises(RuntimeError, match = "bad checkpoint"):+ segmenter._load(str(tmp_path), "cuda:0", torch.bfloat16)+ assert freed == ["cuda:0"] and not segmenter.loaded and segmenter._tokenizer is None+++def test_choose_device_reads_bf16_on_the_chosen_card(monkeypatch):+ import contextlib++ import torch++ # cuda:0 (current) has bf16, cuda:1 (the image model's card) does not.+ current = [0]++ @contextlib.contextmanager+ def _device(ordinal):+ before, current[0] = current[0], ordinal+ try:+ yield+ finally:+ current[0] = before++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: current[0])+ monkeypatch.setattr(torch.cuda, "device", _device)+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: current[0] == 0)+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (20_000, 24_000))+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 1)+ assert seg.choose_device(True) == ("cuda:1", torch.float16)+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 0)+ assert seg.choose_device(True) == ("cuda:0", torch.bfloat16)+ assert current[0] == 0+++def test_oom_on_the_gpu_retries_on_the_cpu(tiny_sam3, monkeypatch):+ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))+ real = segmenter._segment_on+ devices = []++ def _maybe_oom(image, concepts, threshold, path, device, dtype):+ devices.append(device)+ if device != "cpu":+ raise torch.OutOfMemoryError("CUDA out of memory")+ return real(image, concepts, threshold, path, device, dtype)++ monkeypatch.setattr(segmenter, "_segment_on", _maybe_oom)+ result = segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)+ assert devices == ["cuda:0", "cpu"] and result.device == "cpu"+++def test_oom_retry_frees_the_failed_attempt_before_reading_cpu_only(monkeypatch):+ import weakref++ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)++ class _CudaTensors: # stands in for the activations the failed attempt's frame holds+ pass++ pinned: list = []+ freed: list = []++ def _segment_on(image, concepts, threshold, path, device, dtype):+ if device != "cpu":+ activations = _CudaTensors()+ pinned.append(weakref.ref(activations))+ raise torch.OutOfMemoryError("CUDA out of memory")+ assert segmenter._call_device == "cpu"+ return types.SimpleNamespace()++ def _free(device):+ # The cache is released only once the OOM's traceback no longer pins the failed attempt's tensors.+ freed.append((device, pinned[0]() is None, segmenter._call_device))++ monkeypatch.setattr(segmenter, "_segment_on", _segment_on)+ monkeypatch.setattr(seg, "_free_device_cache", _free)+ segmenter.segment(Image.new("RGB", (8, 8)), "cat", path = "unused")+ assert freed and freed[0] == ("cuda:0", True, "cuda:0")+++def test_failed_cpu_retry_still_cleans_up(tiny_sam3, monkeypatch):+ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))+ real_load = segmenter._load++ def _oom_then_fail(image, concepts, threshold, path, device, dtype):+ if device != "cpu":+ raise torch.OutOfMemoryError("CUDA out of memory")+ real_load(path, device, dtype) # the CPU copy is resident when preprocessing fails+ segmenter._release_pending = True # a release that arrived mid-call+ raise RuntimeError("preprocess failed")++ monkeypatch.setattr(segmenter, "_segment_on", _oom_then_fail)+ with pytest.raises(RuntimeError, match = "preprocess failed"):+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)+ # The pending release was honoured, not left with the weights resident and no timer.+ assert not segmenter.loaded and segmenter._release_pending is False+ assert segmenter._call_device is None++ # Without a pending release the CPU copy gets the keep-alive instead.+ def _oom_then_fail_quietly(image, concepts, threshold, path, device, dtype):+ if device != "cpu":+ raise torch.OutOfMemoryError("CUDA out of memory")+ real_load(path, device, dtype)+ raise RuntimeError("preprocess failed")++ monkeypatch.setattr(segmenter, "_segment_on", _oom_then_fail_quietly)+ with pytest.raises(RuntimeError):+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)+ assert segmenter.loaded and segmenter._timer is not None+ segmenter.release("test")+++def test_segment_errors(monkeypatch, tmp_path):+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ image = Image.new("RGB", (8, 8))+ with pytest.raises(ValueError):+ segmenter.segment(image, " , ", path = str(tmp_path))+ with pytest.raises(ValueError):+ segmenter.segment(image, "sofa", threshold = 2.0, path = str(tmp_path))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)+ with pytest.raises(seg.SegmenterUnavailableError):+ segmenter.segment(image, "sofa", path = str(tmp_path))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ monkeypatch.setattr(seg, "model_dir", lambda: None)+ with pytest.raises(seg.SegmenterNotDownloadedError):+ segmenter.segment(image, "sofa")+++# ── files, status and download errors ───────────────────────────────────────────────────────────────────────────────+def test_override_dir_must_hold_a_config(tmp_path, monkeypatch):+ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))+ assert seg.model_dir() is None # a typo'd / empty override is not replaced by the cache+ (tmp_path / "config.json").write_text("{}")+ (tmp_path / "model.safetensors").write_bytes(b"")+ (tmp_path / "tokenizer.json").write_text("{}")+ assert seg.model_dir() == str(tmp_path)+ assert seg.status()["override"] is True+++def test_partial_override_does_not_read_as_downloaded(tmp_path, monkeypatch):+ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ (tmp_path / "config.json").write_text("{}")+ assert seg.model_dir() is None # config only: no weights, no tokenizer+ (tmp_path / "model.safetensors.index.json").write_text('{"weight_map": {"w": "model-1.safetensors"}}')+ (tmp_path / "model-1.safetensors").write_bytes(b"")+ assert seg.model_dir() is None # still no tokenizer+ (tmp_path / "vocab.json").write_text("{}")+ assert seg.model_dir() is None # vocab.json alone: AutoTokenizer refuses it without merges.txt+ (tmp_path / "merges.txt").write_text("#version: 0.2\n")+ assert seg.model_dir() == str(tmp_path) # sharded weights + a tokenizer+ (tmp_path / "model.safetensors.index.json").unlink()+ body = seg.status()+ assert body["downloaded"] is False and body["override"] is True+++def test_sharded_override_needs_every_shard(tmp_path, monkeypatch):+ import json++ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ (tmp_path / "config.json").write_text("{}")+ (tmp_path / "tokenizer.json").write_text("{}")+ shards = ["model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"]+ index = tmp_path / "model.safetensors.index.json"+ index.write_text(json.dumps({"weight_map": {"a": shards[0], "b": shards[1], "c": shards[1]}}))+ (tmp_path / shards[0]).write_bytes(b"")+ assert seg.model_dir() is None # an interrupted copy: the second shard is missing+ assert seg.status()["downloaded"] is False+ (tmp_path / shards[1]).write_bytes(b"")+ assert seg.model_dir() == str(tmp_path)+ for broken in ("not json", json.dumps({"weight_map": {}}), json.dumps({"metadata": {}})):+ index.write_text(broken)+ assert seg.model_dir() is None+++def test_status_shape(monkeypatch):+ monkeypatch.delenv(seg.ENV_PATH_OVERRIDE, raising = False)+ monkeypatch.setattr(seg, "_cached_snapshot", lambda *a, **k: None)+ body = seg.status()+ assert body["repo"] == "facebook/sam3" and body["gated"] is True+ assert body["downloaded"] is False and body["loaded"] is False+ assert body["license_url"].startswith("https://huggingface.co/facebook/sam3")+ assert body["size_bytes"] > 3_000_000_000+ assert {"downloading", "error", "bytes_total", "bytes_done"} <= set(body["download"])+++def test_the_download_names_only_what_sam3model_reads():+ assert seg.SEGMENTER_SPEC.files == seg.SEGMENTER_FILES+ assert "model.safetensors" in seg.SEGMENTER_FILES and "sam3.pt" not in seg.SEGMENTER_FILES+++def test_gated_download_error_names_the_license():+ class GatedRepoError(Exception):+ pass++ message = seg.describe_download_error("sam3", GatedRepoError("403 Client Error"))+ assert "license" in message and seg.SEGMENTER_LICENSE_URL in message+ assert seg.describe_download_error("sam3", RuntimeError("disk full")) is None+++def test_shared_download_state_uses_the_spec_file_list(monkeypatch, tmp_path):+ """The pinned downloader fetches a spec's ``files`` when it has them (the dictation specs keep their pair)."""+ from core.inference import stt_mtmd_sidecar as mtmd++ seen = []++ class _Meta:+ commit_hash = "a" * 40+ size = 10+ etag = "e"++ def _meta(url, token = None):+ seen.append(url)+ raise RuntimeError("stop after the first lookup")++ monkeypatch.setattr("huggingface_hub.get_hf_file_metadata", _meta)+ state = mtmd._MtmdDownloadState(+ specs = {seg.SEGMENTER_ID: seg.SEGMENTER_SPEC},+ cached_paths = lambda *a, **k: None,+ resolve = seg._resolve_segmenter_id,+ label = "SAM 3 segmenter",+ describe_error = lambda model_id, error: "described",+ )+ state._run(seg.SEGMENTER_ID, None, hub_cache = tmp_path)+ assert seen and seen[0].endswith(seg.SEGMENTER_FILES[0])+ assert state.status()["error"] == "described"+++# ── GPU arbiter hooks ───────────────────────────────────────────────────────────────────────────────────────────────+class _Recorder:+ def __init__(self):+ self.calls = []++ def release_segmenter(+ self,+ reason,+ wait = False,+ handover = False,+ ):+ self.calls.append((reason, wait))+ return True+++def test_arbiter_releases_the_segmenter(monkeypatch):+ import sys++ from core.inference import gpu_arbiter++ recorder = _Recorder()+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)+ gpu_arbiter._release_mask_segmenter("x")+ assert recorder.calls == [("x", True)]+ # Training's start path: release(DIFFUSION) whether or not an image model is resident.+ recorder.calls.clear()+ monkeypatch.setattr(gpu_arbiter, "_residents", {})+ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)+ assert recorder.calls and recorder.calls[0][0] == "image GPU released"+ # Never loaded: nothing is imported or created.+ monkeypatch.delitem(sys.modules, "core.inference.diffusion_segment")+ gpu_arbiter._release_mask_segmenter("y")+++def _hold_a_call(segmenter, device):+ """Run a fake call on ``device`` that holds the run lock until the returned event is set."""+ entered, proceed = threading.Event(), threading.Event()++ def _call():+ with segmenter._run_lock:+ segmenter._call_device = device+ entered.set()+ proceed.wait(10)+ segmenter._call_device = None++ worker = threading.Thread(target = _call)+ worker.start()+ assert entered.wait(10)+ return proceed, worker+++def test_a_release_timing_out_as_the_call_settles_still_unloads(monkeypatch):+ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (None, None))++ def _segment_on(image, concepts, threshold, path, device, dtype):+ segmenter._model, segmenter.device = object(), device # the CUDA copy stays resident after the call+ return types.SimpleNamespace()++ real_after = segmenter._after_call+ settled = threading.Event()++ def _after_call():+ real_after() # reads the release flag (none yet) and arms the keep-alive+ settled.set()+ time.sleep(0.3) # the call's tail: the release's wait runs out right here++ monkeypatch.setattr(segmenter, "_segment_on", _segment_on)+ monkeypatch.setattr(segmenter, "_after_call", _after_call)+ worker = threading.Thread(target = segmenter.segment, args = (Image.new("RGB", (8, 8)), "cat"), kwargs = {"path": "x"})+ worker.start()+ assert settled.wait(10)+ # Timed out on a call that will not see the flag: the handover waits out the tail and unloads, never letting the+ # next owner in beside the resident weights.+ assert segmenter.release("handover", wait = True, timeout = 0.05, handover = True) is True+ worker.join(10)+ assert not segmenter.loaded and segmenter._timer is None and segmenter._release_pending is False+ # Codex's interleaving: the call already cleared its device and is only letting go of the lock.+ segmenter._model, segmenter.device = object(), "cuda:0"+ proceed, holder = _hold_a_call(segmenter, None)+ threading.Timer(0.2, proceed.set).start()+ try:+ assert segmenter.release("handover", wait = True, timeout = 0.05, handover = True) is True+ finally:+ proceed.set()+ holder.join(10)+ assert not segmenter.loaded+++def test_handover_refuses_while_a_gpu_call_outlives_the_wait(monkeypatch):+ import sys++ from core.inference import gpu_arbiter++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ monkeypatch.setattr(seg, "_segmenter", segmenter)+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", seg)+ real_release = segmenter.release+ monkeypatch.setattr(+ segmenter, "release", lambda reason, **kw: real_release(reason, **{**kw, "timeout": 0.05})+ )+ for device in ("cuda:0", ""): # on the card, or not placed yet+ proceed, worker = _hold_a_call(segmenter, device)+ try:+ with pytest.raises(seg.SegmenterReleaseTimeoutError):+ seg.release_segmenter("x", wait = True, handover = True)+ # The arbiter aborts the transfer instead of handing over a busy card.+ with pytest.raises(RuntimeError, match = "cannot be transferred"):+ gpu_arbiter._release_mask_segmenter("GPU acquired for chat", handover = True)+ # Release / training paths stay best effort: deferred, logged, never raised.+ gpu_arbiter._release_mask_segmenter("image GPU released")+ assert segmenter._release_pending is True+ finally:+ proceed.set()+ worker.join(10)+ # A call on the CPU holds no VRAM: the handover proceeds and the release stays deferred.+ proceed, worker = _hold_a_call(segmenter, "cpu")+ try:+ assert seg.release_segmenter("x", wait = True, handover = True) is False+ gpu_arbiter._release_mask_segmenter("GPU acquired for chat", handover = True)+ finally:+ proceed.set()+ worker.join(10)+ # Idle: nothing to wait for.+ assert seg.release_segmenter("x", wait = True, handover = True) is False+++def test_acquire_does_not_register_while_the_segmenter_holds_the_card(monkeypatch):+ import sys++ from core.inference import gpu_arbiter++ class _Busy:+ class SegmenterReleaseTimeoutError(RuntimeError):+ pass++ def release_segmenter(+ self,+ reason,+ wait = False,+ handover = False,+ ):+ if handover:+ raise self.SegmenterReleaseTimeoutError("still running")+ return False++ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", _Busy())+ monkeypatch.setattr(gpu_arbiter, "_residents", {})+ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)+ registered = []+ with pytest.raises(RuntimeError, match = "cannot be transferred"):+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = lambda: registered.append(1))+ assert registered == [] and gpu_arbiter.current_owner() is None+++def test_a_call_queued_across_a_handover_stays_off_the_card(tiny_sam3, monkeypatch):+ import torch+ from PIL import Image++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ monkeypatch.setattr(+ seg,+ "choose_device",+ lambda allow_gpu: ("cuda:0", torch.bfloat16) if allow_gpu else ("cpu", torch.float32),+ )+ placed = []+ monkeypatch.setattr(+ segmenter, "_segment_on", lambda *a: placed.append(a[4]) or types.SimpleNamespace()+ )+ image = Image.new("RGB", (16, 16))+ for handover, cpu_only in ((True, False), (True, True), (False, False)):+ placed.clear()+ proceed, worker = _hold_a_call(segmenter, "cuda:0")+ errors = []++ def _queued():+ try:+ segmenter.segment(image, "cat", allow_gpu = not cpu_only, path = tiny_sam3)+ except Exception as exc: # noqa: BLE001+ errors.append(exc)++ queued = threading.Thread(target = _queued)+ queued.start()+ deadline = time.monotonic() + 5+ while not segmenter._run_lock.locked() and time.monotonic() < deadline:+ time.sleep(0.01)+ time.sleep(0.1) # the queued call is now blocked on _run_lock+ # The handover's release waits on the call in flight; it ends, the release takes and drops the lock, and the+ # new owner goes on to register.+ releaser = threading.Thread(+ target = lambda: segmenter.release("handover", wait = True, handover = handover)+ )+ releaser.start()+ time.sleep(0.1)+ proceed.set()+ worker.join(10)+ releaser.join(10)+ queued.join(10)+ if handover and not cpu_only:+ assert (+ placed == [] and len(errors) == 1 and isinstance(errors[0], seg.SegmenterBusyError)+ )+ elif handover:+ assert placed == ["cpu"] and errors == [] # a CPU-only call holds no VRAM+ else:+ assert placed == ["cuda:0"] and errors == [] # an ordinary release does not fence calls+ # A call that starts after the handover runs normally.+ placed.clear()+ segmenter.segment(image, "cat", path = tiny_sam3)+ assert placed == ["cuda:0"]+++def test_a_call_during_a_handovers_register_stays_off_the_card(tiny_sam3, monkeypatch):+ import sys++ import torch+ from PIL import Image++ from core.inference import gpu_arbiter++ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)+ monkeypatch.setattr(seg, "_segmenter", segmenter)+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", seg)+ monkeypatch.setattr(+ seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16) if allow_gpu else ("cpu", torch.float32)+ )+ placed = []+ monkeypatch.setattr(segmenter, "_segment_on", lambda *a: placed.append(a[4]) or types.SimpleNamespace())+ monkeypatch.setattr(gpu_arbiter, "_residents", {})+ monkeypatch.setattr(gpu_arbiter, "_owner", None)+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)+ image = Image.new("RGB", (16, 16))+ seen = {}++ def _register():+ # The segmenter's release already ran and dropped its lock; the new owner is still loading.+ seen["fenced"] = gpu_arbiter.handover_in_progress()+ with pytest.raises(seg.SegmenterBusyError):+ segmenter.segment(image, "cat", path = tiny_sam3)+ segmenter.segment(image, "cat", allow_gpu = False, path = tiny_sam3) # the CPU is not being handed over++ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = _register)+ assert seen["fenced"] is True and placed == ["cpu"]+ assert gpu_arbiter.handover_in_progress() is False+ segmenter.segment(image, "cat", path = tiny_sam3)+ assert placed == ["cpu", "cuda:0"]+ # A raising register still lifts the fence.+ with pytest.raises(RuntimeError):+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = lambda: (_ for _ in ()).throw(RuntimeError("load failed")))+ assert gpu_arbiter.handover_in_progress() is False+++def test_training_beside_an_off_torch_image_model_releases_the_segmenter(monkeypatch):+ import inspect+ import sys+ from types import SimpleNamespace++ import routes.training as training_routes+ from core.inference import diffusion_engine_router, gpu_arbiter++ recorder = _Recorder()+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)+ resident = SimpleNamespace(runs_off_torch_device = True, is_loaded = True, unload = lambda: None)+ monkeypatch.setattr(diffusion_engine_router, "get_active_diffusion_engine", lambda: resident)+ monkeypatch.setattr(gpu_arbiter, "release", lambda owner: None)+ monkeypatch.setattr(+ "core.inference.video.get_video_backend",+ lambda: SimpleNamespace(status = lambda: {"loaded": False}, unload = lambda: None),+ )+ monkeypatch.setattr("routes.training_vram.summarize_resident_chat", lambda: {"any": False})+ training_routes._free_gpu_for_diffusion_training()+ assert ("diffusion training start", True) in recorder.calls+ # The LLM start path keeps an off-torch image model the same way and releases the segmenter there too.+ source = inspect.getsource(training_routes.start_training)+ release = source.index('gpu_arbiter.release_mask_segmenter("training start")')+ assert release < source.index('if getattr(diffusion, "runs_off_torch_device", False) is True:')+++def test_training_start_refuses_while_a_mask_call_holds_the_gpu(monkeypatch):+ import inspect+ import sys+ from types import SimpleNamespace++ import routes.training as training_routes+ from core.inference import diffusion_engine_router, gpu_arbiter++ class _StillRunning:+ class SegmenterReleaseTimeoutError(RuntimeError):+ pass++ def release_segmenter(self, reason, wait = False, handover = False):+ if handover:+ raise self.SegmenterReleaseTimeoutError("still running")+ return False++ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", _StillRunning())+ with pytest.raises(gpu_arbiter.SegmenterStillRunningError, match = "try again"):+ gpu_arbiter.release_mask_segmenter("training start")+ assert gpu_arbiter.SegmenterStillRunningError.blocks_training is True # the core trainer re-raises it+ unloaded = []+ for off_torch in (True, False):+ resident = SimpleNamespace(+ runs_off_torch_device = off_torch, is_loaded = True, unload = lambda: unloaded.append(1)+ )+ monkeypatch.setattr(diffusion_engine_router, "get_active_diffusion_engine", lambda r = resident: r)+ monkeypatch.setattr(gpu_arbiter, "release", lambda owner: None)+ # A RuntimeError: the diffusion start route answers 409 instead of spawning beside SAM 3.+ with pytest.raises(gpu_arbiter.SegmenterStillRunningError):+ training_routes._free_gpu_for_diffusion_training()+ assert unloaded == [] # refused before tearing anything down+ # The LLM start path: re-raised out of its best-effort block and answered 409.+ source = inspect.getsource(training_routes.start_training)+ assert "except gpu_arbiter.SegmenterStillRunningError:\n raise" in source+ assert source.count("SegmenterStillRunningError,") == 2+++def test_no_find_starts_while_a_render_is_on_its_way(monkeypatch):+ import asyncio++ import core.inference.diffusion_engine_router as engine_router+ import routes.inference as inference_routes++ engine = _Engine()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)+ monkeypatch.setattr(+ seg,+ "get_segmenter",+ lambda: pytest.fail("a Find ran between the render's SAM 3 release and its slot"),+ )+ seen = []++ async def _run(request):+ # The render released SAM 3 and has not taken the slot yet: a Find is refused, and frees the slot it took.+ with pytest.raises(seg.SegmenterBusyError, match = "generating"):+ seg.segment_image(_png_b64(), "sofa")+ seen.append(engine._generate_lock.locked())+ return "done"++ monkeypatch.setattr(inference_routes, "_generate_diffusion_image_run", _run)+ assert asyncio.run(inference_routes._generate_diffusion_image_core(object())) == "done"+ assert seen == [False] and seg._renders_pending == 0+++def test_a_render_releases_the_segmenter_first(monkeypatch):+ import inspect+ import sys++ import routes.inference as inference_routes++ recorder = _Recorder()+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)+ inference_routes._release_mask_segmenter_for_render()+ assert recorder.calls == [("image generation", True)]+ # Never loaded: nothing is imported.+ monkeypatch.delitem(sys.modules, "core.inference.diffusion_segment")+ inference_routes._release_mask_segmenter_for_render()+ # Before the run, on every render (a resident model renders with no arbiter acquire).+ source = inspect.getsource(inference_routes._generate_diffusion_image_run)+ release = source.index("_release_mask_segmenter_for_render")+ assert source.index("_enhance_generate_prompts(backend, request)") < release < source.index("for attempt in range(2)")+++def test_release_segmenter_never_creates_one(monkeypatch):+ monkeypatch.setattr(seg, "_segmenter", None)+ assert seg.release_segmenter("x") is False+ assert seg._segmenter is None+++# ── render slot ─────────────────────────────────────────────────────────────────────────────────────────────────────+class _Engine:+ def __init__(self):+ self._generate_lock = threading.Lock()+ self._loading = None+++def test_render_slot_refuses_while_generating_or_loading(monkeypatch):+ import core.inference.diffusion_engine_router as engine_router++ engine = _Engine()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)+ lock, why = seg._take_render_slot()+ assert lock is engine._generate_lock and why == ""+ lock.release()+ with engine._generate_lock:+ assert seg._take_render_slot() == (None, "generating")++ class _Loading:+ error = None++ engine._loading = _Loading()+ assert seg._take_render_slot() == (None, "loading")+ engine._loading.error = "failed"+ lock, why = seg._take_render_slot()+ assert why == "" and lock is not None+ lock.release()+++def test_segment_image_holds_and_frees_the_slot(monkeypatch):+ import core.inference.diffusion_engine_router as engine_router++ engine = _Engine()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)++ class _Stub:+ def segment(self, image, prompt, threshold, allow_gpu):+ assert engine._generate_lock.locked()+ assert image.size == (64, 48)+ return seg.SegmentResult(mask_png = b"png", width = 64, height = 48)++ monkeypatch.setattr(seg, "get_segmenter", lambda: _Stub())+ body = seg.segment_image(_png_b64(), "sofa")+ assert body["width"] == 64 and not engine._generate_lock.locked()+ with engine._generate_lock:+ with pytest.raises(seg.SegmenterBusyError):+ seg.segment_image(_png_b64(), "sofa")+ with pytest.raises(ValueError):+ seg.segment_image("not base64 at all !!", "sofa")+++# ── routes ──────────────────────────────────────────────────────────────────────────────────────────────────────────+def client(monkeypatch):+ from fastapi import FastAPI+ from fastapi.testclient import TestClient++ import core.inference.diffusion_engine_router as engine_router+ from auth.authentication import authenticated_via_api_key, get_current_subject+ from routes.inference import studio_router++ engine = _Engine()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)+ app = FastAPI()+ app.include_router(studio_router, prefix = "/api/inference")+ app.dependency_overrides[get_current_subject] = lambda: "test-user"+ app.dependency_overrides[authenticated_via_api_key] = lambda: False+ return TestClient(app), engine+++def test_segment_route_contract(client, tiny_sam3, monkeypatch):+ http, engine = client+ import routes.inference as inference_routes++ monkeypatch.setattr(seg, "model_dir", lambda: tiny_sam3)+ monkeypatch.setattr(seg, "_segmenter", seg.MaskSegmenter(keep_alive_seconds = 0))+ monkeypatch.setattr(+ inference_routes, "_training_is_active", lambda: True+ ) # CPU, no GPU in tests+ r = http.post(+ "/api/inference/images/segment",+ json = {"image": _png_b64(64, 48), "prompt": "Sofa:2, red lamp", "threshold": 0.0},+ )+ assert r.status_code == 200, r.text+ body = r.json()+ assert body["width"] == 64 and body["height"] == 48 and body["device"] == "cpu"+ assert [p["text"] for p in body["prompts"]] == ["Sofa", "red lamp"]+ assert [p["max_detections"] for p in body["prompts"]] == [2, 1]+ assert len(body["instances"]) == 3+ assert all(+ set(i) == {"prompt", "score", "box"} and len(i["box"]) == 4 for i in body["instances"]+ )+ mask = _decode_mask(body["mask"])+ assert mask.size == (64, 48) and mask.mode == "L"+++def test_segment_route_errors(client, monkeypatch):+ http, engine = client+ url = "/api/inference/images/segment"+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ monkeypatch.setattr(seg, "model_dir", lambda: None)+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})+ assert r.status_code == 409 and "not downloaded" in r.json()["detail"]+ r = http.post(url, json = {"image": _png_b64(), "prompt": " , , "})+ assert r.status_code == 400+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa", "threshold": 3})+ assert r.status_code == 422+ r = http.post(url, json = {"image": "!!!", "prompt": "sofa"})+ assert r.status_code == 400+ monkeypatch.setattr(seg, "model_dir", lambda: "/nonexistent")+ with engine._generate_lock:+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})+ assert r.status_code == 409 and "generating" in r.json()["detail"]+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})+ assert r.status_code == 501+++def test_status_download_cancel_release_routes(client, monkeypatch):+ http, _engine = client+ from hub.services.models import account_access++ monkeypatch.setattr(account_access, "require_installation_owner", lambda: None)+ monkeypatch.setattr(+ seg, "status", lambda: {"supported": True, "downloaded": False, "override": False}+ )+ assert http.get("/api/inference/images/segment/status").json()["downloaded"] is False+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)+ started = {}+ monkeypatch.setattr(+ seg, "start_download", lambda token: started.setdefault("token", token) and "dl-1"+ )+ monkeypatch.setattr(seg, "download_status", lambda: {"downloading": True})+ r = http.post("/api/inference/images/segment/download", json = {"hf_token": "hf_x"})+ assert r.status_code == 200 and r.json()["download_id"] == "dl-1" and started["token"] == "hf_x"+ monkeypatch.setattr(seg, "cancel_download", lambda: True)+ r = http.post("/api/inference/images/segment/cancel")+ assert r.status_code == 200 and r.json()["cancelled"] is True+ released = []+ monkeypatch.setattr(seg, "release_segmenter", lambda reason: released.append(reason) or True)+ r = http.post("/api/inference/images/segment/release")+ assert r.status_code == 200 and released == ["mask painter closed"]+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)+ monkeypatch.setattr(seg, "status", lambda: {"supported": False, "reason": "old transformers"})+ r = http.post("/api/inference/images/segment/download")+ assert r.status_code == 501+++def test_a_no_torch_install_reports_sam3_unsupported(monkeypatch):+ import importlib.util++ real = importlib.util.find_spec+ monkeypatch.setattr(+ importlib.util, "find_spec", lambda name, *a: None if name == "torch" else real(name, *a)+ )+ assert seg.transformers_supports_sam3() is False+ body = seg.status()+ assert body["supported"] is False and "PyTorch" in body["reason"]diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsxindex 0d3624fa160..cfb46c84cfe 100644--- a/studio/frontend/src/features/images/images-page.tsx+++ b/studio/frontend/src/features/images/images-page.tsx@@ -253,6 +253,9 @@ import {withTransparencyPrompt,} from "./edit-conditioning";import { LocalizedEditCanvas } from "./localized-edit-canvas";+import type { AppliedMask } from "./lib/mask-from-text";+import { applyMaskToLayer } from "./mask-apply";+import { MaskFromTextPanel } from "./mask-from-text-panel";import {type ControlNetSpecInput,@@ -932,11 +935,14 @@ function MaskCanvas({brushPct,resetKey,onMaskChange,+ applyMask,}: {image: string;brushPct: number;resetKey: number;onMaskChange: (dataUrl: string | null) => void;+ /** A "Mask from text" result; each new key is drawn into the mask once, then the brush keeps editing it. */+ applyMask?: AppliedMask | null;}) {const dispRef = useRef<HTMLCanvasElement | null>(null);const maskRef = useRef<HTMLCanvasElement | null>(null);@@ -973,6 +979,34 @@ function MaskCanvas({img.src = image;}, [image, resetKey, onMaskChange]);+ // Applied once per key, after the mask is sized for this image; Clear or a new image starts blank as before. A+ // result from before this canvas mounted (another workflow's painter) is not replayed.+ const appliedKey = useRef<number | null>(applyMask?.key ?? null);+ useEffect(() => {+ if (!applyMask || !ready || appliedKey.current === applyMask.key) return;+ appliedKey.current = applyMask.key;+ const disp = dispRef.current;+ const mask = maskRef.current;+ if (!disp || !mask) return;+ let live = true;+ applyMaskToLayer({+ layer: mask,+ display: disp,+ dataUrl: applyMask.dataUrl,+ additive: applyMask.additive,+ overlay: "mask",+ isCurrent: () => live,+ })+ .then((url) => {+ // An empty Replace leaves the mask black: nothing is selected, so Generate's empty-mask check still holds.+ if (live && url !== null) onMaskChange(applyMask.empty ? null : url);+ })+ .catch(() => {});+ return () => {+ live = false;+ };+ }, [applyMask, ready, onMaskChange]);+const radius = useCallback(() => {const base = Math.min(dims.current.w, dims.current.h) || 1024;return Math.max(2, (brushPct / 100) * base);@@ -1619,6 +1653,13 @@ export function ImagesPage({const [maskImage, setMaskImage] = useState<string | null>(null);const [brushPct, setBrushPct] = useState(8);const [maskResetKey, setMaskResetKey] = useState(0);+ // The latest "Mask from text" result, handed to whichever painter is showing; a new key applies it once.+ const [textMask, setTextMask] = useState<AppliedMask | null>(null);+ const onTextMask = useCallback(+ (mask: { dataUrl: string; additive: boolean }) =>+ setTextMask((prev) => ({ ...mask, key: (prev?.key ?? 0) + 1 })),+ [],+ );// Extend (outpaint): how far to grow each dimension and which sides; reuses the inpaint backend at generate time.const [extendPct, setExtendPct] = useState(25);const [extendSides, setExtendSides] = useState<ExtendSides>({@@ -5634,8 +5675,10 @@ export function ImagesPage({brushPct={brushPct}resetKey={maskResetKey}onMaskChange={setMaskImage}+ applyMask={textMask}/></Field>+ <MaskFromTextPanel image={initImage} onMask={onTextMask} /><SliderFieldlabel="Brush size"hint="Brush radius as a percent of the image's shorter side."@@ -5835,7 +5878,11 @@ export function ImagesPage({resetKey={localizedResetKey}onLayerChange={onLocalizedLayer}onColorsChange={onLocalizedColors}+ applyMask={textMask}/>+ {localizedMode !== "annotate" && (+ <MaskFromTextPanel image={initImage} onMask={onTextMask} />+ )}<Buttontype="button"variant="secondary"diff --git a/studio/frontend/src/features/images/lib/mask-from-text.ts b/studio/frontend/src/features/images/lib/mask-from-text.tsnew file mode 100644index 00000000000..3217d688cd6--- /dev/null+++ b/studio/frontend/src/features/images/lib/mask-from-text.ts@@ -0,0 +1,255 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// "Mask from text": SAM 3 finds the objects a short text names and returns their union as a mask the inpaint and+// localized-edit painters load, so it stays editable with the brush. Pure logic only (no "@/" imports) so node:test+// can load it directly.++/** Every segmenter route, in one place. */+export const MASK_SEGMENT_ROUTES = {+ status: "/api/inference/images/segment/status",+ download: "/api/inference/images/segment/download",+ cancel: "/api/inference/images/segment/cancel",+ release: "/api/inference/images/segment/release",+ segment: "/api/inference/images/segment",+} as const;++export const MASK_FROM_TEXT_LABEL = "Mask from text";+export const MASK_FROM_TEXT_PLACEHOLDER = "sofa, red lamp, pillow:2";+export const MASK_FROM_TEXT_HINT =+ "Name what to mask, separated by commas. Each name selects its best match; add :N to select up to N, as in pillow:3. The mask lands in the painter, so you can still brush or erase it.";++export interface SegmenterDownload {+ downloading: boolean;+ download_id: string | null;+ error: string | null;+ cancelled: boolean;+ bytes_total: number | null;+ bytes_done: number | null;+}++export interface SegmenterStatus {+ supported: boolean;+ reason: string | null;+ downloaded: boolean;+ loaded: boolean;+ device: string | null;+ repo: string;+ gated: boolean;+ license_url: string;+ size_bytes: number | null;+ download: SegmenterDownload;+}++export interface SegmentInstance {+ prompt: string;+ score: number;+ box: [number, number, number, number];+}++export interface SegmentPromptResult {+ text: string;+ max_detections: number;+ found: number;+}++export interface SegmentResult {+ mask: string;+ width: number;+ height: number;+ instances: SegmentInstance[];+ prompts: SegmentPromptResult[];+ device: string;+ elapsed_ms: number;+}++export interface SegmentRequest {+ image: string;+ prompt: string;+ threshold?: number;+}++/** What a painter receives: a new ``key`` applies ``dataUrl`` once. */+export interface AppliedMask {+ key: number;+ dataUrl: string;+ additive: boolean;+ /** A Replace that found nothing: the painter clears its canvas and holds no mask (null), not a blank PNG. */+ empty?: boolean;+}++const IDLE_DOWNLOAD: SegmenterDownload = {+ downloading: false,+ download_id: null,+ error: null,+ cancelled: false,+ bytes_total: null,+ bytes_done: null,+};++const num = (v: unknown): number | null =>+ typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;+const str = (v: unknown): string | null => (typeof v === "string" ? v : null);+const obj = (v: unknown): Record<string, unknown> =>+ v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};++/** Tolerant parse of the status body: a missing or malformed field reads as "not there", never a throw. */+export function parseSegmenterStatus(raw: unknown): SegmenterStatus {+ const body = obj(raw);+ const d = obj(body.download);+ return {+ supported: body.supported === true,+ reason: str(body.reason),+ downloaded: body.downloaded === true,+ loaded: body.loaded === true,+ device: str(body.device),+ repo: str(body.repo) ?? "",+ gated: body.gated === true,+ license_url: str(body.license_url) ?? "",+ size_bytes: num(body.size_bytes),+ download: {+ ...IDLE_DOWNLOAD,+ downloading: d.downloading === true,+ download_id: str(d.download_id),+ error: str(d.error),+ cancelled: d.cancelled === true,+ bytes_total: num(d.bytes_total),+ bytes_done: num(d.bytes_done),+ },+ };+}++function parseBox(v: unknown): [number, number, number, number] | null {+ if (!Array.isArray(v) || v.length !== 4) return null;+ const out = v.map((x) => (typeof x === "number" && Number.isFinite(x) ? x : NaN));+ return out.some(Number.isNaN) ? null : (out as [number, number, number, number]);+}++/** Parse the segment response. Throws when there is no usable mask, since that is the one thing the painter needs. */+export function parseSegmentResult(raw: unknown): SegmentResult {+ const body = obj(raw);+ const mask = str(body.mask);+ if (!mask || !mask.startsWith("data:image/")) {+ throw new Error("The segmenter returned no mask.");+ }+ const instances: SegmentInstance[] = [];+ for (const item of Array.isArray(body.instances) ? body.instances : []) {+ const it = obj(item);+ const box = parseBox(it.box);+ const score = typeof it.score === "number" && Number.isFinite(it.score) ? it.score : null;+ if (box === null || score === null) continue;+ instances.push({ prompt: str(it.prompt) ?? "", score, box });+ }+ const prompts: SegmentPromptResult[] = [];+ for (const item of Array.isArray(body.prompts) ? body.prompts : []) {+ const it = obj(item);+ const text = str(it.text);+ if (!text) continue;+ prompts.push({+ text,+ max_detections: Math.max(1, Math.round(num(it.max_detections) ?? 1)),+ found: Math.round(num(it.found) ?? 0),+ });+ }+ return {+ mask,+ width: num(body.width) ?? 0,+ height: num(body.height) ?? 0,+ instances,+ prompts,+ device: str(body.device) ?? "",+ elapsed_ms: num(body.elapsed_ms) ?? 0,+ };+}++/** The Download control is offered: supported, and the weights are not on disk yet. */+export function segmenterNeedsDownload(status: SegmenterStatus | null): boolean {+ return status?.supported === true && !status.downloaded;+}++/** Find can run: supported, downloaded, and no download in flight. */+export function segmenterReady(status: SegmenterStatus | null): boolean {+ return status?.supported === true && status.downloaded && !status.download.downloading;+}++/** 0..1 download fraction, or null when the total is not known yet. */+export function segmenterDownloadFraction(download: SegmenterDownload): number | null {+ const total = download.bytes_total ?? 0;+ if (!(total > 0)) return null;+ return Math.min(1, Math.max(0, (download.bytes_done ?? 0) / total));+}++/** The detector's query count: the backend never keeps more of one concept. */+export const MAX_DETECTIONS = 200;++/** Python's round(): halves go to the even neighbour, so "pillow:2.5" keeps 2 here as it does in the backend. */+function roundHalfEven(value: number): number {+ const floor = Math.floor(value);+ const diff = value - floor;+ if (diff > 0.5) return floor + 1;+ if (diff < 0.5) return floor;+ return floor % 2 === 0 ? floor : floor + 1;+}++/** ComfyUI's SAM 3 prompt syntax (comfy/text_encoders/sam3_clip.py ``_parse_prompts``): parentheses dropped, comma+ * separated concepts, an optional trailing ``:N`` for the most detections of that concept (rounded, at least 1,+ * default 1). For the hint and validation only: the backend parses the prompt it runs. */+export function parseMaskPrompt(text: string): Array<[string, number]> {+ const cleaned = text.replace(/[()]/g, "");+ const out: Array<[string, number]> = [];+ for (const raw of cleaned.split(",")) {+ const part = raw.trim();+ if (!part) continue;+ const m = /^(.+?)\s*:\s*([\d.]+)\s*$/.exec(part);+ if (m) {+ // Number(), not parseFloat: "2.5.1" is not a count, so the whole part stays the concept (as the backend reads it).+ const value = Number(m[2]);+ if (Number.isFinite(value)) {+ out.push([m[1].trim(), Math.min(MAX_DETECTIONS, Math.max(1, roundHalfEven(value)))]);+ continue;+ }+ }+ out.push([part, 1]);+ }+ return out;+}++/** One line after a run: how many concepts matched and how many objects each gave. */+export function segmentSummary(result: Pick<SegmentResult, "prompts" | "instances">): string {+ const prompts = result.prompts;+ if (prompts.length === 0) {+ return result.instances.length > 0+ ? `Found ${result.instances.length} object${result.instances.length === 1 ? "" : "s"}.`+ : "Nothing found.";+ }+ const matched = prompts.filter((p) => p.found > 0);+ if (matched.length === 0) return `Nothing found for ${prompts.map((p) => p.text).join(", ")}.`;+ const parts = matched.map((p) => (p.max_detections > 1 ? `${p.text} x${p.found}` : p.text));+ const missing = prompts.filter((p) => p.found === 0).map((p) => p.text);+ const head = `Found ${matched.length} of ${prompts.length}: ${parts.join(", ")}.`;+ return missing.length ? `${head} Not found: ${missing.join(", ")}.` : head;+}++/** The overlay colour each painter shows the mask in: red for inpaint and the separate mask, white for paint. */+export const MASK_OVERLAY_RGBA: Readonly<Record<"mask" | "paint", readonly [number, number, number, number]>> = {+ mask: [244, 114, 114, 140],+ paint: [255, 255, 255, 255],+};++/** Turn a white-on-black layer's pixels (RGBA, in place copy) into the display overlay: black transparent, white+ * the overlay colour scaled by its luminance. */+export function tintMaskPixels(+ layer: Uint8ClampedArray,+ rgba: readonly [number, number, number, number],+): Uint8ClampedArray<ArrayBuffer> {+ const out = new Uint8ClampedArray(layer.length);+ for (let i = 0; i < layer.length; i += 4) {+ const v = Math.max(layer[i], layer[i + 1], layer[i + 2]);+ if (v === 0) continue;+ out[i] = rgba[0];+ out[i + 1] = rgba[1];+ out[i + 2] = rgba[2];+ out[i + 3] = Math.round((rgba[3] * v) / 255);+ }+ return out;+}diff --git a/studio/frontend/src/features/images/localized-edit-canvas.tsx b/studio/frontend/src/features/images/localized-edit-canvas.tsxindex 01eecd705dc..bb013656398 100644--- a/studio/frontend/src/features/images/localized-edit-canvas.tsx+++ b/studio/frontend/src/features/images/localized-edit-canvas.tsx@@ -11,6 +11,8 @@ import {import type { LocalizedEditMode } from "./api";import { ANNOTATION_COLORS, TRANSPARENCY_CHECKER } from "./edit-conditioning";+import type { AppliedMask } from "./lib/mask-from-text";+import { applyMaskToLayer } from "./mask-apply";/** Draws the localized-edit layer over the source at its NATIVE resolution: an RGBA layer of* coloured strokes for annotate, a white-on-black mask for paint and mask. */@@ -22,6 +24,7 @@ export function LocalizedEditCanvas({resetKey,onLayerChange,onColorsChange,+ applyMask,}: {image: string;mode: LocalizedEditMode;@@ -30,6 +33,8 @@ export function LocalizedEditCanvas({resetKey: number;onLayerChange: (dataUrl: string | null) => void;onColorsChange: (names: string[]) => void;+ /** A "Mask from text" result for the paint and mask modes; each new key is drawn into the layer once. */+ applyMask?: AppliedMask | null;}) {const dispRef = useRef<HTMLCanvasElement | null>(null);const layerRef = useRef<HTMLCanvasElement | null>(null);@@ -83,6 +88,38 @@ export function LocalizedEditCanvas({};}, [image, mode, resetKey, onLayerChange, onColorsChange]);+ // A "Mask from text" result lands in the layer like a stroke, so the brush can still extend or Clear can drop it.+ // Annotate layers are coloured outlines, not a mask, so they never take one.+ // A result from before this canvas mounted (another painter) is not replayed.+ const appliedKey = useRef<number | null>(applyMask?.key ?? null);+ useEffect(() => {+ if (!applyMask || !ready || mode === "annotate") return;+ if (appliedKey.current === applyMask.key) return;+ appliedKey.current = applyMask.key;+ const disp = dispRef.current;+ const layer = layerRef.current;+ if (!disp || !layer) return;+ let live = true;+ applyMaskToLayer({+ layer,+ display: disp,+ dataUrl: applyMask.dataUrl,+ additive: applyMask.additive,+ overlay: mode === "paint" ? "paint" : "mask",+ isCurrent: () => live,+ })+ .then((url) => {+ if (!live || url === null) return;+ // An empty Replace leaves the layer black: nothing is selected, so the page's empty-layer checks still hold.+ dirty.current = !applyMask.empty;+ onLayerChange(applyMask.empty ? null : url);+ })+ .catch(() => {});+ return () => {+ live = false;+ };+ }, [applyMask, ready, mode, onLayerChange]);+const radius = useCallback(() => {const base = Math.min(dims.current.w, dims.current.h) || 1024;// Annotations are outlines, so a fraction of the region brush.diff --git a/studio/frontend/src/features/images/mask-apply.ts b/studio/frontend/src/features/images/mask-apply.tsnew file mode 100644index 00000000000..c7d5f9d0bf5--- /dev/null+++ b/studio/frontend/src/features/images/mask-apply.ts@@ -0,0 +1,60 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Draw a "Mask from text" result into a painter's native-resolution white-on-black layer and rebuild its overlay, so+// the mask stays editable with the same brush. Shared by the inpaint MaskCanvas and LocalizedEditCanvas.++import { MASK_OVERLAY_RGBA, tintMaskPixels } from "./lib/mask-from-text.ts";++function loadMaskImage(src: string): Promise<HTMLImageElement> {+ return new Promise((resolve, reject) => {+ const img = new Image();+ img.onload = () => resolve(img);+ img.onerror = () => reject(new Error("Could not read the mask image."));+ img.src = src;+ });+}++/** Replace (or, with ``additive``, union) ``layer`` with the mask at ``dataUrl``, scaled to the layer's size, then+ * repaint ``display`` from the layer in the ``overlay`` colour. Resolves to the layer as a PNG data URL, or null if+ * either canvas has no 2D context or ``isCurrent`` says the request went stale (Clear, a new source or a newer mask)+ * while the image decoded: the canvases are reused, so a stale result never touches them. */+export async function applyMaskToLayer({+ layer,+ display,+ dataUrl,+ additive,+ overlay,+ isCurrent,+}: {+ layer: HTMLCanvasElement;+ display: HTMLCanvasElement;+ dataUrl: string;+ additive: boolean;+ overlay: "mask" | "paint";+ isCurrent?: () => boolean;+}): Promise<string | null> {+ const img = await loadMaskImage(dataUrl);+ if (isCurrent && !isCurrent()) return null;+ const w = layer.width;+ const h = layer.height;+ const lctx = layer.getContext("2d");+ const dctx = display.getContext("2d");+ if (!lctx || !dctx || w === 0 || h === 0) return null;+ lctx.save();+ if (!additive) {+ lctx.globalCompositeOperation = "source-over";+ lctx.fillStyle = "#000";+ lctx.fillRect(0, 0, w, h);+ }+ // White on black either way: "lighten" keeps whatever is already white, so additive is a union.+ lctx.globalCompositeOperation = "lighten";+ lctx.imageSmoothingEnabled = true;+ lctx.drawImage(img, 0, 0, w, h);+ lctx.restore();+ const pixels = lctx.getImageData(0, 0, w, h);+ const tinted = tintMaskPixels(pixels.data, MASK_OVERLAY_RGBA[overlay]);+ dctx.clearRect(0, 0, w, h);+ dctx.putImageData(new ImageData(tinted, w, h), 0, 0);+ return layer.toDataURL("image/png");+}diff --git a/studio/frontend/src/features/images/mask-from-text-panel.tsx b/studio/frontend/src/features/images/mask-from-text-panel.tsxnew file mode 100644index 00000000000..a9f026830d4--- /dev/null+++ b/studio/frontend/src/features/images/mask-from-text-panel.tsx@@ -0,0 +1,318 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { Cancel01Icon, Download04Icon, Search01Icon } from "@hugeicons/core-free-icons";+import { HugeiconsIcon } from "@hugeicons/react";+import { useCallback, useEffect, useRef, useState } from "react";++import { Button } from "@/components/ui/button";+import { InfoHint } from "@/components/ui/info-hint";+import { Input } from "@/components/ui/input";+import { Progress } from "@/components/ui/progress";+import { Spinner } from "@/components/ui/spinner";+import { Switch } from "@/components/ui/switch";+import { useIsAccountOwner } from "@/features/auth";+import { formatBytes, getHfToken, hfApiToken } from "@/features/hub";+import { toast } from "@/lib/toast";+import { cn } from "@/lib/utils";++import {+ MASK_FROM_TEXT_HINT,+ MASK_FROM_TEXT_LABEL,+ MASK_FROM_TEXT_PLACEHOLDER,+ parseMaskPrompt,+ segmenterDownloadFraction,+ segmenterNeedsDownload,+ segmenterReady,+ segmentSummary,+ type SegmenterStatus,+} from "./lib/mask-from-text";+import {+ cancelSegmenterDownload,+ getSegmenterStatus,+ releaseSegmenter,+ segmentImage,+ startSegmenterDownload,+} from "./mask-segment-api";++/** Segmenter status, read once on mount and every second while a download runs. */+function useSegmenterStatus() {+ const [status, setStatus] = useState<SegmenterStatus | null>(null);+ const mounted = useRef(true);+ const refresh = useCallback(() => {+ getSegmenterStatus()+ .then((next) => {+ if (mounted.current) setStatus(next);+ })+ .catch(() => {});+ }, []);+ const downloading = status?.download.downloading === true;+ useEffect(() => {+ mounted.current = true;+ refresh();+ return () => {+ mounted.current = false;+ };+ }, [refresh]);+ useEffect(() => {+ if (!downloading) return;+ const id = window.setInterval(refresh, 1000);+ return () => window.clearInterval(id);+ }, [downloading, refresh]);+ // Re-read once a download ends, so the notice clears without a reload.+ const wasDownloading = useRef(false);+ useEffect(() => {+ if (wasDownloading.current && !downloading) refresh();+ wasDownloading.current = downloading;+ }, [downloading, refresh]);+ return { status, refresh };+}++function SegmenterDownloadNotice({+ status,+ onChanged,+}: {+ status: SegmenterStatus;+ onChanged: () => void;+}) {+ const [busy, setBusy] = useState(false);+ // The download and cancel routes are owner-only; a managed account is told who can fetch it instead.+ const owner = useIsAccountOwner();+ const { download } = status;+ const fraction = segmenterDownloadFraction(download);+ const size = status.size_bytes ? formatBytes(status.size_bytes) : null;+ const start = async () => {+ setBusy(true);+ try {+ await startSegmenterDownload(hfApiToken(getHfToken()));+ } catch (err) {+ toast.error("Could not download SAM 3", {+ description: err instanceof Error ? err.message : undefined,+ });+ } finally {+ setBusy(false);+ onChanged();+ }+ };+ const cancel = async () => {+ try {+ await cancelSegmenterDownload();+ } catch (err) {+ toast.error(err instanceof Error ? err.message : "Could not cancel the download");+ } finally {+ onChanged();+ }+ };+ return (+ <div+ data-testid="mask-segmenter-download"+ className="flex flex-col gap-1.5 rounded-md border border-border/60 bg-muted/30 px-2.5 py-2 text-ui-11"+ >+ {download.downloading ? (+ <>+ <div className="flex items-center justify-between gap-2">+ <span className="text-muted-foreground">+ {download.bytes_total+ ? `Downloading SAM 3: ${formatBytes(download.bytes_done ?? 0)} of ${formatBytes(download.bytes_total)}`+ : "Downloading SAM 3..."}+ </span>+ <Button+ type="button"+ size="sm"+ variant="ghost"+ className={cn("h-6 gap-1 px-1.5 text-xs", !owner && "hidden")}+ onClick={() => void cancel()}+ >+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />+ Cancel+ </Button>+ </div>+ <Progress+ className="h-1.5"+ value={fraction === null ? undefined : fraction * 100}+ indeterminate={fraction === null}+ />+ </>+ ) : (+ <>+ <span className="text-muted-foreground">+ {`Mask from text uses SAM 3${size ? ` (${size})` : ""}, which is not downloaded yet.`}+ {status.gated ? (+ <>+ {" Its license must be accepted on Hugging Face first"}+ {status.license_url ? (+ <>+ {": "}+ <a+ href={status.license_url}+ target="_blank"+ rel="noreferrer"+ className="underline underline-offset-2"+ >+ {status.repo || "the model page"}+ </a>+ </>+ ) : null}+ {", and your Hugging Face token must be set."}+ </>+ ) : null}+ </span>+ {download.error && !download.cancelled ? (+ <span className="text-destructive">{download.error}</span>+ ) : null}+ {owner ? (+ <Button+ type="button"+ size="sm"+ variant="secondary"+ className="h-7 gap-1.5 self-start text-xs"+ disabled={busy}+ onClick={() => void start()}+ >+ {busy ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Download04Icon} className="size-3.5" />}+ {size ? `Download (${size})` : "Download"}+ </Button>+ ) : (+ <span className="text-muted-foreground">Ask the owner of this Unsloth install to download it.</span>+ )}+ </>+ )}+ </div>+ );+}++/** "Mask from text": finds what the text names in ``image`` and hands the mask to the painter. */+export function MaskFromTextPanel({+ image,+ onMask,+}: {+ image: string;+ onMask: (mask: { dataUrl: string; additive: boolean; empty?: boolean }) => void;+}) {+ const { status, refresh } = useSegmenterStatus();+ const [prompt, setPrompt] = useState("");+ const [additive, setAdditive] = useState(false);+ const [running, setRunning] = useState(false);+ // The last outcome, kept with the source it was for: a new source hides it instead of an effect clearing it.+ const [outcome, setOutcome] = useState<{ image: string; error: string | null; summary: string | null } | null>(+ null,+ );+ const error = outcome?.image === image ? outcome.error : null;+ const summary = outcome?.image === image ? outcome.summary : null;+ const controllerRef = useRef<AbortController | null>(null);+ const latestImage = useRef(image);++ // A new source drops a run still in flight for the old one, so its mask never lands on the new image.+ useEffect(() => {+ latestImage.current = image;+ controllerRef.current?.abort();+ }, [image]);++ // Closing the painter frees the segmenter instead of holding it for its keep-alive.+ useEffect(+ () => () => {+ controllerRef.current?.abort();+ releaseSegmenter().catch(() => {});+ },+ [],+ );++ const ready = segmenterReady(status);+ const concepts = parseMaskPrompt(prompt);+ const canRun = ready && !running && concepts.length > 0;++ const run = async () => {+ if (!canRun) return;+ controllerRef.current?.abort();+ const controller = new AbortController();+ controllerRef.current = controller;+ const source = image;+ setRunning(true);+ setOutcome(null);+ try {+ const result = await segmentImage({ image: source, prompt }, controller.signal);+ if (controller.signal.aborted || latestImage.current !== source) return;+ setOutcome({ image: source, error: null, summary: segmentSummary(result) });+ if (result.instances.length > 0 || additive === false) {+ // Nothing found with Replace still clears the mask: the painter shows exactly what the text selected.+ onMask({ dataUrl: result.mask, additive, empty: result.instances.length === 0 });+ }+ } catch (err) {+ if (controller.signal.aborted) return;+ setOutcome({+ image: source,+ error: err instanceof Error ? err.message : "Could not make a mask from the text.",+ summary: null,+ });+ // A 409 can mean the weights went missing; re-read so the download notice comes back.+ refresh();+ } finally {+ if (controllerRef.current === controller) {+ controllerRef.current = null;+ setRunning(false);+ }+ }+ };++ return (+ <div data-testid="mask-from-text" className="flex flex-col gap-1.5">+ <div className="flex items-center gap-1">+ <label htmlFor="mask-from-text-input" className="text-xs font-medium text-muted-foreground">+ {MASK_FROM_TEXT_LABEL}+ </label>+ <InfoHint>{MASK_FROM_TEXT_HINT}</InfoHint>+ </div>+ {status && !status.supported ? (+ <p className="text-ui-11 leading-snug text-muted-foreground">+ {status.reason || "Mask from text is not available on this install."}+ </p>+ ) : null}+ {segmenterNeedsDownload(status) && status ? (+ <SegmenterDownloadNotice status={status} onChanged={refresh} />+ ) : null}+ <div className="flex gap-2">+ <Input+ id="mask-from-text-input"+ value={prompt}+ placeholder={MASK_FROM_TEXT_PLACEHOLDER}+ disabled={!ready}+ onChange={(e) => setPrompt(e.target.value)}+ onKeyDown={(e) => {+ if (e.key === "Enter") {+ e.preventDefault();+ void run();+ }+ }}+ className="h-8 flex-1 text-xs"+ />+ <Button+ type="button"+ size="sm"+ variant="secondary"+ className="h-8 gap-1.5 text-xs"+ disabled={!canRun}+ onClick={() => void run()}+ >+ {running ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Search01Icon} className="size-3.5" />}+ {running ? "Finding" : "Find"}+ </Button>+ </div>+ <div className="flex items-center gap-2">+ <Switch+ size="sm"+ checked={additive}+ onCheckedChange={setAdditive}+ disabled={!ready}+ aria-label="Add to current mask"+ />+ <span className="text-ui-11 text-muted-foreground">Add to current mask</span>+ </div>+ {error ? <p className="text-ui-11 leading-snug text-destructive">{error}</p> : null}+ {summary && !error ? (+ <p data-testid="mask-from-text-summary" className="text-ui-11 leading-snug text-muted-foreground">+ {summary}+ </p>+ ) : null}+ </div>+ );+}diff --git a/studio/frontend/src/features/images/mask-segment-api.ts b/studio/frontend/src/features/images/mask-segment-api.tsnew file mode 100644index 00000000000..df17fcb484e--- /dev/null+++ b/studio/frontend/src/features/images/mask-segment-api.ts@@ -0,0 +1,67 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// The "Mask from text" segmenter routes, kept out of api.ts like the prompt upsampler's: only the lazily loaded Images+// page uses them.++import { authFetch } from "@/features/auth";+import { readFastApiError } from "@/lib/format-fastapi-error";++import {+ MASK_SEGMENT_ROUTES,+ parseSegmenterStatus,+ parseSegmentResult,+ type SegmenterStatus,+ type SegmentRequest,+ type SegmentResult,+} from "./lib/mask-from-text";++async function parseJson<T>(response: Response): Promise<T> {+ if (!response.ok) {+ throw new Error(await readFastApiError(response));+ }+ return (await response.json()) as T;+}++export async function getSegmenterStatus(signal?: AbortSignal): Promise<SegmenterStatus> {+ return parseSegmenterStatus(+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.status, { signal })),+ );+}++/** Start the background download of the SAM 3 weights. Progress is read back from the status route. */+export async function startSegmenterDownload(hfToken?: string): Promise<void> {+ await parseJson<unknown>(+ await authFetch(MASK_SEGMENT_ROUTES.download, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(hfToken ? { hf_token: hfToken } : {}),+ }),+ );+}++/** Stop the download. Partial files stay cached, so a later download resumes. */+export async function cancelSegmenterDownload(): Promise<void> {+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.cancel, { method: "POST" }));+}++/** The mask for ``prompt`` on ``image``, at the image's own size. */+export async function segmentImage(body: SegmentRequest, signal?: AbortSignal): Promise<SegmentResult> {+ return parseSegmentResult(+ await parseJson<unknown>(+ await authFetch(MASK_SEGMENT_ROUTES.segment, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(body),+ signal,+ }),+ ),+ );+}++/** The painter closed: free the segmenter now instead of holding it for its keep-alive. Best effort. */+export function releaseSegmenter(): Promise<void> {+ return (async () => {+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.release, { method: "POST" }));+ })();+}diff --git a/studio/frontend/tests/image-mask-from-text.test.ts b/studio/frontend/tests/image-mask-from-text.test.tsnew file mode 100644index 00000000000..d7ed33aeae3--- /dev/null+++ b/studio/frontend/tests/image-mask-from-text.test.ts@@ -0,0 +1,265 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { readSrc } from "./helpers/kit.ts";+import {+ MASK_OVERLAY_RGBA,+ MASK_SEGMENT_ROUTES,+ parseMaskPrompt,+ parseSegmenterStatus,+ parseSegmentResult,+ segmenterDownloadFraction,+ segmenterNeedsDownload,+ segmenterReady,+ segmentSummary,+ tintMaskPixels,+} from "../src/features/images/lib/mask-from-text.ts";++const MASK = "data:image/png;base64,iVBORw0KGgo=";++test("every segmenter route sits under the image inference prefix", () => {+ assert.deepEqual(MASK_SEGMENT_ROUTES, {+ status: "/api/inference/images/segment/status",+ download: "/api/inference/images/segment/download",+ cancel: "/api/inference/images/segment/cancel",+ release: "/api/inference/images/segment/release",+ segment: "/api/inference/images/segment",+ });+});++test("the status parse tolerates missing and malformed fields", () => {+ for (const raw of [null, undefined, 3, "x", [], {}]) {+ const s = parseSegmenterStatus(raw);+ assert.equal(s.supported, false);+ assert.equal(s.downloaded, false);+ assert.equal(s.size_bytes, null);+ assert.equal(s.download.downloading, false);+ assert.equal(s.download.bytes_total, null);+ assert.equal(segmenterNeedsDownload(s), false);+ assert.equal(segmenterReady(s), false);+ }+ const s = parseSegmenterStatus({+ supported: true,+ reason: null,+ downloaded: false,+ loaded: false,+ device: null,+ repo: "facebook/sam3",+ gated: true,+ license_url: "https://huggingface.co/facebook/sam3",+ size_bytes: 3_439_938_512,+ download: { downloading: true, download_id: "abc", error: null, cancelled: false, bytes_total: 100, bytes_done: 25 },+ });+ assert.equal(s.repo, "facebook/sam3");+ assert.equal(s.gated, true);+ assert.equal(s.size_bytes, 3_439_938_512);+ assert.equal(s.download.download_id, "abc");+ assert.equal(segmenterDownloadFraction(s.download), 0.25);+ assert.equal(segmenterNeedsDownload(s), true);+ // A download in flight is not ready even once the files report downloaded.+ assert.equal(segmenterReady({ ...s, downloaded: true }), false);+ assert.equal(segmenterReady({ ...s, downloaded: true, download: { ...s.download, downloading: false } }), true);+ // Negative or non-numeric sizes read as unknown.+ assert.equal(parseSegmenterStatus({ size_bytes: -1 }).size_bytes, null);+ assert.equal(segmenterDownloadFraction({ ...s.download, bytes_total: null }), null);+ assert.equal(segmenterDownloadFraction({ ...s.download, bytes_total: 10, bytes_done: 50 }), 1);+});++test("an unsupported install never offers the download", () => {+ const s = parseSegmenterStatus({ supported: false, reason: "Needs transformers 5", downloaded: false });+ assert.equal(s.reason, "Needs transformers 5");+ assert.equal(segmenterNeedsDownload(s), false);+ assert.equal(segmenterReady(s), false);+ assert.equal(segmenterNeedsDownload(null), false);+});++test("the segment result parse keeps well-formed instances and requires a mask", () => {+ const r = parseSegmentResult({+ mask: MASK,+ width: 640,+ height: 480,+ instances: [+ { prompt: "sofa", score: 0.91, box: [1, 2, 300, 200] },+ { prompt: "bad", score: "high", box: [1, 2, 3, 4] },+ { prompt: "bad box", score: 0.5, box: [1, 2] },+ ],+ prompts: [+ { text: "sofa", max_detections: 1, found: 1 },+ { text: "pillow", max_detections: 2, found: 0 },+ { text: "", max_detections: 1, found: 0 },+ ],+ device: "cuda:0",+ elapsed_ms: 412,+ });+ assert.equal(r.width, 640);+ assert.equal(r.instances.length, 1);+ assert.deepEqual(r.instances[0].box, [1, 2, 300, 200]);+ assert.equal(r.prompts.length, 2);+ assert.equal(r.device, "cuda:0");+ assert.throws(() => parseSegmentResult({}), /no mask/);+ assert.throws(() => parseSegmentResult({ mask: "not a data url" }), /no mask/);+});++test("the prompt parser matches ComfyUI's SAM 3 syntax", () => {+ assert.deepEqual(parseMaskPrompt("Sofa:2"), [["Sofa", 2]]);+ assert.deepEqual(parseMaskPrompt("(red lamp), window, pillow:3"), [+ ["red lamp", 1],+ ["window", 1],+ ["pillow", 3],+ ]);+ assert.deepEqual(parseMaskPrompt("a:2.6"), [["a", 3]]);+ assert.deepEqual(parseMaskPrompt("a:0"), [["a", 1]]);+ assert.deepEqual(parseMaskPrompt("a : 4 "), [["a", 4]]);+ assert.deepEqual(parseMaskPrompt(" , sofa,, ,"), [["sofa", 1]]);+ assert.deepEqual(parseMaskPrompt(""), []);+ // A colon not followed by a number stays part of the concept.+ assert.deepEqual(parseMaskPrompt("note: red"), [["note: red", 1]]);+ // Python's round() (half to even), as the backend parses it.+ assert.deepEqual(parseMaskPrompt("a:2.5, b:3.5, c:0.5"), [["a", 2], ["b", 4], ["c", 1]]);+ // Not a number: the whole part is the concept.+ assert.deepEqual(parseMaskPrompt("v:2.5.1"), [["v:2.5.1", 1]]);+ assert.deepEqual(parseMaskPrompt("grain:5000"), [["grain", 200]]);+});++test("the summary names what was found and what was not", () => {+ assert.equal(+ segmentSummary({+ prompts: [+ { text: "sofa", max_detections: 1, found: 1 },+ { text: "pillow", max_detections: 3, found: 2 },+ { text: "lamp", max_detections: 1, found: 0 },+ ],+ instances: [],+ }),+ "Found 2 of 3: sofa, pillow x2. Not found: lamp.",+ );+ assert.equal(+ segmentSummary({ prompts: [{ text: "cat", max_detections: 1, found: 0 }], instances: [] }),+ "Nothing found for cat.",+ );+ assert.equal(segmentSummary({ prompts: [], instances: [] }), "Nothing found.");+ for (const text of ["x"]) {+ const line = segmentSummary({ prompts: [{ text, max_detections: 1, found: 1 }], instances: [] });+ assert.ok(!/—/.test(line));+ }+});++test("the overlay tint keeps black transparent and colours white", () => {+ const layer = new Uint8ClampedArray([0, 0, 0, 255, 255, 255, 255, 255, 128, 128, 128, 255]);+ const out = tintMaskPixels(layer, MASK_OVERLAY_RGBA.mask);+ assert.deepEqual([...out.slice(0, 4)], [0, 0, 0, 0]);+ assert.deepEqual([...out.slice(4, 8)], [244, 114, 114, 140]);+ assert.equal(out[11], 70);+ assert.deepEqual([...tintMaskPixels(layer, MASK_OVERLAY_RGBA.paint).slice(4, 8)], [255, 255, 255, 255]);+});++test("both painters take a text mask and the page renders the panel for inpaint and mask or paint edits", () => {+ const page = readSrc("features/images/images-page.tsx");+ const localized = readSrc("features/images/localized-edit-canvas.tsx");+ const apply = readSrc("features/images/mask-apply.ts");+ assert.match(page, /function MaskCanvas\([\s\S]*?applyMask\?: AppliedMask \| null;/);+ assert.match(localized, /applyMask\?: AppliedMask \| null;/);+ // Not for annotate: its layer is coloured outlines, not a mask.+ assert.match(localized, /mode === "annotate"\) return;/);+ assert.match(page, /onMaskChange=\{setMaskImage\}\s*applyMask=\{textMask\}\s*\/>\s*<\/Field>\s*<MaskFromTextPanel image=\{initImage\}/);+ assert.match(page, /applyMask=\{textMask\}\s*\/>\s*\{localizedMode !== "annotate" && \(\s*<MaskFromTextPanel/);+ // Replace clears to black first; additive unions with "lighten".+ assert.match(apply, /if \(!additive\) \{[\s\S]*?fillRect/);+ assert.match(apply, /globalCompositeOperation = "lighten"/);+ // Each painter replays only results that arrive after it mounted.+ assert.equal((page.match(/useRef<number \| null>\(applyMask\?\.key \?\? null\)/g) ?? []).length, 1);+ assert.equal((localized.match(/useRef<number \| null>\(applyMask\?\.key \?\? null\)/g) ?? []).length, 1);+});++test("the panel releases the segmenter on unmount and uses the shared download idioms", () => {+ const panel = readSrc("features/images/mask-from-text-panel.tsx");+ const api = readSrc("features/images/mask-segment-api.ts");+ assert.match(panel, /releaseSegmenter\(\)\.catch/);+ assert.match(panel, /useIsAccountOwner/);+ assert.match(panel, /hfApiToken\(getHfToken\(\)\)/);+ assert.match(api, /MASK_SEGMENT_ROUTES\.segment/);+ assert.match(api, /readFastApiError/);+ for (const src of [panel, api]) assert.ok(!/—/.test(src), "no em dashes in UI copy");+});++test("a text mask that went stale while it decoded never touches the reused canvases", async () => {+ const { applyMaskToLayer } = await import("../src/features/images/mask-apply.ts");+ const g = globalThis as Record<string, unknown>;+ const saved = { Image: g.Image, ImageData: g.ImageData };+ // The decode resolves on a later tick, as a real <img> does.+ g.Image = class {+ onload: (() => void) | null = null;+ onerror: (() => void) | null = null;+ set src(_v: string) {+ setTimeout(() => this.onload?.(), 0);+ }+ };+ g.ImageData = class {+ data: Uint8ClampedArray;+ constructor(data: Uint8ClampedArray) {+ this.data = data;+ }+ };+ const calls: string[] = [];+ const ctx = new Proxy(+ {},+ {+ get: (_t, key) =>+ key === "getImageData"+ ? () => ({ data: new Uint8ClampedArray(4) })+ : typeof key === "string" && /^[a-z]+[A-Z]|^(save|restore|fillRect|drawImage)$/.test(key)+ ? () => calls.push(key)+ : undefined,+ set: (_t, key) => {+ calls.push(`set:${String(key)}`);+ return true;+ },+ },+ );+ const canvas = () =>+ ({ width: 1, height: 1, getContext: () => ctx, toDataURL: () => "data:image/png;base64,AA==" }) as unknown as HTMLCanvasElement;+ try {+ let live = true;+ const stale = applyMaskToLayer({+ layer: canvas(),+ display: canvas(),+ dataUrl: MASK,+ additive: false,+ overlay: "mask",+ isCurrent: () => live,+ });+ live = false; // Clear, a new source or a newer mask before the decode finished+ assert.equal(await stale, null);+ assert.deepEqual(calls, []);+ const fresh = await applyMaskToLayer({+ layer: canvas(),+ display: canvas(),+ dataUrl: MASK,+ additive: false,+ overlay: "mask",+ isCurrent: () => true,+ });+ assert.equal(fresh, "data:image/png;base64,AA==");+ assert.ok(calls.includes("drawImage") && calls.includes("putImageData"));+ } finally {+ g.Image = saved.Image;+ g.ImageData = saved.ImageData;+ }+ // Both painters pass their effect's liveness.+ assert.match(readSrc("features/images/images-page.tsx"), /overlay: "mask",\s*isCurrent: \(\) => live,/);+ assert.match(readSrc("features/images/localized-edit-canvas.tsx"), /isCurrent: \(\) => live,/);+});++test("a Replace that found nothing leaves both painters holding no mask", () => {+ const panel = readSrc("features/images/mask-from-text-panel.tsx");+ const page = readSrc("features/images/images-page.tsx");+ const localized = readSrc("features/images/localized-edit-canvas.tsx");+ // The canvas still clears (the black PNG is applied), but the result is flagged as selecting nothing...+ assert.match(panel, /onMask\(\{ dataUrl: result\.mask, additive, empty: result\.instances\.length === 0 \}\)/);+ // ...so the logical value stays null and the page's !maskImage / !localizedLayer checks still refuse Generate.+ assert.match(page, /onMaskChange\(applyMask\.empty \? null : url\)/);+ assert.match(localized, /dirty\.current = !applyMask\.empty;\s*onLayerChange\(applyMask\.empty \? null : url\)/);+});