Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make inpaint and localized-edit masks from text with SAM 3 (#14235)

Daniel Hanbff0a25b4bfc
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.py
new file mode 100644
index 00000000000..9e1e26b4395
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_segment.py
@@ -0,0 +1,1031 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Text-prompted masks for the Images page: SAM 3 (Segment Anything with Concepts) turns "sofa, red lamp, pillow:2"
+into a white-on-black mask the inpaint and localized-edit painters load and keep editable.
+
+Behaviour follows ComfyUI's ``SAM3_Detect`` node and its "Image Segmentation (SAM3)" blueprint
+(comfy_extras/nodes_sam3.py, comfy/text_encoders/sam3_clip.py): comma-separated concepts, ``:N`` caps a concept's
+detections (default 1), the image is stretched to 1008x1008 with bilinear resampling, a detection is kept when its
+score > 0.5, the best ``N`` per concept are kept, and the union of their masks (mask logits upsampled bilinearly to
+the source size, > 0) is the result. Two departures: the score is sigmoid(logit) * sigmoid(presence), as in Meta's
+SAM 3 and transformers' post-processing (ComfyUI drops the presence term, and then a concept that is not in the
+picture, "elephant" in a living room, masks the sofa at 0.72; with it every absent concept scores below 0.02 and no
+present one changes), and ComfyUI's optional crop-and-refine pass through the tracker's SAM decoder is not run.
+
+Weights are the official transformers-format ``facebook/sam3`` checkpoint (gated: the license is accepted on Hugging
+Face). SAM 3.1 ships only an original-format multiplex checkpoint with no transformers integration, so it is not
+loaded here; ``UNSLOTH_SAM3_PATH`` points at any local transformers-format SAM 3 directory instead.
+
+The model is auxiliary to the image model, like the FLUX.3 prompt upsampler: loaded on the first request, kept for
+a short keep-alive, unloaded right after a call when the accelerator is tight, and dropped whenever the GPU arbiter
+hands the GPU to a load, evicts the image model, or training starts. It runs on the CPU while training holds the GPU
+or when the accelerator has no room.
+"""
+
+from __future__ import annotations
+
+import base64
+import io
+import json
+import math
+import os
+import re
+import sys
+import threading
+import time
+from contextlib import contextmanager, suppress
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterator, Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+# ── weights ─────────────────────────────────────────────────────────────────────────────────────────────────────────
+SEGMENTER_ID = "sam3"
+SEGMENTER_REPO = "facebook/sam3"
+SEGMENTER_LICENSE_URL = "https://huggingface.co/facebook/sam3"
+SEGMENTER_WEIGHTS_FILE = "model.safetensors"
+# Only what Sam3Model and its tokenizer read: the repo also holds sam3.pt, the same weights in the original format.
+SEGMENTER_FILES = (
+ "config.json",
+ "processor_config.json",
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "vocab.json",
+ "merges.txt",
+ SEGMENTER_WEIGHTS_FILE,
+)
+# fp32 safetensors as published (half of it once loaded in bf16 / fp16), plus the tokenizer and configs.
+SEGMENTER_SIZE_BYTES = 3_439_938_512 + 5_060_000
+ENV_PATH_OVERRIDE = "UNSLOTH_SAM3_PATH"
+ENV_KEEP_ALIVE = "UNSLOTH_SAM3_KEEP_ALIVE"
+
+# ── ComfyUI SAM3_Detect defaults ────────────────────────────────────────────────────────────────────────────────────
+IMAGE_SIZE = 1008
+DEFAULT_THRESHOLD = 0.5
+MAX_CONCEPTS = 16
+# The detector proposes 200 queries per concept.
+MAX_DETECTIONS = 200
+TEXT_MAX_LENGTH = 32
+
+# ── memory ──────────────────────────────────────────────────────────────────────────────────────────────────────────
+KEEP_ALIVE_SECONDS = 120.0
+# bf16 weights (~1.7 GB) plus the 1008x1008 ViT activations and the allocator's slack.
+_GPU_NEED_MIB = 4096
+# After a call, below this much free (or this fraction of the card) the model leaves at once, so the image model
+# never denoises next to it. The prompt upsampler's tight rule.
+_TIGHT_FREE_MIB = 6144
+_TIGHT_FREE_FRACTION = 0.20
+
+
+class SegmenterUnavailableError(RuntimeError):
+ """This install cannot run SAM 3 (transformers without it)."""
+
+
+class SegmenterNotDownloadedError(RuntimeError):
+ """The weights are not in the cache; the UI offers the download."""
+
+
+class SegmenterBusyError(RuntimeError):
+ """The image model is rendering or loading; the GPU is not free for the segmenter."""
+
+
+class SegmenterReleaseTimeoutError(RuntimeError):
+ """A GPU handover waited out a call still running on the accelerator: the card is not free yet."""
+
+
+# ── prompt syntax ───────────────────────────────────────────────────────────────────────────────────────────────────
+@dataclass(frozen = True)
+class Concept:
+ text: str
+ max_detections: int
+
+
+_MAX_SUFFIX = re.compile(r"^(.+?)\s*:\s*([\d.]+)\s*$")
+
+
+def parse_prompt(text: Optional[str]) -> list[Concept]:
+ """ComfyUI's ``_parse_prompts`` (comfy/text_encoders/sam3_clip.py): drop parentheses, split on commas, and read a
+ trailing ``:N`` as that concept's maximum detections (rounded, at least 1; 1 when absent). Duplicates are kept,
+ as ComfyUI keeps them. Capped at MAX_CONCEPTS concepts and MAX_DETECTIONS per concept."""
+ text = (text or "").replace("(", "").replace(")", "")
+ parts = [p.strip() for p in text.split(",") if p.strip()]
+ concepts: list[Concept] = []
+ for part in parts:
+ match = _MAX_SUFFIX.match(part)
+ if match:
+ label = match.group(1).strip()
+ try:
+ count = max(1, round(float(match.group(2))))
+ except (ValueError, OverflowError): # "1.2.3", or hundreds of digits (float -> inf)
+ label, count = part, 1
+ else:
+ label, count = part, 1
+ if label:
+ concepts.append(Concept(label, min(count, MAX_DETECTIONS)))
+ if len(concepts) > MAX_CONCEPTS:
+ raise ValueError(f"At most {MAX_CONCEPTS} concepts per mask (got {len(concepts)}).")
+ return concepts
+
+
+# ── availability and files ──────────────────────────────────────────────────────────────────────────────────────────
+def transformers_supports_sam3() -> bool:
+ """Whether the installed transformers ships SAM 3 (5.x), read from the package layout: the status route must not
+ import the model code, and an attribute probe on transformers' lazy module during another thread's import of it
+ can read as missing."""
+ try:
+ import importlib.util
+
+ # A --no-torch install ships transformers without torch: SAM 3 has the code there and still cannot run.
+ if importlib.util.find_spec("torch") is None:
+ return False
+ return importlib.util.find_spec("transformers.models.sam3.modeling_sam3") is not None
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _override_dir() -> Optional[str]:
+ """The override directory when it holds what Sam3Model and its tokenizer load (a config, the weights, single or
+ sharded, and a tokenizer), else None: a partial or unrelated folder must not read as downloaded."""
+ value = (os.environ.get(ENV_PATH_OVERRIDE) or "").strip()
+ if not value:
+ return None
+ path = Path(value).expanduser()
+ has = lambda *names: any((path / name).is_file() for name in names) # noqa: E731
+ complete = (
+ has("config.json")
+ and (has(SEGMENTER_WEIGHTS_FILE) or _shards_complete(path / (SEGMENTER_WEIGHTS_FILE + ".index.json")))
+ # AutoTokenizer (CLIPTokenizer for sam3) loads tokenizer.json, else needs vocab.json and merges.txt together.
+ and (has("tokenizer.json") or (has("vocab.json") and has("merges.txt")))
+ )
+ return str(path) if complete else None
+
+
+def _shards_complete(index: Path) -> bool:
+ """A sharded checkpoint's index is present and every shard its weight_map names is beside it."""
+ try:
+ weight_map = json.loads(index.read_text(encoding = "utf-8")).get("weight_map")
+ except Exception: # noqa: BLE001 -- missing or unreadable: not a checkpoint
+ return False
+ if not isinstance(weight_map, dict) or not weight_map:
+ return False
+ shards = set(weight_map.values())
+ return all(isinstance(name, str) and (index.parent / name).is_file() for name in shards)
+
+
+def _cached_snapshot(
+ model_id: str = SEGMENTER_ID,
+ *,
+ hub_cache: Optional[Path] = None,
+ revision: Optional[str] = None,
+) -> Optional[str]:
+ """The snapshot directory holding every SEGMENTER_FILES at one revision (the recorded one first), else None."""
+ try:
+ from huggingface_hub import hf_hub_download
+
+ from core.inference.stt_sidecar import (
+ _active_hf_hub_cache,
+ _fallback_revisions,
+ _read_revision_record,
+ )
+
+ root = hub_cache if hub_cache is not None else _active_hf_hub_cache()
+ if revision is not None:
+ revisions = [revision]
+ else:
+ recorded = _read_revision_record(SEGMENTER_REPO)
+ revisions = ([recorded] if recorded else []) + [
+ rev
+ for rev in _fallback_revisions(SEGMENTER_REPO, hub_cache = root)
+ if rev != recorded
+ ]
+ for rev in revisions:
+ paths = []
+ for filename in SEGMENTER_FILES:
+ try:
+ paths.append(
+ hf_hub_download(
+ repo_id = SEGMENTER_REPO,
+ filename = filename,
+ revision = rev,
+ local_files_only = True,
+ cache_dir = str(root),
+ )
+ )
+ except Exception: # noqa: BLE001 -- missing at this revision
+ paths = []
+ break
+ if paths:
+ return str(Path(paths[0]).parent)
+ except Exception: # noqa: BLE001 -- an unreadable cache reads as not downloaded
+ return None
+ return None
+
+
+def model_dir() -> Optional[str]:
+ """The env override when set (a typo'd override is not silently replaced by the cache), else the cached repo."""
+ if (os.environ.get(ENV_PATH_OVERRIDE) or "").strip():
+ return _override_dir()
+ return _cached_snapshot()
+
+
+@dataclass(frozen = True)
+class _SegmenterSpec:
+ repo: str
+ files: tuple
+ label: str
+ model_file: str = SEGMENTER_WEIGHTS_FILE
+ mmproj_file: str = "config.json"
+
+
+SEGMENTER_SPEC = _SegmenterSpec(repo = SEGMENTER_REPO, files = SEGMENTER_FILES, label = "SAM 3 segmenter")
+
+
+def _resolve_segmenter_id(model_id: Optional[str]) -> str:
+ if (model_id or SEGMENTER_ID) != SEGMENTER_ID:
+ raise ValueError(f"Unknown segmenter '{model_id}'.")
+ return SEGMENTER_ID
+
+
+def describe_download_error(_model_id: str, error: object) -> Optional[str]:
+ """A gated-repo refusal names the license to accept; anything else keeps the generic message."""
+ name = type(error).__name__
+ text = str(error)
+ if (
+ name == "GatedRepoError"
+ or "gated" in text.lower()
+ or "403" in text
+ or "401" in text
+ or "awaiting a review" in text
+ ):
+ return (
+ f"{SEGMENTER_REPO} is gated on Hugging Face. Accept the SAM 3 license at {SEGMENTER_LICENSE_URL} "
+ "with your account, add a Hugging Face token in Settings, then download again."
+ )
+ return None
+
+
+_download_state_lock = threading.Lock()
+_download_state = None
+
+
+def _cached_download_paths(
+ model_id: str,
+ *,
+ hub_cache: Optional[Path] = None,
+ revision: Optional[str] = None,
+):
+ path = _cached_snapshot(model_id, hub_cache = hub_cache, revision = revision)
+ return None if path is None else (path, path)
+
+
+def _get_download_state():
+ global _download_state
+ with _download_state_lock:
+ if _download_state is None:
+ from core.inference.stt_mtmd_sidecar import _MtmdDownloadState
+ _download_state = _MtmdDownloadState(
+ specs = {SEGMENTER_ID: SEGMENTER_SPEC},
+ cached_paths = _cached_download_paths,
+ resolve = _resolve_segmenter_id,
+ label = "SAM 3 segmenter",
+ describe_error = describe_download_error,
+ )
+ return _download_state
+
+
+def start_download(hf_token: Optional[str] = None) -> str:
+ return _get_download_state().start(SEGMENTER_ID, hf_token)
+
+
+def download_status() -> dict:
+ return _get_download_state().status()
+
+
+def cancel_download(download_id: Optional[str] = None) -> bool:
+ return _get_download_state().cancel(SEGMENTER_ID, download_id)
+
+
+def status() -> dict:
+ """The status route body: can it run, is it downloaded, is it loaded (and where), and the download progress."""
+ supported = transformers_supports_sam3()
+ path = model_dir() if supported else None
+ segmenter = _segmenter
+ return {
+ "supported": supported,
+ "reason": None
+ if supported
+ else "Mask from text needs PyTorch and transformers 5 (Python 3.10 or newer); this install lacks one of them.",
+ "downloaded": path is not None,
+ "loaded": bool(segmenter is not None and segmenter.loaded),
+ "device": segmenter.device if segmenter is not None and segmenter.loaded else None,
+ "repo": SEGMENTER_REPO,
+ "gated": True,
+ "license_url": SEGMENTER_LICENSE_URL,
+ "size_bytes": SEGMENTER_SIZE_BYTES,
+ "override": bool((os.environ.get(ENV_PATH_OVERRIDE) or "").strip()),
+ "download": download_status(),
+ }
+
+
+# ── device choice ───────────────────────────────────────────────────────────────────────────────────────────────────
+def _resident_cuda_ordinal() -> Optional[int]:
+ """The CUDA card the resident image model sits on, else None."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ ordinal = getattr(
+ getattr(get_active_diffusion_engine(), "_state", None), "placed_ordinal", None
+ )
+ return None if ordinal is None else int(ordinal)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _cuda_free_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
+ try:
+ import torch
+ free, total = torch.cuda.mem_get_info(ordinal)
+ return int(free // (1024 * 1024)), int(total // (1024 * 1024))
+ except Exception: # noqa: BLE001
+ return None, None
+
+
+def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:
+ if free_mib is None:
+ return False
+ floor = _TIGHT_FREE_MIB
+ if total_mib:
+ floor = max(floor, int(total_mib * _TIGHT_FREE_FRACTION))
+ return free_mib < floor
+
+
+def _other_owner_on(ordinal: Optional[int]) -> Optional[str]:
+ """The GPU arbiter's resident other than the image model on CUDA card ``ordinal`` (one with unknown GPUs counts
+ for every card; ``ordinal`` None is Apple's single Metal device, which every resident shares), else None. Called under the run lock after the handover fence: a later handover waits for the
+ call (or aborts) instead."""
+ try:
+ from core.inference import gpu_arbiter
+
+ held = gpu_arbiter.residents()
+ physical = None if ordinal is None else gpu_arbiter.physical_gpu_ids([ordinal])
+ except Exception: # noqa: BLE001 -- no arbiter: nothing else is placed through it
+ return None
+ for owner, gpus in held.items():
+ if owner == gpu_arbiter.DIFFUSION:
+ continue
+ if gpus is None or physical is None or gpus & physical:
+ return owner
+ return None
+
+
+def choose_device(allow_gpu: bool = True) -> tuple[str, Any]:
+ """(torch device string, dtype): the image model's CUDA card (else the current one) in bf16 (fp16 where bf16 is
+ not supported) when it has room for the segmenter, Apple's MPS in fp32, else the CPU in fp32."""
+ import torch
+
+ if allow_gpu and torch.cuda.is_available():
+ ordinal = _resident_cuda_ordinal()
+ if ordinal is None or ordinal >= torch.cuda.device_count():
+ ordinal = torch.cuda.current_device()
+ other = _other_owner_on(ordinal)
+ if other is not None:
+ # A chat / video / other model owns the card: it grows its working memory without knowing about SAM 3.
+ logger.info("images.segment: cuda:%s is held by %s, running on the CPU", ordinal, other)
+ return "cpu", torch.float32
+ free_mib, _total = _cuda_free_mib(ordinal)
+ if free_mib is None or free_mib >= _GPU_NEED_MIB:
+ bf16 = False
+ with suppress(Exception):
+ # is_bf16_supported reads the CURRENT card; the segmenter runs on ``ordinal``.
+ with torch.cuda.device(ordinal):
+ bf16 = bool(torch.cuda.is_bf16_supported())
+ return f"cuda:{ordinal}", torch.bfloat16 if bf16 else torch.float16
+ logger.info("images.segment: %s MiB free on cuda:%s, running on the CPU", free_mib, ordinal)
+ if (
+ allow_gpu
+ and getattr(getattr(torch, "backends", None), "mps", None) is not None
+ and torch.backends.mps.is_available()
+ ):
+ other = _other_owner_on(None)
+ if other is None:
+ return "mps", torch.float32
+ logger.info("images.segment: Metal is held by %s, running on the CPU", other)
+ return "cpu", torch.float32
+
+
+# ── inference ───────────────────────────────────────────────────────────────────────────────────────────────────────
+@dataclass
+class Instance:
+ concept: str
+ score: float
+ box: tuple[float, float, float, float]
+
+
+@dataclass
+class SegmentResult:
+ mask_png: bytes
+ width: int
+ height: int
+ instances: list[Instance] = field(default_factory = list)
+ concepts: list[dict] = field(default_factory = list)
+ device: str = "cpu"
+ elapsed_ms: float = 0.0
+
+ def to_json(self) -> dict:
+ return {
+ "mask": "data:image/png;base64," + base64.b64encode(self.mask_png).decode("ascii"),
+ "width": self.width,
+ "height": self.height,
+ "instances": [
+ {
+ "prompt": i.concept,
+ "score": round(i.score, 4),
+ "box": [round(v, 1) for v in i.box],
+ }
+ for i in self.instances
+ ],
+ "prompts": self.concepts,
+ "device": self.device,
+ "elapsed_ms": round(self.elapsed_ms, 1),
+ }
+
+
+def model_image_size(model: Any) -> int:
+ """The square input the checkpoint was built for (1008 for SAM 3)."""
+ try:
+ return int(model.config.vision_config.backbone_config.image_size)
+ except Exception: # noqa: BLE001
+ return IMAGE_SIZE
+
+
+def preprocess(
+ image: Any,
+ device: str,
+ dtype: Any,
+ size: int = IMAGE_SIZE,
+) -> Any:
+ """ComfyUI's input: RGB in [0, 1] stretched to 1008x1008 with bilinear interpolation (no crop, no antialias),
+ then SAM 3's normalisation (mean 0.5, std 0.5)."""
+ import numpy as np
+ import torch
+ import torch.nn.functional as F
+
+ array = np.array(image.convert("RGB"), dtype = np.uint8)
+ tensor = torch.from_numpy(array).to(device).permute(2, 0, 1).unsqueeze(0).float() / 255.0
+ tensor = F.interpolate(tensor, size = (size, size), mode = "bilinear", align_corners = False)
+ return ((tensor - 0.5) / 0.5).to(dtype)
+
+
+def select_detections(
+ pred_logits: Any,
+ pred_boxes: Any,
+ pred_masks: Any,
+ *,
+ max_detections: int,
+ threshold: float,
+ presence_logit: Any = None,
+) -> tuple[Any, Any, Any]:
+ """ComfyUI's keep rule for one concept: score > threshold, best first, at most ``max_detections``, with the score
+ sigmoid(logit) * sigmoid(presence) when the presence logit is given. Takes one image's (Q,), (Q, 4) normalised
+ xyxy and (Q, h, w) logits; returns the kept scores, boxes, masks."""
+ probs = pred_logits.float().sigmoid()
+ if presence_logit is not None:
+ probs = probs * presence_logit.float().sigmoid().reshape(-1)[0]
+ keep = probs > threshold
+ probs, boxes, masks = probs[keep], pred_boxes[keep], pred_masks[keep]
+ order = probs.argsort(descending = True)[:max_detections]
+ return probs[order], boxes[order], masks[order]
+
+
+def union_mask(masks: Any, height: int, width: int) -> Any:
+ """(N, h, w) mask logits -> one (height, width) bool mask: bilinear upsample, > 0, any."""
+ import torch
+ import torch.nn.functional as F
+
+ if masks is None or masks.shape[0] == 0:
+ return torch.zeros(
+ height, width, dtype = torch.bool, device = None if masks is None else masks.device
+ )
+ union = torch.zeros(height, width, dtype = torch.bool, device = masks.device)
+ # One instance at a time keeps a 4K source to a single full-size float plane.
+ for logits in masks:
+ up = F.interpolate(
+ logits[None, None].float(), size = (height, width), mode = "bilinear", align_corners = False
+ )[0, 0]
+ union |= up > 0
+ return union
+
+
+def mask_to_png(mask: Any) -> bytes:
+ from PIL import Image
+
+ array = (mask.to("cpu").numpy().astype("uint8")) * 255
+ buffer = io.BytesIO()
+ Image.fromarray(array, mode = "L").save(buffer, format = "PNG", optimize = True)
+ return buffer.getvalue()
+
+
+def _free_device_cache(device: Optional[str]) -> None:
+ """Collect what nothing references any more, then hand ``device``'s cached blocks back to the driver."""
+ import gc
+
+ gc.collect()
+ if device and device.startswith("cuda"):
+ with suppress(Exception):
+ import torch
+
+ with torch.cuda.device(device):
+ torch.cuda.empty_cache()
+ elif device == "mps":
+ with suppress(Exception):
+ import torch
+
+ torch.mps.empty_cache()
+
+
+class MaskSegmenter:
+ """One lazily loaded Sam3Model + its tokenizer. ``_run_lock`` serialises loads, calls and unloads."""
+
+ def __init__(self, keep_alive_seconds: Optional[float] = None) -> None:
+ if keep_alive_seconds is None:
+ try:
+ keep_alive_seconds = float(os.environ.get(ENV_KEEP_ALIVE) or KEEP_ALIVE_SECONDS)
+ except ValueError:
+ keep_alive_seconds = KEEP_ALIVE_SECONDS
+ self.keep_alive_seconds = max(0.0, keep_alive_seconds)
+ self._run_lock = threading.Lock()
+ self._state_lock = threading.Lock()
+ self._model = None
+ self._tokenizer = None
+ self._path: Optional[str] = None
+ self.device: Optional[str] = None
+ self._dtype = None
+ self._timer: Optional[threading.Timer] = None
+ self._release_pending = False
+ self._release_epoch = 0
+ # Bumped by every GPU handover's release: a call that queued before one never runs on the accelerator.
+ self._handover_epoch = 0
+ # The device of the call in flight: "" until it is chosen, None when no call runs.
+ self._call_device: Optional[str] = None
+ # Set (under _state_lock) once the call in flight has read its release flag: a release that times out after
+ # that is not honoured by the call, so it waits out the call's short tail itself.
+ self._call_settled = False
+
+ @property
+ def loaded(self) -> bool:
+ return self._model is not None
+
+ def holds_vram(self) -> bool:
+ return self._model is not None and bool(self.device) and self.device != "cpu"
+
+ # -- lifecycle -------------------------------------------------------------------------------------------------
+ def _load(self, path: str, device: str, dtype: Any) -> None:
+ if (
+ self._model is not None
+ and self._path == path
+ and self.device == device
+ and self._dtype == dtype
+ ):
+ return
+ if self._model is not None:
+ self._unload_locked("reload")
+ from transformers import AutoTokenizer, Sam3Model
+
+ started = time.perf_counter()
+ # The tokenizer first: a broken one fails before any weights are placed on the device.
+ tokenizer = AutoTokenizer.from_pretrained(path)
+ model = None
+ try:
+ if device.startswith("cuda"):
+ # Straight onto the card: about a third faster than loading on the CPU and moving it.
+ model = Sam3Model.from_pretrained(path, dtype = dtype, device_map = device)
+ else:
+ model = Sam3Model.from_pretrained(path, dtype = dtype)
+ model.to(device)
+ model.eval()
+ except BaseException:
+ # Weights placed before the failure are referenced nowhere else: drop them and their cached blocks.
+ model = None
+ _free_device_cache(device)
+ raise
+ self._model, self._tokenizer = model, tokenizer
+ self._path, self.device, self._dtype = path, device, dtype
+ logger.info(
+ "images.segment: loaded %s on %s (%s) in %.1fs",
+ path,
+ device,
+ str(dtype).replace("torch.", ""),
+ time.perf_counter() - started,
+ )
+
+ def _unload_locked(self, reason: str) -> bool:
+ with self._state_lock:
+ if self._timer is not None:
+ self._timer.cancel()
+ self._timer = None
+ if self._model is None:
+ return False
+ device = self.device
+ self._model = None
+ self._tokenizer = None
+ self._path = None
+ self.device = None
+ self._dtype = None
+ _free_device_cache(device)
+ logger.info("images.segment: unloaded from %s (%s)", device, reason)
+ return True
+
+
+ def _call_on_accelerator(self) -> bool:
+ """A call is in flight on the GPU / MPS, or has not chosen its device yet."""
+ device = self._call_device
+ return device is not None and device != "cpu"
+
+ def release(
+ self,
+ reason: str = "release",
+ *,
+ wait: bool = False,
+ timeout: float = 30.0,
+ handover: bool = False,
+ ) -> bool:
+ """Unload now if idle. A call in flight unloads when it ends (``wait`` waits up to ``timeout`` for it
+ instead). True when the weights left now. ``handover`` (the GPU is changing hands) raises
+ SegmenterReleaseTimeoutError when the wait ran out on a call still on the accelerator."""
+ with self._state_lock:
+ self._release_epoch += 1
+ if handover:
+ self._handover_epoch += 1
+ acquired = (
+ self._run_lock.acquire(timeout = timeout)
+ if wait
+ else self._run_lock.acquire(blocking = False)
+ )
+ if not acquired:
+ with self._state_lock:
+ self._release_pending = True
+ # Read with the flag: a call that has not settled yet sees the flag and unloads when it ends.
+ honoured = self._call_device is not None and not self._call_settled
+ on_accelerator = self._call_on_accelerator()
+ if wait and not honoured:
+ # The call ended (or is ending) without seeing the flag: its tail is short, so wait it out and unload
+ # here rather than leave the weights resident for the next owner.
+ acquired = self._run_lock.acquire(timeout = _SETTLE_TIMEOUT_SECONDS)
+ if not acquired:
+ if handover and on_accelerator:
+ raise SegmenterReleaseTimeoutError(
+ "A mask from text is still running on the GPU; GPU ownership cannot be transferred yet."
+ )
+ return False
+ try:
+ with self._state_lock:
+ self._release_pending = False
+ return self._unload_locked(reason)
+ finally:
+ self._run_lock.release()
+
+ def _arm_keep_alive(self) -> None:
+ with self._state_lock:
+ if self._timer is not None:
+ self._timer.cancel()
+ epoch = self._release_epoch
+ timer = threading.Timer(self.keep_alive_seconds, self._expire, args = (epoch,))
+ timer.daemon = True
+ self._timer = timer
+ timer.start()
+
+ def _expire(self, epoch: int) -> None:
+ if not self._run_lock.acquire(blocking = False):
+ return # in use: the call re-arms the keep-alive when it ends
+ try:
+ with self._state_lock:
+ if epoch != self._release_epoch and self._model is None:
+ return
+ self._unload_locked("keep-alive expired")
+ finally:
+ self._run_lock.release()
+
+ def _after_call(self) -> None:
+ """Called with ``_run_lock`` held: honour a release that arrived mid-call, drop a GPU copy when the card is
+ tight, else keep it for the keep-alive."""
+ with self._state_lock:
+ pending, self._release_pending = self._release_pending, False
+ self._call_settled = True
+ if pending:
+ self._unload_locked("released during a call")
+ return
+ if self.device and self.device.startswith("cuda"):
+ free_mib, total_mib = _cuda_free_mib(int(self.device.split(":", 1)[1]))
+ if vram_is_tight(free_mib, total_mib):
+ self._unload_locked(f"tight: {free_mib} of {total_mib} MiB free")
+ return
+ if self.keep_alive_seconds <= 0:
+ self._unload_locked("no keep-alive")
+ return
+ self._arm_keep_alive()
+
+ # -- inference -------------------------------------------------------------------------------------------------
+ def segment(
+ self,
+ image: Any,
+ prompt: str,
+ *,
+ threshold: float = DEFAULT_THRESHOLD,
+ allow_gpu: bool = True,
+ path: Optional[str] = None,
+ ) -> SegmentResult:
+ concepts = parse_prompt(prompt)
+ if not concepts:
+ raise ValueError("Describe what to mask, for example: sofa, red lamp, pillow:2")
+ if not 0.0 <= float(threshold) <= 1.0:
+ raise ValueError("threshold must be between 0 and 1.")
+ if not transformers_supports_sam3():
+ raise SegmenterUnavailableError(
+ "This install's transformers has no SAM 3. It needs transformers 5 (Python 3.10 or newer)."
+ )
+ path = path or model_dir()
+ if path is None:
+ raise SegmenterNotDownloadedError(
+ f"The SAM 3 segmenter is not downloaded. Download {SEGMENTER_REPO} first."
+ )
+ queued_at = self._handover_epoch
+ with self._run_lock:
+ if allow_gpu and (self._handover_epoch != queued_at or _gpu_handover_in_progress()):
+ # The GPU changed hands while this call waited, or is changing hands now: the new owner may still be
+ # loading onto the card.
+ raise SegmenterBusyError("The GPU is changing hands.")
+ with self._state_lock:
+ self._call_device, self._call_settled = "", False
+ try:
+ return self._segment_locked(image, concepts, threshold, path, allow_gpu)
+ finally:
+ with self._state_lock:
+ self._call_device, self._call_settled = None, False
+
+ def _segment_locked(
+ self, image: Any, concepts: list[Concept], threshold: float, path: str, allow_gpu: bool
+ ) -> SegmentResult:
+ started = time.perf_counter()
+ device, dtype = choose_device(allow_gpu)
+ self._call_device = device
+ oom = False
+ try:
+ result = self._segment_on(image, concepts, threshold, path, device, dtype)
+ except Exception as exc:
+ if not _is_oom(exc) or device == "cpu":
+ self._after_call()
+ raise
+ oom = True
+ if oom:
+ # Out of the except block: the OOM's traceback (and the frames pinning the failed attempt's tensors) is
+ # gone, so the unload and the cache release below actually free the card before the call reads CPU-only.
+ logger.warning("images.segment: out of memory on %s, retrying on the CPU", device)
+ self._unload_locked("out of memory")
+ _free_device_cache(device) # also when the OOM hit mid-load, before any model was kept
+ import torch
+
+ device, dtype = "cpu", torch.float32
+ self._call_device = device
+ try:
+ result = self._segment_on(image, concepts, threshold, path, device, dtype)
+ except Exception:
+ # The CPU copy and any release asked for mid-call are settled here too, not left resident.
+ self._after_call()
+ raise
+ result.elapsed_ms = (time.perf_counter() - started) * 1000.0
+ self._after_call()
+ return result
+
+ def _segment_on(
+ self,
+ image: Any,
+ concepts: list[Concept],
+ threshold: float,
+ path: str,
+ device: str,
+ dtype: Any,
+ ) -> SegmentResult:
+ import torch
+
+ self._load(path, device, dtype)
+ model, tokenizer = self._model, self._tokenizer
+ width, height = image.size
+ scale = torch.tensor([width, height, width, height], dtype = torch.float32)
+ union = torch.zeros(height, width, dtype = torch.bool, device = device)
+ instances: list[Instance] = []
+ summary: list[dict] = []
+ with torch.inference_mode():
+ pixels = preprocess(image, device, dtype, model_image_size(model))
+ vision = model.get_vision_features(pixel_values = pixels)
+ # Duplicates are encoded once; each concept keeps its own cap, as ComfyUI runs them one by one.
+ texts = list(dict.fromkeys(c.text for c in concepts))
+ tokens = tokenizer(
+ texts,
+ padding = "max_length",
+ max_length = TEXT_MAX_LENGTH,
+ truncation = True,
+ return_tensors = "pt",
+ ).to(device)
+ # Only each text's kept detections (best first, up to its largest cap) outlive its forward, never the raw
+ # output's 200 mask planes; a repeat with a smaller cap takes the best of them, as its own run would.
+ caps: dict[str, int] = {}
+ for concept in concepts:
+ caps[concept.text] = max(caps.get(concept.text, 0), concept.max_detections)
+ # Per text only its kept scores and boxes outlive the forward: the union of a text's best-``cap`` masks
+ # covers every occurrence's best-``k`` (k <= cap), so the masks are OR-ed in at once and dropped.
+ kept: dict[str, tuple[Any, Any]] = {}
+ for concept in concepts:
+ if concept.text not in kept:
+ index = texts.index(concept.text)
+ out = model(
+ vision_embeds = vision,
+ input_ids = tokens["input_ids"][index : index + 1],
+ attention_mask = tokens["attention_mask"][index : index + 1],
+ )
+ presence = getattr(out, "presence_logits", None)
+ scores, boxes, masks = select_detections(
+ out.pred_logits[0],
+ out.pred_boxes[0],
+ out.pred_masks[0],
+ max_detections = caps[concept.text],
+ threshold = threshold,
+ presence_logit = None if presence is None else presence[0],
+ )
+ del out, presence
+ union |= union_mask(masks, height, width)
+ del masks
+ kept[concept.text] = (scores, boxes)
+ scores, boxes = (t[: concept.max_detections] for t in kept[concept.text])
+ boxes_px = (boxes.float().cpu() * scale).tolist()
+ for score, box in zip(scores.tolist(), boxes_px):
+ x1, y1, x2, y2 = (float(v) for v in box)
+ instances.append(
+ Instance(
+ concept.text,
+ float(score),
+ (
+ max(0.0, x1),
+ max(0.0, y1),
+ min(float(width), x2),
+ min(float(height), y2),
+ ),
+ )
+ )
+ summary.append(
+ {
+ "text": concept.text,
+ "max_detections": concept.max_detections,
+ "found": int(scores.shape[0]),
+ }
+ )
+ del kept, vision, pixels
+ return SegmentResult(
+ mask_png = mask_to_png(union),
+ width = width,
+ height = height,
+ instances = instances,
+ concepts = summary,
+ device = device,
+ )
+
+
+# How long a timed-out release waits for a call that already settled (unload / keep-alive, then the lock) to let go.
+_SETTLE_TIMEOUT_SECONDS = 5.0
+
+
+def _gpu_handover_in_progress() -> bool:
+ """The arbiter is handing the GPU to a new owner (its load still registering). Never imports the arbiter."""
+ arbiter = sys.modules.get("core.inference.gpu_arbiter")
+ probe = getattr(arbiter, "handover_in_progress", None)
+ try:
+ return bool(probe()) if callable(probe) else False
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _is_oom(exc: BaseException) -> bool:
+ name = type(exc).__name__
+ return name in ("OutOfMemoryError",) or "out of memory" in str(exc).lower()
+
+
+_segmenter: Optional[MaskSegmenter] = None
+_segmenter_lock = threading.Lock()
+
+
+def get_segmenter() -> MaskSegmenter:
+ global _segmenter
+ with _segmenter_lock:
+ if _segmenter is None:
+ _segmenter = MaskSegmenter()
+ return _segmenter
+
+
+def release_segmenter(
+ reason: str = "release",
+ *,
+ wait: bool = False,
+ handover: bool = False,
+) -> bool:
+ """Drop the weights if they were ever loaded. Never creates the segmenter. ``handover``: see
+ MaskSegmenter.release."""
+ segmenter = _segmenter
+ return segmenter is not None and segmenter.release(reason, wait = wait, handover = handover)
+
+
+_renders_pending = 0
+_renders_lock = threading.Lock()
+
+
+@contextmanager
+def render_fence() -> Iterator[None]:
+ """Held by an image render from before it releases the segmenter until it ends: no Find starts meanwhile."""
+ global _renders_pending
+ with _renders_lock:
+ _renders_pending += 1
+ try:
+ yield
+ finally:
+ with _renders_lock:
+ _renders_pending -= 1
+
+
+def _take_render_slot() -> tuple[Any, str]:
+ """(the active image engine's generation lock, acquired) or (None, why not). ``generating`` while a render or
+ a model transition holds it, ``loading`` while a load is in flight; an engine with no lock (or none readable)
+ runs nothing on torch, so the call proceeds with no slot. Never blocks."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ engine = get_active_diffusion_engine()
+ except Exception: # noqa: BLE001 -- no engine: nothing to collide with
+ return None, ""
+ loading = getattr(engine, "_loading", None)
+ if loading is not None and not getattr(loading, "error", None):
+ return None, "loading"
+ lock = getattr(engine, "_generate_lock", None)
+ if lock is None:
+ return None, ""
+ if not lock.acquire(blocking = False):
+ return None, "generating"
+ loading = getattr(engine, "_loading", None)
+ if loading is not None and not getattr(loading, "error", None):
+ lock.release()
+ return None, "loading"
+ return lock, ""
+
+
+def segment_image(
+ image_b64: str,
+ prompt: str,
+ *,
+ threshold: float = DEFAULT_THRESHOLD,
+ allow_gpu: bool = True,
+) -> dict:
+ """The segment route's work: decode the upload (EXIF-upright, as inpaint decodes it), refuse while the image
+ model renders or loads, run SAM 3, answer the route body."""
+ from core.inference.diffusion import decode_b64_image
+
+ if not parse_prompt(prompt):
+ raise ValueError("Describe what to mask, for example: sofa, red lamp, pillow:2")
+ image = decode_b64_image(image_b64, mode = "RGB")
+ slot, why = _take_render_slot()
+ if not why and _renders_pending:
+ # A render released SAM 3 and is on its way to the slot: a Find taking it first would reload SAM 3 under it.
+ if slot is not None:
+ slot.release()
+ slot, why = None, "generating"
+ if why:
+ raise SegmenterBusyError(
+ "The image model is generating."
+ if why == "generating"
+ else "An image model is loading."
+ )
+ try:
+ return (
+ get_segmenter()
+ .segment(image, prompt, threshold = threshold, allow_gpu = allow_gpu)
+ .to_json()
+ )
+ finally:
+ if slot is not None:
+ slot.release()
+
+
+__all__ = [
+ "Concept",
+ "DEFAULT_THRESHOLD",
+ "MaskSegmenter",
+ "SEGMENTER_REPO",
+ "SegmenterBusyError",
+ "SegmenterNotDownloadedError",
+ "SegmenterReleaseTimeoutError",
+ "SegmenterUnavailableError",
+ "get_segmenter",
+ "parse_prompt",
+ "release_segmenter",
+ "segment_image",
+ "status",
+]
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 9ad0c200a6b..b5a5471738f 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -19,8 +19,9 @@ from __future__ import annotations
import sys
import threading
+from contextlib import contextmanager, nullcontext
from contextvars import ContextVar
-from typing import Any, Callable, Iterable, Optional
+from typing import Any, Callable, Iterable, Iterator, Optional
from loggers import get_logger
@@ -191,6 +192,7 @@ def _evict_diffusion() -> None:
_cancel_prompt_upsampler_warm_start()
get_active_diffusion_engine().unload()
_stop_prompt_upsampler()
+ _release_mask_segmenter("image model evicted", handover = True)
def _cancel_prompt_upsampler_warm_start() -> None:
@@ -215,6 +217,59 @@ def _stop_prompt_upsampler() -> None:
logger.warning("gpu_arbiter: could not stop the FLUX.3 prompt upsampler: %s", exc)
+class SegmenterStillRunningError(RuntimeError):
+ """A mask from text still runs on the GPU after the release's wait: the GPU cannot change hands yet.
+ ``blocks_training`` makes the core trainer abort its start on it rather than log and continue."""
+
+ blocks_training = True
+
+
+def _release_mask_segmenter(reason: str, *, handover: bool = False) -> None:
+ """The Images text-to-mask segmenter (SAM 3) is auxiliary: it leaves whenever the GPU changes hands, an image model
+ is evicted or released, or training starts. Only if Images ever loaded it; a call in flight finishes first.
+ ``handover``: a call still on the GPU after the wait aborts the transfer, as an engine that did not stop does."""
+ module = sys.modules.get("core.inference.diffusion_segment")
+ if module is None:
+ return
+ try:
+ module.release_segmenter(reason, wait = True, handover = handover)
+ except Exception as exc: # noqa: BLE001 - never block an acquire on anything else
+ if handover and type(exc).__name__ == "SegmenterReleaseTimeoutError":
+ raise SegmenterStillRunningError(
+ "A mask from text is still running on the GPU; GPU ownership cannot be transferred."
+ ) from exc
+ logger.warning("gpu_arbiter: could not release the mask segmenter: %s", exc)
+
+
+_handovers = 0
+
+
+@contextmanager
+def _handover_fence() -> Iterator[None]:
+ global _handovers
+ _handovers += 1
+ try:
+ yield
+ finally:
+ _handovers -= 1
+
+
+def handover_in_progress() -> bool:
+ """An acquire is handing the GPU to a new owner (evicting, or its load registering). Lock-free read."""
+ return _handovers > 0
+
+
+def release_mask_segmenter(reason: str) -> None:
+ """Release the text-to-mask segmenter for a training start: an in-flight call finishes first, and one still on the
+ GPU after the wait raises SegmenterStillRunningError so the start is refused instead of spawning beside it."""
+ try:
+ _release_mask_segmenter(reason, handover = True)
+ except SegmenterStillRunningError as exc:
+ raise SegmenterStillRunningError(
+ "A mask from text is still running on the GPU; try again when it finishes."
+ ) from exc
+
+
def _release_idle_video_servers(reason: str) -> None:
try:
from core.inference.video_minimax_h3 import release_h3_native_servers
@@ -616,7 +671,9 @@ def acquire_for(
from utils.account_context import current_account_id
acting = account_id or current_account_id()
- with _lock:
+ # The segmenter's VRAM goes to this owner: a mask call arriving before its register returns stays off the card.
+ handover = owner != DIFFUSION or register is not None or replacing
+ with _lock, (_handover_fence() if handover else nullcontext()):
if expected_current is not None and (_owner, _owner_epoch) != expected_current:
raise OwnerChangedError("The resident GPU model changed; retry the load.")
coreside = policy != "off" and owner in _KEPT_UNDER_ALWAYS
@@ -672,6 +729,9 @@ def acquire_for(
if owner != VIDEO and not (coreside and VIDEO in _residents):
# A resident H3 sd-server sits outside every owner's teardown.
_release_idle_video_servers(f"GPU acquired for {owner}")
+ if owner != DIFFUSION or register is not None or replacing:
+ # A load (or another kind taking the GPU) gets the segmenter's VRAM; an image model's re-assert keeps it.
+ _release_mask_segmenter(f"GPU acquired for {owner}", handover = True)
# Records who LOADED the model; a plain re-assert must not hand it to whoever asked last.
# A resident parked behind a later claimant is a re-assert, not a new load.
rejoins = prev_owner != owner and owner in _residents and parked
@@ -820,6 +880,9 @@ def release(owner: str) -> None:
with _lock:
if owner == DIFFUSION and owner in _view():
_stop_prompt_upsampler()
+ if owner == DIFFUSION:
+ # Training releases DIFFUSION before it starts, resident or not: the segmenter goes too.
+ _release_mask_segmenter("image GPU released")
_drop_resident(owner)
@@ -835,6 +898,7 @@ def release_if(owner: str, predicate: Callable[[], bool]) -> bool:
return False
if owner == DIFFUSION:
_stop_prompt_upsampler()
+ _release_mask_segmenter("image GPU released")
_drop_resident(owner)
return True
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index ab6e113a839..6b10ef4dfdf 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -317,7 +317,8 @@ class _MtmdDownloadState:
"""Tracks the two-file download of one mtmd dictation model.
``specs`` / ``cached_paths`` / ``resolve`` let another llama.cpp sidecar (the FLUX.3 prompt upsampler) reuse the
- same pinned two-file download; None keeps the dictation catalogue, looked up at call time.
+ same pinned two-file download; None keeps the dictation catalogue, looked up at call time. A spec with a ``files``
+ tuple downloads those files instead of its model and projector (the Images text-to-mask segmenter).
"""
def __init__(
@@ -326,11 +327,14 @@ class _MtmdDownloadState:
cached_paths = None,
resolve = None,
label: str = "mtmd STT",
+ describe_error = None,
) -> None:
self._specs = specs
self._cached_paths_fn = cached_paths
self._resolve_fn = resolve
self._label = label
+ # (model_id, exception or worker stderr) -> a message for the UI, or None for the generic one.
+ self._describe_error_fn = describe_error
self._lock = threading.Lock()
self._thread: Optional[threading.Thread] = None
self._process: Optional[subprocess.Popen] = None
@@ -350,6 +354,14 @@ class _MtmdDownloadState:
def _resolve(self, model_id: Optional[str]) -> str:
return (self._resolve_fn or resolve_mtmd_model_id)(model_id)
+ def _describe_error(self, model_id: str, error: object) -> Optional[str]:
+ if self._describe_error_fn is None:
+ return None
+ try:
+ return self._describe_error_fn(model_id, error)
+ except Exception: # noqa: BLE001 -- the generic message still reaches the UI
+ return None
+
def _cached_paths(self, model_id: str, **kwargs):
return (self._cached_paths_fn or _cached_model_paths)(model_id, **kwargs)
@@ -496,7 +508,7 @@ class _MtmdDownloadState:
selected: list[_SelectedHubFile] = []
revision: Optional[str] = None
- for filename in (spec.model_file, spec.mmproj_file):
+ for filename in getattr(spec, "files", None) or (spec.model_file, spec.mmproj_file):
meta = get_hf_file_metadata(
hf_hub_url(spec.repo, filename, revision = revision),
token = normalize_token(hf_token),
@@ -573,12 +585,20 @@ class _MtmdDownloadState:
detail = stderr.decode("utf-8", "replace").strip()
logger.warning("%s download failed for %s: %s", self._label, model_id, detail)
with self._lock:
- self._error = modelscope_missing(detail) or f"Download failed for '{model_id}'."
+ self._error = (
+ self._describe_error(model_id, detail)
+ or modelscope_missing(detail)
+ or f"Download failed for '{model_id}'."
+ )
except Exception as exc:
with self._lock:
if not self._cancelled:
logger.warning("%s download failed for %s: %s", self._label, model_id, exc)
- self._error = modelscope_missing(exc) or f"Download failed for '{model_id}'."
+ self._error = (
+ self._describe_error(model_id, exc)
+ or modelscope_missing(exc)
+ or f"Download failed for '{model_id}'."
+ )
finally:
# Release first: it is the half that would wedge the repository.
if registry is not None and owner is not None:
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index e324faacbaf..547e72a8ca7 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5308,6 +5308,38 @@ class PromptUpsamplerDownloadRequest(BaseModel):
hf_token: Optional[str] = Field(None, description = "Hugging Face token for the download")
+class MaskSegmenterDownloadRequest(BaseModel):
+ """Body of POST /images/segment/download (optional)."""
+
+ hf_token: Optional[str] = Field(
+ None, description = "Hugging Face token for the gated facebook/sam3 download"
+ )
+
+
+class MaskSegmentRequest(BaseModel):
+ """Make an inpaint / localized-edit mask from text with SAM 3."""
+
+ image: str = Field(
+ ...,
+ min_length = 1,
+ max_length = 32 * 1024 * 1024,
+ description = "Base64/data-URL source image (as init_image on /images/generate)",
+ )
+ prompt: str = Field(
+ ...,
+ min_length = 1,
+ max_length = 2000,
+ description = "Comma-separated concepts; a trailing :N keeps up to N of that concept (default 1), "
+ "e.g. 'sofa, red lamp, pillow:2'",
+ )
+ threshold: float = Field(
+ 0.5,
+ ge = 0.0,
+ le = 1.0,
+ description = "Detection score threshold (ComfyUI SAM3 Detect's default 0.5)",
+ )
+
+
class PromptUpsamplerPreviewRequest(BaseModel):
"""Enhance one prompt with the FLUX.3 prompt upsampler without generating."""
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 33dba74d875..d283b534462 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3796,6 +3796,8 @@ from models.inference import (
DiffusionLoadRequest,
DiffusionGenerateRequest,
DiffusionGenerateResponse,
+ MaskSegmentRequest,
+ MaskSegmenterDownloadRequest,
PromptUpsamplerDownloadRequest,
PromptUpsamplerPreviewRequest,
PromptUpsamplerPreviewResponse,
@@ -51703,6 +51705,17 @@ async def _generate_diffusion_image_core(
request: DiffusionGenerateRequest,
) -> DiffusionGenerateResponse:
"""The local-engine image run, shared by the route and the chat generate_image tool."""
+ from core.inference.diffusion_segment import render_fence
+
+ # From before the run releases the mask segmenter until it ends: a Find cannot reload SAM 3 in the gap before the
+ # render takes the engine's generation slot.
+ with render_fence():
+ return await _generate_diffusion_image_run(request)
+
+
+async def _generate_diffusion_image_run(
+ request: DiffusionGenerateRequest,
+) -> DiffusionGenerateResponse:
from core.inference import image_gallery
from core.inference.diffusion_engine_router import get_active_diffusion_engine
from core.inference.diffusion_families import (
@@ -51738,6 +51751,8 @@ async def _generate_diffusion_image_core(
# A warm start still launching holds the engine's generation slot: a run that is not using the upsampler does
# not wait it out.
_cancel_upsampler_warm_start()
+ # As the upsampler leaves before the DiT denoises, so does the mask segmenter (Find's SAM 3 on its keep-alive).
+ await asyncio.to_thread(_release_mask_segmenter_for_render)
result = None
for attempt in range(2):
request = submitted
@@ -52971,6 +52986,17 @@ def _cancel_upsampler_warm_start() -> None:
logger.warning("flux3.prompt_upsampler: could not cancel a warm start: %s", exc)
+def _release_mask_segmenter_for_render() -> None:
+ """Drop the text-to-mask segmenter's weights before a render: a resident image model renders without an arbiter
+ acquire, so no handover would unload it. Only if Images ever loaded it; a Find in flight finishes first."""
+ module = sys.modules.get("core.inference.diffusion_segment")
+ if module is not None:
+ try:
+ module.release_segmenter("image generation", wait = True)
+ except Exception as exc: # noqa: BLE001 -- never fail a render on this
+ logger.warning("images.segment: could not release before a render: %s", exc)
+
+
@studio_router.post("/images/prompt-upsampler/release")
async def prompt_upsampler_release(current_subject: str = Depends(get_current_subject)):
"""Enhance prompt was turned off: cancel a warm start still launching and stop an idle server instead of holding it
@@ -53065,6 +53091,112 @@ async def prompt_upsampler_preview(
return PromptUpsamplerPreviewResponse(enhanced_prompt = caption)
+# ── Text-to-mask segmenter (SAM 3) ───────────────────────────────────────────────────────────────────────────────────
+
+
+@studio_router.get("/images/segment/status")
+async def mask_segmenter_status(current_subject: str = Depends(get_current_subject)):
+ """Whether SAM 3 can run here, is downloaded (gated facebook/sam3), is loaded and where, and the download
+ progress (the dictation download's shape)."""
+ from core.inference import diffusion_segment
+ return JSONResponse(content = await asyncio.to_thread(diffusion_segment.status))
+
+
+@studio_router.post("/images/segment/download")
+async def mask_segmenter_download(
+ payload: Optional[MaskSegmenterDownloadRequest] = None,
+ current_subject: str = Depends(get_current_subject),
+ hf_token: HfTokenArg = Depends(get_request_hf_token),
+):
+ """Start the background download of facebook/sam3's transformers weights; progress on /status."""
+ from core.inference import diffusion_segment
+ from core.inference.stt_sidecar import SttDownloadCacheError, SttModelIdError
+
+ account_access.require_installation_owner()
+ if not diffusion_segment.transformers_supports_sam3():
+ raise HTTPException(status_code = 501, detail = diffusion_segment.status()["reason"])
+ if diffusion_segment.status().get("override"):
+ raise HTTPException(
+ status_code = 409,
+ detail = "The segmenter weights come from UNSLOTH_SAM3_PATH; unset it to download facebook/sam3.",
+ )
+ token = account_access.account_hf_token(
+ (payload.hf_token if payload is not None and payload.hf_token else None) or hf_token
+ )
+ try:
+ download_id = await asyncio.to_thread(diffusion_segment.start_download, token)
+ except (SttModelIdError, SttDownloadCacheError) as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ return JSONResponse(content = {**diffusion_segment.download_status(), "download_id": download_id})
+
+
+@studio_router.post("/images/segment/cancel")
+async def mask_segmenter_cancel(current_subject: str = Depends(get_current_subject)):
+ """Stop the segmenter download; partial files stay cached, so the next download resumes."""
+ from core.inference import diffusion_segment
+
+ account_access.require_installation_owner()
+ cancelled = await asyncio.to_thread(diffusion_segment.cancel_download)
+ return JSONResponse(content = {**diffusion_segment.download_status(), "cancelled": cancelled})
+
+
+@studio_router.post("/images/segment/release")
+async def mask_segmenter_release(current_subject: str = Depends(get_current_subject)):
+ """The mask painter closed: drop the segmenter now instead of holding it for the keep-alive. A call in flight
+ unloads when it ends. Never an error."""
+ from core.inference import diffusion_segment
+ from auth import policy
+
+ if policy.installation_has_managed_accounts():
+ return {"released": False}
+ released = await asyncio.to_thread(diffusion_segment.release_segmenter, "mask painter closed")
+ return {"released": bool(released)}
+
+
+@studio_router.post("/images/segment")
+@account_access.gpu_busy_route
+async def mask_segment(
+ request: MaskSegmentRequest, current_subject: str = Depends(get_current_subject)
+):
+ """A white-on-black mask (PNG data URL, source size) of what ``prompt`` names in ``image``, plus each kept
+ instance's box and score. 409 when not downloaded or the image model is busy, 501 without SAM 3 support."""
+ from core.inference import diffusion_segment as seg
+
+ account_access.require_idle_other_accounts()
+ # Training holds the GPU: the segmenter runs on the CPU rather than beside it. On the GPU the diffusion trainer's
+ # admission is held for the call, so a start cannot free the GPU under it.
+ allow_gpu = not _training_is_active()
+ admission = _diffusion_training_admission() if allow_gpu else nullcontext()
+ try:
+ with account_access.media_generation("diffusion"), admission:
+ body = await asyncio.to_thread(
+ functools.partial(
+ seg.segment_image,
+ request.image,
+ request.prompt,
+ threshold = request.threshold,
+ allow_gpu = allow_gpu,
+ )
+ )
+ except seg.SegmenterNotDownloadedError as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ except seg.SegmenterUnavailableError as exc:
+ raise HTTPException(status_code = 501, detail = str(exc))
+ except seg.SegmenterBusyError as exc:
+ raise HTTPException(status_code = 409, detail = f"{exc} Try again when it finishes.")
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc))
+ except HTTPException:
+ raise
+ except Exception as exc: # noqa: BLE001
+ if type(exc).__name__ == "TrainingActiveError":
+ # A diffusion training start won the race for the admission.
+ raise HTTPException(status_code = 409, detail = str(exc))
+ logger.error("images.segment_failed: %s", exc, exc_info = True)
+ raise HTTPException(status_code = 500, detail = "Making the mask failed.")
+ return JSONResponse(content = body)
+
+
# ──────────────────────────────────────────────────────────────────────────
# OpenAI-compatible images API (POST /v1/images/generations). The inference router is mounted at both /api/inference and /v1, so this
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index fc84de953d2..c379da6a4bf 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -2437,6 +2437,8 @@ async def start_training(
# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the diffusers backend
# reports unloaded while the native engine still holds state.
+ # The mask segmenter sits on torch's card whichever engine renders; one still running refuses the start.
+ gpu_arbiter.release_mask_segmenter("training start")
diffusion = get_active_diffusion_engine()
if getattr(diffusion, "runs_off_torch_device", False) is True:
logger.info("Keeping the Images model: it runs outside torch's GPUs")
@@ -2447,6 +2449,8 @@ async def start_training(
)
diffusion.unload()
gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ except gpu_arbiter.SegmenterStillRunningError:
+ raise
except Exception as e:
logger.warning("Could not unload diffusion model for training: %s", e)
@@ -2502,6 +2506,7 @@ async def start_training(
logger.warning("Inference/training memory coordination failed; proceeding: %s", e)
# The hook runs only once start guards pass -> VRAM freed iff training starts.
+ from core.inference.gpu_arbiter import SegmenterStillRunningError
from routes.training_vram import ManagedEngineStillRunning
from utils.transformers_version import SidecarSwapInProgress
@@ -2518,6 +2523,7 @@ async def start_training(
SidecarSwapInProgress,
ExactResumeResourcesUnavailable,
ManagedEngineStillRunning,
+ SegmenterStillRunningError,
) as exc:
_reject_start_request(backend, reserved_start_request_id, str(exc))
raise
@@ -2575,7 +2581,11 @@ async def start_training(
except SidecarSwapInProgress as exc:
# Expected loss of the race against a sidecar install: a retryable 409, not an internal error.
raise HTTPException(status_code = 409, detail = str(exc))
- except (ExactResumeResourcesUnavailable, ManagedEngineStillRunning) as exc:
+ except (
+ ExactResumeResourcesUnavailable,
+ ManagedEngineStillRunning,
+ SegmenterStillRunningError,
+ ) as exc:
raise HTTPException(status_code = 409, detail = str(exc))
if not success:
@@ -3479,6 +3489,8 @@ def _free_gpu_for_diffusion_training() -> None:
# The ACTIVE engine, not the diffusers singleton: on a native (sd_cpp) selection the resident sd-server still
# holds the GPU, so unloading only the singleton is a no-op.
+ # The mask segmenter sits on torch's card whichever engine renders; one still running refuses the start.
+ gpu_arbiter.release_mask_segmenter("diffusion training start")
diffusion = get_active_diffusion_engine()
if getattr(diffusion, "runs_off_torch_device", False) is True:
logger.info("Keeping the Images model: it runs outside torch's GPUs")
@@ -3487,6 +3499,8 @@ def _free_gpu_for_diffusion_training() -> None:
logger.info("Unloading resident Images pipeline to free GPU memory for training")
diffusion.unload()
gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ except gpu_arbiter.SegmenterStillRunningError:
+ raise # a RuntimeError: the start route answers 409
except Exception as e: # noqa: BLE001
logger.warning("Could not unload Images pipeline for diffusion training: %s", e)
diff --git a/studio/backend/tests/test_diffusion_segment.py b/studio/backend/tests/test_diffusion_segment.py
new file mode 100644
index 00000000000..7a3f911c572
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_segment.py
@@ -0,0 +1,1334 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Text-to-mask (SAM 3) for the Images painters: ComfyUI's prompt syntax, the keep rule, the union mask at the
+source size, the lifecycle (keep-alive, release, GPU arbiter hooks) and the routes. A tiny random-weight Sam3Model
+built from a shrunk config runs the real forward on the CPU; no download and no network."""
+
+from __future__ import annotations
+
+import base64
+import io
+import threading
+import time
+import types
+
+import pytest
+
+from core.inference import diffusion_segment as seg
+
+
+def _png_b64(
+ width: int = 64,
+ height: int = 48,
+ color = (200, 30, 30),
+) -> str:
+ from PIL import Image
+
+ buffer = io.BytesIO()
+ Image.new("RGB", (width, height), color).save(buffer, format = "PNG")
+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+
+
+def _decode_mask(data_url: str):
+ from PIL import Image
+ raw = base64.b64decode(data_url.split(",", 1)[1])
+ return Image.open(io.BytesIO(raw))
+
+
+# ── prompt syntax (comfy/text_encoders/sam3_clip.py _parse_prompts) ─────────────────────────────────────────────────
+ "text, expected",
+ [
+ ("Sofa:2", [("Sofa", 2)]),
+ ("sofa", [("sofa", 1)]),
+ (
+ "Sofa:2, Red lamp, Window, Pillow:3",
+ [("Sofa", 2), ("Red lamp", 1), ("Window", 1), ("Pillow", 3)],
+ ),
+ ("(red lamp), window", [("red lamp", 1), ("window", 1)]),
+ ("a : 4 ", [("a", 4)]),
+ ("a:2.6", [("a", 3)]),
+ # Python's round(): half to even, as ComfyUI computes it.
+ ("a:2.5, b:3.5", [("a", 2), ("b", 4)]),
+ ("a:0", [("a", 1)]),
+ (" , sofa,, ,", [("sofa", 1)]),
+ ("note: red", [("note: red", 1)]),
+ ("v:2.5.1", [("v:2.5.1", 1)]),
+ ("grain:5000", [("grain", seg.MAX_DETECTIONS)]),
+ ("", []),
+ (None, []),
+ ("sofa, sofa:3", [("sofa", 1), ("sofa", 3)]),
+ ],
+)
+def test_parse_prompt_matches_comfyui(text, expected):
+ assert [(c.text, c.max_detections) for c in seg.parse_prompt(text)] == expected
+
+
+def test_parse_prompt_overflowing_count_is_a_malformed_suffix():
+ # float() of hundreds of digits is inf and round(inf) raises OverflowError: read it like "v:2.5.1".
+ huge = "9" * 400
+ assert [(c.text, c.max_detections) for c in seg.parse_prompt(f"object:{huge}, cat")] == [
+ (f"object:{huge}", 1),
+ ("cat", 1),
+ ]
+
+
+def test_parse_prompt_caps_the_concept_count():
+ with pytest.raises(ValueError, match = "At most"):
+ seg.parse_prompt(",".join(f"thing{i}" for i in range(seg.MAX_CONCEPTS + 1)))
+
+
+# ── keep rule and mask ──────────────────────────────────────────────────────────────────────────────────────────────
+def test_select_detections_thresholds_sorts_and_caps():
+ import torch
+
+ logits = torch.tensor([0.0, 3.0, -3.0, 2.0, 4.0]) # sigmoid: .5, .95, .05, .88, .98
+ boxes = torch.arange(20, dtype = torch.float32).reshape(5, 4)
+ masks = torch.arange(5, dtype = torch.float32)[:, None, None].expand(5, 2, 2).clone()
+ scores, kept_boxes, kept_masks = seg.select_detections(
+ logits, boxes, masks, max_detections = 2, threshold = 0.5
+ )
+ # 0.5 itself is not above the threshold; best first; at most 2.
+ assert [round(v, 2) for v in scores.tolist()] == [0.98, 0.95]
+ assert kept_boxes[:, 0].tolist() == [16.0, 4.0]
+ assert kept_masks[:, 0, 0].tolist() == [4.0, 1.0]
+
+
+def test_presence_suppresses_an_absent_concept():
+ import torch
+
+ logits = torch.tensor([1.0, 0.2]) # 0.73 and 0.55 alone: both kept without presence
+ boxes = torch.zeros(2, 4)
+ masks = torch.zeros(2, 2, 2)
+ scores, _, _ = seg.select_detections(logits, boxes, masks, max_detections = 5, threshold = 0.5)
+ assert scores.shape[0] == 2
+ absent = torch.tensor([-7.0]) # sigmoid ~ 0.001
+ scores, _, _ = seg.select_detections(
+ logits, boxes, masks, max_detections = 5, threshold = 0.5, presence_logit = absent
+ )
+ assert scores.shape[0] == 0
+ present = torch.tensor([7.0])
+ scores, _, _ = seg.select_detections(
+ logits, boxes, masks, max_detections = 5, threshold = 0.5, presence_logit = present
+ )
+ assert scores.shape[0] == 2
+
+
+def test_union_mask_upsamples_to_the_source_size():
+ import torch
+
+ left = torch.full((1, 4, 4), -5.0)
+ left[0, :, :2] = 5.0
+ bottom = torch.full((1, 4, 4), -5.0)
+ bottom[0, 3:, :] = 5.0
+ union = seg.union_mask(torch.cat([left, bottom]), 30, 40)
+ assert tuple(union.shape) == (30, 40)
+ assert union[0, 0] and union[29, 39] and not union[0, 39]
+ assert not seg.union_mask(torch.zeros(0, 4, 4), 7, 9).any()
+ # Nothing kept: the empty union stays on the masks' device (a CUDA run ORs it into a CUDA union).
+ empty = seg.union_mask(torch.zeros(0, 4, 4, device = "meta"), 7, 9)
+ assert empty.device.type == "meta" and tuple(empty.shape) == (7, 9)
+ png = seg.mask_to_png(union)
+ from PIL import Image
+
+ image = Image.open(io.BytesIO(png))
+ assert image.mode == "L" and image.size == (40, 30)
+ assert set(image.histogram()[1:255]) == {0}
+
+
+def test_preprocess_is_comfyui_bilinear_1008_normalised():
+ import torch
+ from PIL import Image
+
+ image = Image.new("RGB", (37, 23), (255, 0, 128))
+ tensor = seg.preprocess(image, "cpu", torch.float32)
+ assert tuple(tensor.shape) == (1, 3, seg.IMAGE_SIZE, seg.IMAGE_SIZE)
+ assert torch.allclose(tensor[0, 0], torch.ones(1)) and torch.allclose(
+ tensor[0, 1], -torch.ones(1)
+ )
+
+
+# ── the real forward on a tiny random Sam3Model ─────────────────────────────────────────────────────────────────────
+def _tiny_config():
+ from transformers import Sam3Config
+ return Sam3Config(
+ vision_config = {
+ "backbone_config": {
+ "hidden_size": 32,
+ "intermediate_size": 64,
+ "num_hidden_layers": 2,
+ "num_attention_heads": 2,
+ "window_size": 4,
+ "global_attn_indexes": [1],
+ "pretrain_image_size": 56,
+ "image_size": 112,
+ "patch_size": 14,
+ },
+ "fpn_hidden_size": 32,
+ # 112 / 14 = 8 tokens a side, at the 4x, 2x and 1x levels.
+ "backbone_feature_sizes": [[32, 32], [16, 16], [8, 8]],
+ },
+ text_config = {
+ "vocab_size": 128,
+ "hidden_size": 32,
+ "intermediate_size": 64,
+ "projection_dim": 32,
+ "num_hidden_layers": 1,
+ "num_attention_heads": 2,
+ "max_position_embeddings": 32,
+ },
+ geometry_encoder_config = {
+ "hidden_size": 32,
+ "num_layers": 1,
+ "num_attention_heads": 2,
+ "intermediate_size": 64,
+ },
+ detr_encoder_config = {
+ "hidden_size": 32,
+ "num_layers": 1,
+ "num_attention_heads": 2,
+ "intermediate_size": 64,
+ },
+ detr_decoder_config = {
+ "hidden_size": 32,
+ "num_layers": 1,
+ "num_queries": 12,
+ "num_attention_heads": 2,
+ "intermediate_size": 64,
+ },
+ mask_decoder_config = {
+ "hidden_size": 32,
+ "num_upsampling_stages": 3,
+ "num_attention_heads": 2,
+ },
+ )
+
+
+def test_tiny_config_is_small_but_the_real_input_is_1008():
+ """The tiny model runs at 112 px to keep the CPU tests fast; the size comes from the checkpoint's config."""
+ from transformers import Sam3Config
+
+ assert seg.IMAGE_SIZE == 1008
+ assert Sam3Config().vision_config.backbone_config.image_size == seg.IMAGE_SIZE
+
+
+class _FakeTokenizer:
+ """CLIP-shaped ids for the tiny vocab: one id per character, padded to max_length."""
+
+ def __call__(
+ self,
+ texts,
+ padding = None,
+ max_length = 32,
+ truncation = True,
+ return_tensors = "pt",
+ ):
+ import torch
+
+ ids = torch.zeros(len(texts), max_length, dtype = torch.long)
+ mask = torch.zeros(len(texts), max_length, dtype = torch.long)
+ for row, text in enumerate(texts):
+ tokens = [1] + [2 + (ord(ch) % 100) for ch in text][: max_length - 2] + [127]
+ ids[row, : len(tokens)] = torch.tensor(tokens)
+ mask[row, : len(tokens)] = 1
+
+ class _Batch(dict):
+ def to(self, device):
+ return _Batch({k: v.to(device) for k, v in self.items()})
+
+ return _Batch(input_ids = ids, attention_mask = mask)
+
+
+def tiny_sam3(tmp_path, monkeypatch):
+ transformers = pytest.importorskip("transformers")
+ if not hasattr(transformers, "Sam3Model"):
+ pytest.skip("transformers without SAM 3")
+ import torch
+
+ torch.manual_seed(0)
+ model = transformers.Sam3Model(_tiny_config())
+ model.save_pretrained(tmp_path)
+ monkeypatch.setattr(
+ transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: _FakeTokenizer()
+ )
+ return str(tmp_path)
+
+
+def test_tiny_model_segments_at_source_size(tiny_sam3):
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ image = Image.new("RGB", (64, 40), (120, 80, 40))
+ # A threshold of 0 keeps every query up to each concept's cap, so the caps are what is checked.
+ result = segmenter.segment(
+ image, "sofa, pillow:3, sofa", threshold = 0.0, allow_gpu = False, path = tiny_sam3
+ )
+ assert result.device == "cpu" and (result.width, result.height) == (64, 40)
+ assert [c["found"] for c in result.concepts] == [1, 3, 1]
+ assert [c["max_detections"] for c in result.concepts] == [1, 3, 1]
+ assert [i.concept for i in result.instances] == ["sofa", "pillow", "pillow", "pillow", "sofa"]
+ for inst in result.instances:
+ x1, y1, x2, y2 = inst.box
+ assert 0 <= x1 <= 64 and 0 <= y1 <= 40 and x2 <= 64 and y2 <= 40
+ mask = Image.open(io.BytesIO(result.mask_png))
+ assert mask.mode == "L" and mask.size == (64, 40)
+ body = result.to_json()
+ assert body["mask"].startswith("data:image/png;base64,")
+ assert set(body) == {"mask", "width", "height", "instances", "prompts", "device", "elapsed_ms"}
+ # No keep-alive: the weights left with the call.
+ assert not segmenter.loaded
+ # An unreachable threshold keeps nothing: an all-black mask, still at the source size.
+ result = segmenter.segment(image, "sofa", threshold = 1.0, allow_gpu = False, path = tiny_sam3)
+ assert result.instances == [] and Image.open(io.BytesIO(result.mask_png)).getbbox() is None
+ assert torch.is_grad_enabled()
+
+
+def test_mask_planes_do_not_outlive_their_text(tiny_sam3, monkeypatch):
+ import weakref
+
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ real_load, real_select = segmenter._load, seg.select_detections
+ planes: list = []
+
+ def _select(*args, **kwargs):
+ scores, boxes, masks = real_select(*args, **kwargs)
+ planes.append(weakref.ref(masks))
+ return scores, boxes, masks
+
+ class _Watch:
+ def __init__(self, model):
+ self._model = model
+
+ def __getattr__(self, name):
+ return getattr(self._model, name)
+
+ def __call__(self, **kwargs):
+ # Every earlier text's kept mask planes are already OR-ed into the union and gone.
+ assert all(ref() is None for ref in planes), "an earlier concept's mask planes are still held"
+ return self._model(**kwargs)
+
+ def _load(path, device, dtype):
+ real_load(path, device, dtype)
+ if not isinstance(segmenter._model, _Watch):
+ segmenter._model = _Watch(segmenter._model)
+
+ monkeypatch.setattr(seg, "select_detections", _select)
+ monkeypatch.setattr(segmenter, "_load", _load)
+ image = Image.new("RGB", (32, 32), (120, 80, 40))
+ prompt = "cat:200, dog:200, cat:2, bird:200"
+ result = segmenter.segment(image, prompt, threshold = 0.0, allow_gpu = False, path = tiny_sam3)
+ assert len(planes) == 3
+ # The union is what every occurrence's own best-k masks give: each text's largest cap covers its smaller ones.
+ monkeypatch.setattr(seg, "select_detections", real_select)
+ alone = [
+ segmenter.segment(image, part, threshold = 0.0, allow_gpu = False, path = tiny_sam3)
+ for part in ("cat:200", "dog:200", "bird:200")
+ ]
+ from PIL import ImageChops
+
+ combined = Image.open(io.BytesIO(alone[0].mask_png))
+ for other in alone[1:]:
+ combined = ImageChops.lighter(combined, Image.open(io.BytesIO(other.mask_png)))
+ assert ImageChops.difference(combined, Image.open(io.BytesIO(result.mask_png))).getbbox() is None
+
+
+def test_raw_outputs_do_not_outlive_their_concept(tiny_sam3, monkeypatch):
+ import weakref
+
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ real_load = segmenter._load
+ outputs: list = []
+
+ class _Watch:
+ def __init__(self, model):
+ self._model = model
+
+ def __getattr__(self, name):
+ return getattr(self._model, name)
+
+ def __call__(self, **kwargs):
+ # Every earlier concept's full output (all its mask planes) is gone before the next forward.
+ assert all(ref() is None for ref in outputs), "a raw SAM 3 output is still held"
+ out = self._model(**kwargs)
+ outputs.append(weakref.ref(out))
+ return out
+
+ def _load(path, device, dtype):
+ real_load(path, device, dtype)
+ if not isinstance(segmenter._model, _Watch):
+ segmenter._model = _Watch(segmenter._model)
+
+ monkeypatch.setattr(segmenter, "_load", _load)
+ image = Image.new("RGB", (32, 32), (120, 80, 40))
+ result = segmenter.segment(
+ image, "cat, dog, cat:3", threshold = 0.0, allow_gpu = False, path = tiny_sam3
+ )
+ assert len(outputs) == 2 # one forward per distinct text
+ # The repeat keeps the best of the larger cap: cat:1 is cat:3's first detection.
+ found = {(c["text"], c["max_detections"]): c["found"] for c in result.concepts}
+ assert found[("cat", 1)] == 1 and found[("cat", 3)] == 3
+ cats = [i for i in result.instances if i.concept == "cat"]
+ assert cats[0] == cats[1] and cats[0].score >= cats[2].score >= cats[3].score
+
+
+def test_keep_alive_then_release(tiny_sam3):
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ image = Image.new("RGB", (32, 32))
+ segmenter.segment(image, "cat", allow_gpu = False, path = tiny_sam3)
+ assert segmenter.loaded and segmenter._timer is not None
+ assert segmenter.release("test") is True
+ assert not segmenter.loaded and segmenter._timer is None
+ assert segmenter.release("again") is False
+
+
+def test_keep_alive_expiry_unloads(tiny_sam3):
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", allow_gpu = False, path = tiny_sam3)
+ segmenter._timer.cancel()
+ segmenter._expire(segmenter._release_epoch)
+ assert not segmenter.loaded
+
+
+def test_release_during_a_call_unloads_when_it_ends(tiny_sam3, monkeypatch):
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ entered, proceed = threading.Event(), threading.Event()
+ real = segmenter._segment_on
+
+ def _slow(*args, **kwargs):
+ entered.set()
+ proceed.wait(10)
+ return real(*args, **kwargs)
+
+ monkeypatch.setattr(segmenter, "_segment_on", _slow)
+ worker = threading.Thread(
+ target = segmenter.segment,
+ args = (Image.new("RGB", (16, 16)), "cat"),
+ kwargs = {"allow_gpu": False, "path": tiny_sam3},
+ )
+ worker.start()
+ assert entered.wait(10)
+ # Non-blocking: the call is in flight, so the release is deferred, not refused.
+ assert segmenter.release("arbiter") is False
+ proceed.set()
+ worker.join(30)
+ assert not segmenter.loaded
+
+
+def test_tight_gpu_unloads_right_after_the_call(tiny_sam3, monkeypatch):
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", allow_gpu = False, path = tiny_sam3)
+ assert segmenter.loaded
+ # Pretend the copy sits on a card that is now tight.
+ segmenter.device = "cuda:0"
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (1000, 80_000))
+ with segmenter._run_lock:
+ segmenter._after_call()
+ assert not segmenter.loaded
+
+
+def test_vram_is_tight_rule():
+ assert seg.vram_is_tight(1000, 24_000)
+ assert seg.vram_is_tight(10_000, 80_000) # under 20% of the card
+ assert not seg.vram_is_tight(30_000, 80_000)
+ assert not seg.vram_is_tight(None, None)
+
+
+def test_choose_device_falls_back_to_cpu(monkeypatch):
+ import torch
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ if getattr(torch.backends, "mps", None) is not None:
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
+ assert seg.choose_device(True) == ("cpu", torch.float32)
+ # Training (allow_gpu False) never takes a GPU.
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ assert seg.choose_device(False) == ("cpu", torch.float32)
+
+
+def test_choose_device_needs_room_on_the_card(monkeypatch):
+ import contextlib
+
+ import torch
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device", lambda ordinal: contextlib.nullcontext())
+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 1)
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (20_000, 24_000))
+ assert seg.choose_device(True) == ("cuda:1", torch.bfloat16)
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (1_000, 24_000))
+ if getattr(torch.backends, "mps", None) is not None:
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
+ assert seg.choose_device(True) == ("cpu", torch.float32)
+
+
+def test_choose_device_never_shares_a_card_another_owner_holds(monkeypatch):
+ import contextlib
+
+ import torch
+
+ from core.inference import gpu_arbiter
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "device", lambda ordinal: contextlib.nullcontext())
+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: True)
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (60_000, 80_000)) # plenty of room either way
+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: None)
+ monkeypatch.setattr(gpu_arbiter, "physical_gpu_ids", lambda visible: frozenset(visible))
+ if getattr(torch.backends, "mps", None) is not None:
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
+ cases = [
+ ({}, "cuda:0"), # nobody
+ ({gpu_arbiter.DIFFUSION: frozenset({0})}, "cuda:0"), # the image model: SAM 3 runs beside it by design
+ ({gpu_arbiter.CHAT: frozenset({0})}, "cpu"),
+ ({gpu_arbiter.VIDEO: frozenset({0}), gpu_arbiter.DIFFUSION: frozenset({0})}, "cpu"),
+ ({gpu_arbiter.CHAT: None}, "cpu"), # unknown GPUs: every card
+ ({gpu_arbiter.CHAT: frozenset({1})}, "cuda:0"), # another card
+ ]
+ for held, expected in cases:
+ monkeypatch.setattr(gpu_arbiter, "residents", lambda held = held: dict(held))
+ device, dtype = seg.choose_device(True)
+ assert device == expected, held
+ assert dtype == (torch.bfloat16 if expected != "cpu" else torch.float32)
+
+
+def test_metal_is_not_shared_with_another_owner(monkeypatch):
+ import torch
+
+ from core.inference import gpu_arbiter
+
+ if getattr(torch.backends, "mps", None) is None:
+ pytest.skip("torch without an MPS backend")
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: True)
+ # Apple has one device: any resident but the image model counts, whatever GPU ids it reports.
+ for held, expected in (
+ ({}, "mps"),
+ ({gpu_arbiter.DIFFUSION: None}, "mps"),
+ ({gpu_arbiter.CHAT: frozenset({0})}, "cpu"),
+ ({gpu_arbiter.VIDEO: None, gpu_arbiter.DIFFUSION: None}, "cpu"),
+ ({gpu_arbiter.CHAT: frozenset()}, "cpu"),
+ ):
+ monkeypatch.setattr(gpu_arbiter, "residents", lambda held = held: dict(held))
+ assert seg.choose_device(True) == (expected, torch.float32), held
+
+
+def test_a_failed_load_places_no_weights(tmp_path, monkeypatch):
+ import torch
+ import transformers
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ placed, freed = [], []
+
+ def _broken_tokenizer(*a, **k):
+ raise OSError("no tokenizer files")
+
+ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", _broken_tokenizer)
+ monkeypatch.setattr(transformers.Sam3Model, "from_pretrained", lambda *a, **k: placed.append(a) or object())
+ with pytest.raises(OSError):
+ segmenter._load(str(tmp_path), "cuda:0", torch.bfloat16)
+ assert placed == [] and not segmenter.loaded # the tokenizer fails before any weights reach the card
+
+ # Weights placed, then the load fails: they are dropped and the device cache handed back.
+ monkeypatch.setattr(transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: object())
+
+ class _Half:
+ def eval(self):
+ raise RuntimeError("bad checkpoint")
+
+ monkeypatch.setattr(transformers.Sam3Model, "from_pretrained", lambda *a, **k: _Half())
+ monkeypatch.setattr(seg, "_free_device_cache", lambda device: freed.append(device))
+ with pytest.raises(RuntimeError, match = "bad checkpoint"):
+ segmenter._load(str(tmp_path), "cuda:0", torch.bfloat16)
+ assert freed == ["cuda:0"] and not segmenter.loaded and segmenter._tokenizer is None
+
+
+def test_choose_device_reads_bf16_on_the_chosen_card(monkeypatch):
+ import contextlib
+
+ import torch
+
+ # cuda:0 (current) has bf16, cuda:1 (the image model's card) does not.
+ current = [0]
+
+ @contextlib.contextmanager
+ def _device(ordinal):
+ before, current[0] = current[0], ordinal
+ try:
+ yield
+ finally:
+ current[0] = before
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: current[0])
+ monkeypatch.setattr(torch.cuda, "device", _device)
+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda *a, **k: current[0] == 0)
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (20_000, 24_000))
+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 1)
+ assert seg.choose_device(True) == ("cuda:1", torch.float16)
+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 0)
+ assert seg.choose_device(True) == ("cuda:0", torch.bfloat16)
+ assert current[0] == 0
+
+
+def test_oom_on_the_gpu_retries_on_the_cpu(tiny_sam3, monkeypatch):
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))
+ real = segmenter._segment_on
+ devices = []
+
+ def _maybe_oom(image, concepts, threshold, path, device, dtype):
+ devices.append(device)
+ if device != "cpu":
+ raise torch.OutOfMemoryError("CUDA out of memory")
+ return real(image, concepts, threshold, path, device, dtype)
+
+ monkeypatch.setattr(segmenter, "_segment_on", _maybe_oom)
+ result = segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)
+ assert devices == ["cuda:0", "cpu"] and result.device == "cpu"
+
+
+def test_oom_retry_frees_the_failed_attempt_before_reading_cpu_only(monkeypatch):
+ import weakref
+
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+
+ class _CudaTensors: # stands in for the activations the failed attempt's frame holds
+ pass
+
+ pinned: list = []
+ freed: list = []
+
+ def _segment_on(image, concepts, threshold, path, device, dtype):
+ if device != "cpu":
+ activations = _CudaTensors()
+ pinned.append(weakref.ref(activations))
+ raise torch.OutOfMemoryError("CUDA out of memory")
+ assert segmenter._call_device == "cpu"
+ return types.SimpleNamespace()
+
+ def _free(device):
+ # The cache is released only once the OOM's traceback no longer pins the failed attempt's tensors.
+ freed.append((device, pinned[0]() is None, segmenter._call_device))
+
+ monkeypatch.setattr(segmenter, "_segment_on", _segment_on)
+ monkeypatch.setattr(seg, "_free_device_cache", _free)
+ segmenter.segment(Image.new("RGB", (8, 8)), "cat", path = "unused")
+ assert freed and freed[0] == ("cuda:0", True, "cuda:0")
+
+
+def test_failed_cpu_retry_still_cleans_up(tiny_sam3, monkeypatch):
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))
+ real_load = segmenter._load
+
+ def _oom_then_fail(image, concepts, threshold, path, device, dtype):
+ if device != "cpu":
+ raise torch.OutOfMemoryError("CUDA out of memory")
+ real_load(path, device, dtype) # the CPU copy is resident when preprocessing fails
+ segmenter._release_pending = True # a release that arrived mid-call
+ raise RuntimeError("preprocess failed")
+
+ monkeypatch.setattr(segmenter, "_segment_on", _oom_then_fail)
+ with pytest.raises(RuntimeError, match = "preprocess failed"):
+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)
+ # The pending release was honoured, not left with the weights resident and no timer.
+ assert not segmenter.loaded and segmenter._release_pending is False
+ assert segmenter._call_device is None
+
+ # Without a pending release the CPU copy gets the keep-alive instead.
+ def _oom_then_fail_quietly(image, concepts, threshold, path, device, dtype):
+ if device != "cpu":
+ raise torch.OutOfMemoryError("CUDA out of memory")
+ real_load(path, device, dtype)
+ raise RuntimeError("preprocess failed")
+
+ monkeypatch.setattr(segmenter, "_segment_on", _oom_then_fail_quietly)
+ with pytest.raises(RuntimeError):
+ segmenter.segment(Image.new("RGB", (16, 16)), "cat", path = tiny_sam3)
+ assert segmenter.loaded and segmenter._timer is not None
+ segmenter.release("test")
+
+
+def test_segment_errors(monkeypatch, tmp_path):
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ image = Image.new("RGB", (8, 8))
+ with pytest.raises(ValueError):
+ segmenter.segment(image, " , ", path = str(tmp_path))
+ with pytest.raises(ValueError):
+ segmenter.segment(image, "sofa", threshold = 2.0, path = str(tmp_path))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)
+ with pytest.raises(seg.SegmenterUnavailableError):
+ segmenter.segment(image, "sofa", path = str(tmp_path))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ monkeypatch.setattr(seg, "model_dir", lambda: None)
+ with pytest.raises(seg.SegmenterNotDownloadedError):
+ segmenter.segment(image, "sofa")
+
+
+# ── files, status and download errors ───────────────────────────────────────────────────────────────────────────────
+def test_override_dir_must_hold_a_config(tmp_path, monkeypatch):
+ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))
+ assert seg.model_dir() is None # a typo'd / empty override is not replaced by the cache
+ (tmp_path / "config.json").write_text("{}")
+ (tmp_path / "model.safetensors").write_bytes(b"")
+ (tmp_path / "tokenizer.json").write_text("{}")
+ assert seg.model_dir() == str(tmp_path)
+ assert seg.status()["override"] is True
+
+
+def test_partial_override_does_not_read_as_downloaded(tmp_path, monkeypatch):
+ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ (tmp_path / "config.json").write_text("{}")
+ assert seg.model_dir() is None # config only: no weights, no tokenizer
+ (tmp_path / "model.safetensors.index.json").write_text('{"weight_map": {"w": "model-1.safetensors"}}')
+ (tmp_path / "model-1.safetensors").write_bytes(b"")
+ assert seg.model_dir() is None # still no tokenizer
+ (tmp_path / "vocab.json").write_text("{}")
+ assert seg.model_dir() is None # vocab.json alone: AutoTokenizer refuses it without merges.txt
+ (tmp_path / "merges.txt").write_text("#version: 0.2\n")
+ assert seg.model_dir() == str(tmp_path) # sharded weights + a tokenizer
+ (tmp_path / "model.safetensors.index.json").unlink()
+ body = seg.status()
+ assert body["downloaded"] is False and body["override"] is True
+
+
+def test_sharded_override_needs_every_shard(tmp_path, monkeypatch):
+ import json
+
+ monkeypatch.setenv(seg.ENV_PATH_OVERRIDE, str(tmp_path))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ (tmp_path / "config.json").write_text("{}")
+ (tmp_path / "tokenizer.json").write_text("{}")
+ shards = ["model-00001-of-00002.safetensors", "model-00002-of-00002.safetensors"]
+ index = tmp_path / "model.safetensors.index.json"
+ index.write_text(json.dumps({"weight_map": {"a": shards[0], "b": shards[1], "c": shards[1]}}))
+ (tmp_path / shards[0]).write_bytes(b"")
+ assert seg.model_dir() is None # an interrupted copy: the second shard is missing
+ assert seg.status()["downloaded"] is False
+ (tmp_path / shards[1]).write_bytes(b"")
+ assert seg.model_dir() == str(tmp_path)
+ for broken in ("not json", json.dumps({"weight_map": {}}), json.dumps({"metadata": {}})):
+ index.write_text(broken)
+ assert seg.model_dir() is None
+
+
+def test_status_shape(monkeypatch):
+ monkeypatch.delenv(seg.ENV_PATH_OVERRIDE, raising = False)
+ monkeypatch.setattr(seg, "_cached_snapshot", lambda *a, **k: None)
+ body = seg.status()
+ assert body["repo"] == "facebook/sam3" and body["gated"] is True
+ assert body["downloaded"] is False and body["loaded"] is False
+ assert body["license_url"].startswith("https://huggingface.co/facebook/sam3")
+ assert body["size_bytes"] > 3_000_000_000
+ assert {"downloading", "error", "bytes_total", "bytes_done"} <= set(body["download"])
+
+
+def test_the_download_names_only_what_sam3model_reads():
+ assert seg.SEGMENTER_SPEC.files == seg.SEGMENTER_FILES
+ assert "model.safetensors" in seg.SEGMENTER_FILES and "sam3.pt" not in seg.SEGMENTER_FILES
+
+
+def test_gated_download_error_names_the_license():
+ class GatedRepoError(Exception):
+ pass
+
+ message = seg.describe_download_error("sam3", GatedRepoError("403 Client Error"))
+ assert "license" in message and seg.SEGMENTER_LICENSE_URL in message
+ assert seg.describe_download_error("sam3", RuntimeError("disk full")) is None
+
+
+def test_shared_download_state_uses_the_spec_file_list(monkeypatch, tmp_path):
+ """The pinned downloader fetches a spec's ``files`` when it has them (the dictation specs keep their pair)."""
+ from core.inference import stt_mtmd_sidecar as mtmd
+
+ seen = []
+
+ class _Meta:
+ commit_hash = "a" * 40
+ size = 10
+ etag = "e"
+
+ def _meta(url, token = None):
+ seen.append(url)
+ raise RuntimeError("stop after the first lookup")
+
+ monkeypatch.setattr("huggingface_hub.get_hf_file_metadata", _meta)
+ state = mtmd._MtmdDownloadState(
+ specs = {seg.SEGMENTER_ID: seg.SEGMENTER_SPEC},
+ cached_paths = lambda *a, **k: None,
+ resolve = seg._resolve_segmenter_id,
+ label = "SAM 3 segmenter",
+ describe_error = lambda model_id, error: "described",
+ )
+ state._run(seg.SEGMENTER_ID, None, hub_cache = tmp_path)
+ assert seen and seen[0].endswith(seg.SEGMENTER_FILES[0])
+ assert state.status()["error"] == "described"
+
+
+# ── GPU arbiter hooks ───────────────────────────────────────────────────────────────────────────────────────────────
+class _Recorder:
+ def __init__(self):
+ self.calls = []
+
+ def release_segmenter(
+ self,
+ reason,
+ wait = False,
+ handover = False,
+ ):
+ self.calls.append((reason, wait))
+ return True
+
+
+def test_arbiter_releases_the_segmenter(monkeypatch):
+ import sys
+
+ from core.inference import gpu_arbiter
+
+ recorder = _Recorder()
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)
+ gpu_arbiter._release_mask_segmenter("x")
+ assert recorder.calls == [("x", True)]
+ # Training's start path: release(DIFFUSION) whether or not an image model is resident.
+ recorder.calls.clear()
+ monkeypatch.setattr(gpu_arbiter, "_residents", {})
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ assert recorder.calls and recorder.calls[0][0] == "image GPU released"
+ # Never loaded: nothing is imported or created.
+ monkeypatch.delitem(sys.modules, "core.inference.diffusion_segment")
+ gpu_arbiter._release_mask_segmenter("y")
+
+
+def _hold_a_call(segmenter, device):
+ """Run a fake call on ``device`` that holds the run lock until the returned event is set."""
+ entered, proceed = threading.Event(), threading.Event()
+
+ def _call():
+ with segmenter._run_lock:
+ segmenter._call_device = device
+ entered.set()
+ proceed.wait(10)
+ segmenter._call_device = None
+
+ worker = threading.Thread(target = _call)
+ worker.start()
+ assert entered.wait(10)
+ return proceed, worker
+
+
+def test_a_release_timing_out_as_the_call_settles_still_unloads(monkeypatch):
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 600)
+ monkeypatch.setattr(seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16))
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ monkeypatch.setattr(seg, "_cuda_free_mib", lambda ordinal: (None, None))
+
+ def _segment_on(image, concepts, threshold, path, device, dtype):
+ segmenter._model, segmenter.device = object(), device # the CUDA copy stays resident after the call
+ return types.SimpleNamespace()
+
+ real_after = segmenter._after_call
+ settled = threading.Event()
+
+ def _after_call():
+ real_after() # reads the release flag (none yet) and arms the keep-alive
+ settled.set()
+ time.sleep(0.3) # the call's tail: the release's wait runs out right here
+
+ monkeypatch.setattr(segmenter, "_segment_on", _segment_on)
+ monkeypatch.setattr(segmenter, "_after_call", _after_call)
+ worker = threading.Thread(target = segmenter.segment, args = (Image.new("RGB", (8, 8)), "cat"), kwargs = {"path": "x"})
+ worker.start()
+ assert settled.wait(10)
+ # Timed out on a call that will not see the flag: the handover waits out the tail and unloads, never letting the
+ # next owner in beside the resident weights.
+ assert segmenter.release("handover", wait = True, timeout = 0.05, handover = True) is True
+ worker.join(10)
+ assert not segmenter.loaded and segmenter._timer is None and segmenter._release_pending is False
+ # Codex's interleaving: the call already cleared its device and is only letting go of the lock.
+ segmenter._model, segmenter.device = object(), "cuda:0"
+ proceed, holder = _hold_a_call(segmenter, None)
+ threading.Timer(0.2, proceed.set).start()
+ try:
+ assert segmenter.release("handover", wait = True, timeout = 0.05, handover = True) is True
+ finally:
+ proceed.set()
+ holder.join(10)
+ assert not segmenter.loaded
+
+
+def test_handover_refuses_while_a_gpu_call_outlives_the_wait(monkeypatch):
+ import sys
+
+ from core.inference import gpu_arbiter
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ monkeypatch.setattr(seg, "_segmenter", segmenter)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", seg)
+ real_release = segmenter.release
+ monkeypatch.setattr(
+ segmenter, "release", lambda reason, **kw: real_release(reason, **{**kw, "timeout": 0.05})
+ )
+ for device in ("cuda:0", ""): # on the card, or not placed yet
+ proceed, worker = _hold_a_call(segmenter, device)
+ try:
+ with pytest.raises(seg.SegmenterReleaseTimeoutError):
+ seg.release_segmenter("x", wait = True, handover = True)
+ # The arbiter aborts the transfer instead of handing over a busy card.
+ with pytest.raises(RuntimeError, match = "cannot be transferred"):
+ gpu_arbiter._release_mask_segmenter("GPU acquired for chat", handover = True)
+ # Release / training paths stay best effort: deferred, logged, never raised.
+ gpu_arbiter._release_mask_segmenter("image GPU released")
+ assert segmenter._release_pending is True
+ finally:
+ proceed.set()
+ worker.join(10)
+ # A call on the CPU holds no VRAM: the handover proceeds and the release stays deferred.
+ proceed, worker = _hold_a_call(segmenter, "cpu")
+ try:
+ assert seg.release_segmenter("x", wait = True, handover = True) is False
+ gpu_arbiter._release_mask_segmenter("GPU acquired for chat", handover = True)
+ finally:
+ proceed.set()
+ worker.join(10)
+ # Idle: nothing to wait for.
+ assert seg.release_segmenter("x", wait = True, handover = True) is False
+
+
+def test_acquire_does_not_register_while_the_segmenter_holds_the_card(monkeypatch):
+ import sys
+
+ from core.inference import gpu_arbiter
+
+ class _Busy:
+ class SegmenterReleaseTimeoutError(RuntimeError):
+ pass
+
+ def release_segmenter(
+ self,
+ reason,
+ wait = False,
+ handover = False,
+ ):
+ if handover:
+ raise self.SegmenterReleaseTimeoutError("still running")
+ return False
+
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", _Busy())
+ monkeypatch.setattr(gpu_arbiter, "_residents", {})
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)
+ registered = []
+ with pytest.raises(RuntimeError, match = "cannot be transferred"):
+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = lambda: registered.append(1))
+ assert registered == [] and gpu_arbiter.current_owner() is None
+
+
+def test_a_call_queued_across_a_handover_stays_off_the_card(tiny_sam3, monkeypatch):
+ import torch
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ monkeypatch.setattr(
+ seg,
+ "choose_device",
+ lambda allow_gpu: ("cuda:0", torch.bfloat16) if allow_gpu else ("cpu", torch.float32),
+ )
+ placed = []
+ monkeypatch.setattr(
+ segmenter, "_segment_on", lambda *a: placed.append(a[4]) or types.SimpleNamespace()
+ )
+ image = Image.new("RGB", (16, 16))
+ for handover, cpu_only in ((True, False), (True, True), (False, False)):
+ placed.clear()
+ proceed, worker = _hold_a_call(segmenter, "cuda:0")
+ errors = []
+
+ def _queued():
+ try:
+ segmenter.segment(image, "cat", allow_gpu = not cpu_only, path = tiny_sam3)
+ except Exception as exc: # noqa: BLE001
+ errors.append(exc)
+
+ queued = threading.Thread(target = _queued)
+ queued.start()
+ deadline = time.monotonic() + 5
+ while not segmenter._run_lock.locked() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ time.sleep(0.1) # the queued call is now blocked on _run_lock
+ # The handover's release waits on the call in flight; it ends, the release takes and drops the lock, and the
+ # new owner goes on to register.
+ releaser = threading.Thread(
+ target = lambda: segmenter.release("handover", wait = True, handover = handover)
+ )
+ releaser.start()
+ time.sleep(0.1)
+ proceed.set()
+ worker.join(10)
+ releaser.join(10)
+ queued.join(10)
+ if handover and not cpu_only:
+ assert (
+ placed == [] and len(errors) == 1 and isinstance(errors[0], seg.SegmenterBusyError)
+ )
+ elif handover:
+ assert placed == ["cpu"] and errors == [] # a CPU-only call holds no VRAM
+ else:
+ assert placed == ["cuda:0"] and errors == [] # an ordinary release does not fence calls
+ # A call that starts after the handover runs normally.
+ placed.clear()
+ segmenter.segment(image, "cat", path = tiny_sam3)
+ assert placed == ["cuda:0"]
+
+
+def test_a_call_during_a_handovers_register_stays_off_the_card(tiny_sam3, monkeypatch):
+ import sys
+
+ import torch
+ from PIL import Image
+
+ from core.inference import gpu_arbiter
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ monkeypatch.setattr(seg, "_segmenter", segmenter)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", seg)
+ monkeypatch.setattr(
+ seg, "choose_device", lambda allow_gpu: ("cuda:0", torch.bfloat16) if allow_gpu else ("cpu", torch.float32)
+ )
+ placed = []
+ monkeypatch.setattr(segmenter, "_segment_on", lambda *a: placed.append(a[4]) or types.SimpleNamespace())
+ monkeypatch.setattr(gpu_arbiter, "_residents", {})
+ monkeypatch.setattr(gpu_arbiter, "_owner", None)
+ monkeypatch.setattr(gpu_arbiter, "_release_idle_video_servers", lambda reason: None)
+ image = Image.new("RGB", (16, 16))
+ seen = {}
+
+ def _register():
+ # The segmenter's release already ran and dropped its lock; the new owner is still loading.
+ seen["fenced"] = gpu_arbiter.handover_in_progress()
+ with pytest.raises(seg.SegmenterBusyError):
+ segmenter.segment(image, "cat", path = tiny_sam3)
+ segmenter.segment(image, "cat", allow_gpu = False, path = tiny_sam3) # the CPU is not being handed over
+
+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = _register)
+ assert seen["fenced"] is True and placed == ["cpu"]
+ assert gpu_arbiter.handover_in_progress() is False
+ segmenter.segment(image, "cat", path = tiny_sam3)
+ assert placed == ["cpu", "cuda:0"]
+ # A raising register still lifts the fence.
+ with pytest.raises(RuntimeError):
+ gpu_arbiter.acquire_for(gpu_arbiter.VIDEO, register = lambda: (_ for _ in ()).throw(RuntimeError("load failed")))
+ assert gpu_arbiter.handover_in_progress() is False
+
+
+def test_training_beside_an_off_torch_image_model_releases_the_segmenter(monkeypatch):
+ import inspect
+ import sys
+ from types import SimpleNamespace
+
+ import routes.training as training_routes
+ from core.inference import diffusion_engine_router, gpu_arbiter
+
+ recorder = _Recorder()
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)
+ resident = SimpleNamespace(runs_off_torch_device = True, is_loaded = True, unload = lambda: None)
+ monkeypatch.setattr(diffusion_engine_router, "get_active_diffusion_engine", lambda: resident)
+ monkeypatch.setattr(gpu_arbiter, "release", lambda owner: None)
+ monkeypatch.setattr(
+ "core.inference.video.get_video_backend",
+ lambda: SimpleNamespace(status = lambda: {"loaded": False}, unload = lambda: None),
+ )
+ monkeypatch.setattr("routes.training_vram.summarize_resident_chat", lambda: {"any": False})
+ training_routes._free_gpu_for_diffusion_training()
+ assert ("diffusion training start", True) in recorder.calls
+ # The LLM start path keeps an off-torch image model the same way and releases the segmenter there too.
+ source = inspect.getsource(training_routes.start_training)
+ release = source.index('gpu_arbiter.release_mask_segmenter("training start")')
+ assert release < source.index('if getattr(diffusion, "runs_off_torch_device", False) is True:')
+
+
+def test_training_start_refuses_while_a_mask_call_holds_the_gpu(monkeypatch):
+ import inspect
+ import sys
+ from types import SimpleNamespace
+
+ import routes.training as training_routes
+ from core.inference import diffusion_engine_router, gpu_arbiter
+
+ class _StillRunning:
+ class SegmenterReleaseTimeoutError(RuntimeError):
+ pass
+
+ def release_segmenter(self, reason, wait = False, handover = False):
+ if handover:
+ raise self.SegmenterReleaseTimeoutError("still running")
+ return False
+
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", _StillRunning())
+ with pytest.raises(gpu_arbiter.SegmenterStillRunningError, match = "try again"):
+ gpu_arbiter.release_mask_segmenter("training start")
+ assert gpu_arbiter.SegmenterStillRunningError.blocks_training is True # the core trainer re-raises it
+ unloaded = []
+ for off_torch in (True, False):
+ resident = SimpleNamespace(
+ runs_off_torch_device = off_torch, is_loaded = True, unload = lambda: unloaded.append(1)
+ )
+ monkeypatch.setattr(diffusion_engine_router, "get_active_diffusion_engine", lambda r = resident: r)
+ monkeypatch.setattr(gpu_arbiter, "release", lambda owner: None)
+ # A RuntimeError: the diffusion start route answers 409 instead of spawning beside SAM 3.
+ with pytest.raises(gpu_arbiter.SegmenterStillRunningError):
+ training_routes._free_gpu_for_diffusion_training()
+ assert unloaded == [] # refused before tearing anything down
+ # The LLM start path: re-raised out of its best-effort block and answered 409.
+ source = inspect.getsource(training_routes.start_training)
+ assert "except gpu_arbiter.SegmenterStillRunningError:\n raise" in source
+ assert source.count("SegmenterStillRunningError,") == 2
+
+
+def test_no_find_starts_while_a_render_is_on_its_way(monkeypatch):
+ import asyncio
+
+ import core.inference.diffusion_engine_router as engine_router
+ import routes.inference as inference_routes
+
+ engine = _Engine()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)
+ monkeypatch.setattr(
+ seg,
+ "get_segmenter",
+ lambda: pytest.fail("a Find ran between the render's SAM 3 release and its slot"),
+ )
+ seen = []
+
+ async def _run(request):
+ # The render released SAM 3 and has not taken the slot yet: a Find is refused, and frees the slot it took.
+ with pytest.raises(seg.SegmenterBusyError, match = "generating"):
+ seg.segment_image(_png_b64(), "sofa")
+ seen.append(engine._generate_lock.locked())
+ return "done"
+
+ monkeypatch.setattr(inference_routes, "_generate_diffusion_image_run", _run)
+ assert asyncio.run(inference_routes._generate_diffusion_image_core(object())) == "done"
+ assert seen == [False] and seg._renders_pending == 0
+
+
+def test_a_render_releases_the_segmenter_first(monkeypatch):
+ import inspect
+ import sys
+
+ import routes.inference as inference_routes
+
+ recorder = _Recorder()
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)
+ inference_routes._release_mask_segmenter_for_render()
+ assert recorder.calls == [("image generation", True)]
+ # Never loaded: nothing is imported.
+ monkeypatch.delitem(sys.modules, "core.inference.diffusion_segment")
+ inference_routes._release_mask_segmenter_for_render()
+ # Before the run, on every render (a resident model renders with no arbiter acquire).
+ source = inspect.getsource(inference_routes._generate_diffusion_image_run)
+ release = source.index("_release_mask_segmenter_for_render")
+ assert source.index("_enhance_generate_prompts(backend, request)") < release < source.index("for attempt in range(2)")
+
+
+def test_release_segmenter_never_creates_one(monkeypatch):
+ monkeypatch.setattr(seg, "_segmenter", None)
+ assert seg.release_segmenter("x") is False
+ assert seg._segmenter is None
+
+
+# ── render slot ─────────────────────────────────────────────────────────────────────────────────────────────────────
+class _Engine:
+ def __init__(self):
+ self._generate_lock = threading.Lock()
+ self._loading = None
+
+
+def test_render_slot_refuses_while_generating_or_loading(monkeypatch):
+ import core.inference.diffusion_engine_router as engine_router
+
+ engine = _Engine()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)
+ lock, why = seg._take_render_slot()
+ assert lock is engine._generate_lock and why == ""
+ lock.release()
+ with engine._generate_lock:
+ assert seg._take_render_slot() == (None, "generating")
+
+ class _Loading:
+ error = None
+
+ engine._loading = _Loading()
+ assert seg._take_render_slot() == (None, "loading")
+ engine._loading.error = "failed"
+ lock, why = seg._take_render_slot()
+ assert why == "" and lock is not None
+ lock.release()
+
+
+def test_segment_image_holds_and_frees_the_slot(monkeypatch):
+ import core.inference.diffusion_engine_router as engine_router
+
+ engine = _Engine()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)
+
+ class _Stub:
+ def segment(self, image, prompt, threshold, allow_gpu):
+ assert engine._generate_lock.locked()
+ assert image.size == (64, 48)
+ return seg.SegmentResult(mask_png = b"png", width = 64, height = 48)
+
+ monkeypatch.setattr(seg, "get_segmenter", lambda: _Stub())
+ body = seg.segment_image(_png_b64(), "sofa")
+ assert body["width"] == 64 and not engine._generate_lock.locked()
+ with engine._generate_lock:
+ with pytest.raises(seg.SegmenterBusyError):
+ seg.segment_image(_png_b64(), "sofa")
+ with pytest.raises(ValueError):
+ seg.segment_image("not base64 at all !!", "sofa")
+
+
+# ── routes ──────────────────────────────────────────────────────────────────────────────────────────────────────────
+def client(monkeypatch):
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ import core.inference.diffusion_engine_router as engine_router
+ from auth.authentication import authenticated_via_api_key, get_current_subject
+ from routes.inference import studio_router
+
+ engine = _Engine()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)
+ app = FastAPI()
+ app.include_router(studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-user"
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
+ return TestClient(app), engine
+
+
+def test_segment_route_contract(client, tiny_sam3, monkeypatch):
+ http, engine = client
+ import routes.inference as inference_routes
+
+ monkeypatch.setattr(seg, "model_dir", lambda: tiny_sam3)
+ monkeypatch.setattr(seg, "_segmenter", seg.MaskSegmenter(keep_alive_seconds = 0))
+ monkeypatch.setattr(
+ inference_routes, "_training_is_active", lambda: True
+ ) # CPU, no GPU in tests
+ r = http.post(
+ "/api/inference/images/segment",
+ json = {"image": _png_b64(64, 48), "prompt": "Sofa:2, red lamp", "threshold": 0.0},
+ )
+ assert r.status_code == 200, r.text
+ body = r.json()
+ assert body["width"] == 64 and body["height"] == 48 and body["device"] == "cpu"
+ assert [p["text"] for p in body["prompts"]] == ["Sofa", "red lamp"]
+ assert [p["max_detections"] for p in body["prompts"]] == [2, 1]
+ assert len(body["instances"]) == 3
+ assert all(
+ set(i) == {"prompt", "score", "box"} and len(i["box"]) == 4 for i in body["instances"]
+ )
+ mask = _decode_mask(body["mask"])
+ assert mask.size == (64, 48) and mask.mode == "L"
+
+
+def test_segment_route_errors(client, monkeypatch):
+ http, engine = client
+ url = "/api/inference/images/segment"
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ monkeypatch.setattr(seg, "model_dir", lambda: None)
+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})
+ assert r.status_code == 409 and "not downloaded" in r.json()["detail"]
+ r = http.post(url, json = {"image": _png_b64(), "prompt": " , , "})
+ assert r.status_code == 400
+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa", "threshold": 3})
+ assert r.status_code == 422
+ r = http.post(url, json = {"image": "!!!", "prompt": "sofa"})
+ assert r.status_code == 400
+ monkeypatch.setattr(seg, "model_dir", lambda: "/nonexistent")
+ with engine._generate_lock:
+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})
+ assert r.status_code == 409 and "generating" in r.json()["detail"]
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)
+ r = http.post(url, json = {"image": _png_b64(), "prompt": "sofa"})
+ assert r.status_code == 501
+
+
+def test_status_download_cancel_release_routes(client, monkeypatch):
+ http, _engine = client
+ from hub.services.models import account_access
+
+ monkeypatch.setattr(account_access, "require_installation_owner", lambda: None)
+ monkeypatch.setattr(
+ seg, "status", lambda: {"supported": True, "downloaded": False, "override": False}
+ )
+ assert http.get("/api/inference/images/segment/status").json()["downloaded"] is False
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: True)
+ started = {}
+ monkeypatch.setattr(
+ seg, "start_download", lambda token: started.setdefault("token", token) and "dl-1"
+ )
+ monkeypatch.setattr(seg, "download_status", lambda: {"downloading": True})
+ r = http.post("/api/inference/images/segment/download", json = {"hf_token": "hf_x"})
+ assert r.status_code == 200 and r.json()["download_id"] == "dl-1" and started["token"] == "hf_x"
+ monkeypatch.setattr(seg, "cancel_download", lambda: True)
+ r = http.post("/api/inference/images/segment/cancel")
+ assert r.status_code == 200 and r.json()["cancelled"] is True
+ released = []
+ monkeypatch.setattr(seg, "release_segmenter", lambda reason: released.append(reason) or True)
+ r = http.post("/api/inference/images/segment/release")
+ assert r.status_code == 200 and released == ["mask painter closed"]
+ monkeypatch.setattr(seg, "transformers_supports_sam3", lambda: False)
+ monkeypatch.setattr(seg, "status", lambda: {"supported": False, "reason": "old transformers"})
+ r = http.post("/api/inference/images/segment/download")
+ assert r.status_code == 501
+
+
+def test_a_no_torch_install_reports_sam3_unsupported(monkeypatch):
+ import importlib.util
+
+ real = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util, "find_spec", lambda name, *a: None if name == "torch" else real(name, *a)
+ )
+ assert seg.transformers_supports_sam3() is False
+ body = seg.status()
+ assert body["supported"] is False and "PyTorch" in body["reason"]
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 0d3624fa160..cfb46c84cfe 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -253,6 +253,9 @@ import {
withTransparencyPrompt,
} from "./edit-conditioning";
import { LocalizedEditCanvas } from "./localized-edit-canvas";
+import type { AppliedMask } from "./lib/mask-from-text";
+import { applyMaskToLayer } from "./mask-apply";
+import { MaskFromTextPanel } from "./mask-from-text-panel";
import {
type ControlNetSpecInput,
@@ -932,11 +935,14 @@ function MaskCanvas({
brushPct,
resetKey,
onMaskChange,
+ applyMask,
}: {
image: string;
brushPct: number;
resetKey: number;
onMaskChange: (dataUrl: string | null) => void;
+ /** A "Mask from text" result; each new key is drawn into the mask once, then the brush keeps editing it. */
+ applyMask?: AppliedMask | null;
}) {
const dispRef = useRef<HTMLCanvasElement | null>(null);
const maskRef = useRef<HTMLCanvasElement | null>(null);
@@ -973,6 +979,34 @@ function MaskCanvas({
img.src = image;
}, [image, resetKey, onMaskChange]);
+ // Applied once per key, after the mask is sized for this image; Clear or a new image starts blank as before. A
+ // result from before this canvas mounted (another workflow's painter) is not replayed.
+ const appliedKey = useRef<number | null>(applyMask?.key ?? null);
+ useEffect(() => {
+ if (!applyMask || !ready || appliedKey.current === applyMask.key) return;
+ appliedKey.current = applyMask.key;
+ const disp = dispRef.current;
+ const mask = maskRef.current;
+ if (!disp || !mask) return;
+ let live = true;
+ applyMaskToLayer({
+ layer: mask,
+ display: disp,
+ dataUrl: applyMask.dataUrl,
+ additive: applyMask.additive,
+ overlay: "mask",
+ isCurrent: () => live,
+ })
+ .then((url) => {
+ // An empty Replace leaves the mask black: nothing is selected, so Generate's empty-mask check still holds.
+ if (live && url !== null) onMaskChange(applyMask.empty ? null : url);
+ })
+ .catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [applyMask, ready, onMaskChange]);
+
const radius = useCallback(() => {
const base = Math.min(dims.current.w, dims.current.h) || 1024;
return Math.max(2, (brushPct / 100) * base);
@@ -1619,6 +1653,13 @@ export function ImagesPage({
const [maskImage, setMaskImage] = useState<string | null>(null);
const [brushPct, setBrushPct] = useState(8);
const [maskResetKey, setMaskResetKey] = useState(0);
+ // The latest "Mask from text" result, handed to whichever painter is showing; a new key applies it once.
+ const [textMask, setTextMask] = useState<AppliedMask | null>(null);
+ const onTextMask = useCallback(
+ (mask: { dataUrl: string; additive: boolean }) =>
+ setTextMask((prev) => ({ ...mask, key: (prev?.key ?? 0) + 1 })),
+ [],
+ );
// Extend (outpaint): how far to grow each dimension and which sides; reuses the inpaint backend at generate time.
const [extendPct, setExtendPct] = useState(25);
const [extendSides, setExtendSides] = useState<ExtendSides>({
@@ -5634,8 +5675,10 @@ export function ImagesPage({
brushPct={brushPct}
resetKey={maskResetKey}
onMaskChange={setMaskImage}
+ applyMask={textMask}
/>
</Field>
+ <MaskFromTextPanel image={initImage} onMask={onTextMask} />
<SliderField
label="Brush size"
hint="Brush radius as a percent of the image's shorter side."
@@ -5835,7 +5878,11 @@ export function ImagesPage({
resetKey={localizedResetKey}
onLayerChange={onLocalizedLayer}
onColorsChange={onLocalizedColors}
+ applyMask={textMask}
/>
+ {localizedMode !== "annotate" && (
+ <MaskFromTextPanel image={initImage} onMask={onTextMask} />
+ )}
<Button
type="button"
variant="secondary"
diff --git a/studio/frontend/src/features/images/lib/mask-from-text.ts b/studio/frontend/src/features/images/lib/mask-from-text.ts
new file mode 100644
index 00000000000..3217d688cd6
--- /dev/null
+++ b/studio/frontend/src/features/images/lib/mask-from-text.ts
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// "Mask from text": SAM 3 finds the objects a short text names and returns their union as a mask the inpaint and
+// localized-edit painters load, so it stays editable with the brush. Pure logic only (no "@/" imports) so node:test
+// can load it directly.
+
+/** Every segmenter route, in one place. */
+export const MASK_SEGMENT_ROUTES = {
+ status: "/api/inference/images/segment/status",
+ download: "/api/inference/images/segment/download",
+ cancel: "/api/inference/images/segment/cancel",
+ release: "/api/inference/images/segment/release",
+ segment: "/api/inference/images/segment",
+} as const;
+
+export const MASK_FROM_TEXT_LABEL = "Mask from text";
+export const MASK_FROM_TEXT_PLACEHOLDER = "sofa, red lamp, pillow:2";
+export const MASK_FROM_TEXT_HINT =
+ "Name what to mask, separated by commas. Each name selects its best match; add :N to select up to N, as in pillow:3. The mask lands in the painter, so you can still brush or erase it.";
+
+export interface SegmenterDownload {
+ downloading: boolean;
+ download_id: string | null;
+ error: string | null;
+ cancelled: boolean;
+ bytes_total: number | null;
+ bytes_done: number | null;
+}
+
+export interface SegmenterStatus {
+ supported: boolean;
+ reason: string | null;
+ downloaded: boolean;
+ loaded: boolean;
+ device: string | null;
+ repo: string;
+ gated: boolean;
+ license_url: string;
+ size_bytes: number | null;
+ download: SegmenterDownload;
+}
+
+export interface SegmentInstance {
+ prompt: string;
+ score: number;
+ box: [number, number, number, number];
+}
+
+export interface SegmentPromptResult {
+ text: string;
+ max_detections: number;
+ found: number;
+}
+
+export interface SegmentResult {
+ mask: string;
+ width: number;
+ height: number;
+ instances: SegmentInstance[];
+ prompts: SegmentPromptResult[];
+ device: string;
+ elapsed_ms: number;
+}
+
+export interface SegmentRequest {
+ image: string;
+ prompt: string;
+ threshold?: number;
+}
+
+/** What a painter receives: a new ``key`` applies ``dataUrl`` once. */
+export interface AppliedMask {
+ key: number;
+ dataUrl: string;
+ additive: boolean;
+ /** A Replace that found nothing: the painter clears its canvas and holds no mask (null), not a blank PNG. */
+ empty?: boolean;
+}
+
+const IDLE_DOWNLOAD: SegmenterDownload = {
+ downloading: false,
+ download_id: null,
+ error: null,
+ cancelled: false,
+ bytes_total: null,
+ bytes_done: null,
+};
+
+const num = (v: unknown): number | null =>
+ typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
+const str = (v: unknown): string | null => (typeof v === "string" ? v : null);
+const obj = (v: unknown): Record<string, unknown> =>
+ v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {};
+
+/** Tolerant parse of the status body: a missing or malformed field reads as "not there", never a throw. */
+export function parseSegmenterStatus(raw: unknown): SegmenterStatus {
+ const body = obj(raw);
+ const d = obj(body.download);
+ return {
+ supported: body.supported === true,
+ reason: str(body.reason),
+ downloaded: body.downloaded === true,
+ loaded: body.loaded === true,
+ device: str(body.device),
+ repo: str(body.repo) ?? "",
+ gated: body.gated === true,
+ license_url: str(body.license_url) ?? "",
+ size_bytes: num(body.size_bytes),
+ download: {
+ ...IDLE_DOWNLOAD,
+ downloading: d.downloading === true,
+ download_id: str(d.download_id),
+ error: str(d.error),
+ cancelled: d.cancelled === true,
+ bytes_total: num(d.bytes_total),
+ bytes_done: num(d.bytes_done),
+ },
+ };
+}
+
+function parseBox(v: unknown): [number, number, number, number] | null {
+ if (!Array.isArray(v) || v.length !== 4) return null;
+ const out = v.map((x) => (typeof x === "number" && Number.isFinite(x) ? x : NaN));
+ return out.some(Number.isNaN) ? null : (out as [number, number, number, number]);
+}
+
+/** Parse the segment response. Throws when there is no usable mask, since that is the one thing the painter needs. */
+export function parseSegmentResult(raw: unknown): SegmentResult {
+ const body = obj(raw);
+ const mask = str(body.mask);
+ if (!mask || !mask.startsWith("data:image/")) {
+ throw new Error("The segmenter returned no mask.");
+ }
+ const instances: SegmentInstance[] = [];
+ for (const item of Array.isArray(body.instances) ? body.instances : []) {
+ const it = obj(item);
+ const box = parseBox(it.box);
+ const score = typeof it.score === "number" && Number.isFinite(it.score) ? it.score : null;
+ if (box === null || score === null) continue;
+ instances.push({ prompt: str(it.prompt) ?? "", score, box });
+ }
+ const prompts: SegmentPromptResult[] = [];
+ for (const item of Array.isArray(body.prompts) ? body.prompts : []) {
+ const it = obj(item);
+ const text = str(it.text);
+ if (!text) continue;
+ prompts.push({
+ text,
+ max_detections: Math.max(1, Math.round(num(it.max_detections) ?? 1)),
+ found: Math.round(num(it.found) ?? 0),
+ });
+ }
+ return {
+ mask,
+ width: num(body.width) ?? 0,
+ height: num(body.height) ?? 0,
+ instances,
+ prompts,
+ device: str(body.device) ?? "",
+ elapsed_ms: num(body.elapsed_ms) ?? 0,
+ };
+}
+
+/** The Download control is offered: supported, and the weights are not on disk yet. */
+export function segmenterNeedsDownload(status: SegmenterStatus | null): boolean {
+ return status?.supported === true && !status.downloaded;
+}
+
+/** Find can run: supported, downloaded, and no download in flight. */
+export function segmenterReady(status: SegmenterStatus | null): boolean {
+ return status?.supported === true && status.downloaded && !status.download.downloading;
+}
+
+/** 0..1 download fraction, or null when the total is not known yet. */
+export function segmenterDownloadFraction(download: SegmenterDownload): number | null {
+ const total = download.bytes_total ?? 0;
+ if (!(total > 0)) return null;
+ return Math.min(1, Math.max(0, (download.bytes_done ?? 0) / total));
+}
+
+/** The detector's query count: the backend never keeps more of one concept. */
+export const MAX_DETECTIONS = 200;
+
+/** Python's round(): halves go to the even neighbour, so "pillow:2.5" keeps 2 here as it does in the backend. */
+function roundHalfEven(value: number): number {
+ const floor = Math.floor(value);
+ const diff = value - floor;
+ if (diff > 0.5) return floor + 1;
+ if (diff < 0.5) return floor;
+ return floor % 2 === 0 ? floor : floor + 1;
+}
+
+/** ComfyUI's SAM 3 prompt syntax (comfy/text_encoders/sam3_clip.py ``_parse_prompts``): parentheses dropped, comma
+ * separated concepts, an optional trailing ``:N`` for the most detections of that concept (rounded, at least 1,
+ * default 1). For the hint and validation only: the backend parses the prompt it runs. */
+export function parseMaskPrompt(text: string): Array<[string, number]> {
+ const cleaned = text.replace(/[()]/g, "");
+ const out: Array<[string, number]> = [];
+ for (const raw of cleaned.split(",")) {
+ const part = raw.trim();
+ if (!part) continue;
+ const m = /^(.+?)\s*:\s*([\d.]+)\s*$/.exec(part);
+ if (m) {
+ // Number(), not parseFloat: "2.5.1" is not a count, so the whole part stays the concept (as the backend reads it).
+ const value = Number(m[2]);
+ if (Number.isFinite(value)) {
+ out.push([m[1].trim(), Math.min(MAX_DETECTIONS, Math.max(1, roundHalfEven(value)))]);
+ continue;
+ }
+ }
+ out.push([part, 1]);
+ }
+ return out;
+}
+
+/** One line after a run: how many concepts matched and how many objects each gave. */
+export function segmentSummary(result: Pick<SegmentResult, "prompts" | "instances">): string {
+ const prompts = result.prompts;
+ if (prompts.length === 0) {
+ return result.instances.length > 0
+ ? `Found ${result.instances.length} object${result.instances.length === 1 ? "" : "s"}.`
+ : "Nothing found.";
+ }
+ const matched = prompts.filter((p) => p.found > 0);
+ if (matched.length === 0) return `Nothing found for ${prompts.map((p) => p.text).join(", ")}.`;
+ const parts = matched.map((p) => (p.max_detections > 1 ? `${p.text} x${p.found}` : p.text));
+ const missing = prompts.filter((p) => p.found === 0).map((p) => p.text);
+ const head = `Found ${matched.length} of ${prompts.length}: ${parts.join(", ")}.`;
+ return missing.length ? `${head} Not found: ${missing.join(", ")}.` : head;
+}
+
+/** The overlay colour each painter shows the mask in: red for inpaint and the separate mask, white for paint. */
+export const MASK_OVERLAY_RGBA: Readonly<Record<"mask" | "paint", readonly [number, number, number, number]>> = {
+ mask: [244, 114, 114, 140],
+ paint: [255, 255, 255, 255],
+};
+
+/** Turn a white-on-black layer's pixels (RGBA, in place copy) into the display overlay: black transparent, white
+ * the overlay colour scaled by its luminance. */
+export function tintMaskPixels(
+ layer: Uint8ClampedArray,
+ rgba: readonly [number, number, number, number],
+): Uint8ClampedArray<ArrayBuffer> {
+ const out = new Uint8ClampedArray(layer.length);
+ for (let i = 0; i < layer.length; i += 4) {
+ const v = Math.max(layer[i], layer[i + 1], layer[i + 2]);
+ if (v === 0) continue;
+ out[i] = rgba[0];
+ out[i + 1] = rgba[1];
+ out[i + 2] = rgba[2];
+ out[i + 3] = Math.round((rgba[3] * v) / 255);
+ }
+ return out;
+}
diff --git a/studio/frontend/src/features/images/localized-edit-canvas.tsx b/studio/frontend/src/features/images/localized-edit-canvas.tsx
index 01eecd705dc..bb013656398 100644
--- a/studio/frontend/src/features/images/localized-edit-canvas.tsx
+++ b/studio/frontend/src/features/images/localized-edit-canvas.tsx
@@ -11,6 +11,8 @@ import {
import type { LocalizedEditMode } from "./api";
import { ANNOTATION_COLORS, TRANSPARENCY_CHECKER } from "./edit-conditioning";
+import type { AppliedMask } from "./lib/mask-from-text";
+import { applyMaskToLayer } from "./mask-apply";
/** Draws the localized-edit layer over the source at its NATIVE resolution: an RGBA layer of
* coloured strokes for annotate, a white-on-black mask for paint and mask. */
@@ -22,6 +24,7 @@ export function LocalizedEditCanvas({
resetKey,
onLayerChange,
onColorsChange,
+ applyMask,
}: {
image: string;
mode: LocalizedEditMode;
@@ -30,6 +33,8 @@ export function LocalizedEditCanvas({
resetKey: number;
onLayerChange: (dataUrl: string | null) => void;
onColorsChange: (names: string[]) => void;
+ /** A "Mask from text" result for the paint and mask modes; each new key is drawn into the layer once. */
+ applyMask?: AppliedMask | null;
}) {
const dispRef = useRef<HTMLCanvasElement | null>(null);
const layerRef = useRef<HTMLCanvasElement | null>(null);
@@ -83,6 +88,38 @@ export function LocalizedEditCanvas({
};
}, [image, mode, resetKey, onLayerChange, onColorsChange]);
+ // A "Mask from text" result lands in the layer like a stroke, so the brush can still extend or Clear can drop it.
+ // Annotate layers are coloured outlines, not a mask, so they never take one.
+ // A result from before this canvas mounted (another painter) is not replayed.
+ const appliedKey = useRef<number | null>(applyMask?.key ?? null);
+ useEffect(() => {
+ if (!applyMask || !ready || mode === "annotate") return;
+ if (appliedKey.current === applyMask.key) return;
+ appliedKey.current = applyMask.key;
+ const disp = dispRef.current;
+ const layer = layerRef.current;
+ if (!disp || !layer) return;
+ let live = true;
+ applyMaskToLayer({
+ layer,
+ display: disp,
+ dataUrl: applyMask.dataUrl,
+ additive: applyMask.additive,
+ overlay: mode === "paint" ? "paint" : "mask",
+ isCurrent: () => live,
+ })
+ .then((url) => {
+ if (!live || url === null) return;
+ // An empty Replace leaves the layer black: nothing is selected, so the page's empty-layer checks still hold.
+ dirty.current = !applyMask.empty;
+ onLayerChange(applyMask.empty ? null : url);
+ })
+ .catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [applyMask, ready, mode, onLayerChange]);
+
const radius = useCallback(() => {
const base = Math.min(dims.current.w, dims.current.h) || 1024;
// Annotations are outlines, so a fraction of the region brush.
diff --git a/studio/frontend/src/features/images/mask-apply.ts b/studio/frontend/src/features/images/mask-apply.ts
new file mode 100644
index 00000000000..c7d5f9d0bf5
--- /dev/null
+++ b/studio/frontend/src/features/images/mask-apply.ts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Draw a "Mask from text" result into a painter's native-resolution white-on-black layer and rebuild its overlay, so
+// the mask stays editable with the same brush. Shared by the inpaint MaskCanvas and LocalizedEditCanvas.
+
+import { MASK_OVERLAY_RGBA, tintMaskPixels } from "./lib/mask-from-text.ts";
+
+function loadMaskImage(src: string): Promise<HTMLImageElement> {
+ return new Promise((resolve, reject) => {
+ const img = new Image();
+ img.onload = () => resolve(img);
+ img.onerror = () => reject(new Error("Could not read the mask image."));
+ img.src = src;
+ });
+}
+
+/** Replace (or, with ``additive``, union) ``layer`` with the mask at ``dataUrl``, scaled to the layer's size, then
+ * repaint ``display`` from the layer in the ``overlay`` colour. Resolves to the layer as a PNG data URL, or null if
+ * either canvas has no 2D context or ``isCurrent`` says the request went stale (Clear, a new source or a newer mask)
+ * while the image decoded: the canvases are reused, so a stale result never touches them. */
+export async function applyMaskToLayer({
+ layer,
+ display,
+ dataUrl,
+ additive,
+ overlay,
+ isCurrent,
+}: {
+ layer: HTMLCanvasElement;
+ display: HTMLCanvasElement;
+ dataUrl: string;
+ additive: boolean;
+ overlay: "mask" | "paint";
+ isCurrent?: () => boolean;
+}): Promise<string | null> {
+ const img = await loadMaskImage(dataUrl);
+ if (isCurrent && !isCurrent()) return null;
+ const w = layer.width;
+ const h = layer.height;
+ const lctx = layer.getContext("2d");
+ const dctx = display.getContext("2d");
+ if (!lctx || !dctx || w === 0 || h === 0) return null;
+ lctx.save();
+ if (!additive) {
+ lctx.globalCompositeOperation = "source-over";
+ lctx.fillStyle = "#000";
+ lctx.fillRect(0, 0, w, h);
+ }
+ // White on black either way: "lighten" keeps whatever is already white, so additive is a union.
+ lctx.globalCompositeOperation = "lighten";
+ lctx.imageSmoothingEnabled = true;
+ lctx.drawImage(img, 0, 0, w, h);
+ lctx.restore();
+ const pixels = lctx.getImageData(0, 0, w, h);
+ const tinted = tintMaskPixels(pixels.data, MASK_OVERLAY_RGBA[overlay]);
+ dctx.clearRect(0, 0, w, h);
+ dctx.putImageData(new ImageData(tinted, w, h), 0, 0);
+ return layer.toDataURL("image/png");
+}
diff --git a/studio/frontend/src/features/images/mask-from-text-panel.tsx b/studio/frontend/src/features/images/mask-from-text-panel.tsx
new file mode 100644
index 00000000000..a9f026830d4
--- /dev/null
+++ b/studio/frontend/src/features/images/mask-from-text-panel.tsx
@@ -0,0 +1,318 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Cancel01Icon, Download04Icon, Search01Icon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { useCallback, useEffect, useRef, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { InfoHint } from "@/components/ui/info-hint";
+import { Input } from "@/components/ui/input";
+import { Progress } from "@/components/ui/progress";
+import { Spinner } from "@/components/ui/spinner";
+import { Switch } from "@/components/ui/switch";
+import { useIsAccountOwner } from "@/features/auth";
+import { formatBytes, getHfToken, hfApiToken } from "@/features/hub";
+import { toast } from "@/lib/toast";
+import { cn } from "@/lib/utils";
+
+import {
+ MASK_FROM_TEXT_HINT,
+ MASK_FROM_TEXT_LABEL,
+ MASK_FROM_TEXT_PLACEHOLDER,
+ parseMaskPrompt,
+ segmenterDownloadFraction,
+ segmenterNeedsDownload,
+ segmenterReady,
+ segmentSummary,
+ type SegmenterStatus,
+} from "./lib/mask-from-text";
+import {
+ cancelSegmenterDownload,
+ getSegmenterStatus,
+ releaseSegmenter,
+ segmentImage,
+ startSegmenterDownload,
+} from "./mask-segment-api";
+
+/** Segmenter status, read once on mount and every second while a download runs. */
+function useSegmenterStatus() {
+ const [status, setStatus] = useState<SegmenterStatus | null>(null);
+ const mounted = useRef(true);
+ const refresh = useCallback(() => {
+ getSegmenterStatus()
+ .then((next) => {
+ if (mounted.current) setStatus(next);
+ })
+ .catch(() => {});
+ }, []);
+ const downloading = status?.download.downloading === true;
+ useEffect(() => {
+ mounted.current = true;
+ refresh();
+ return () => {
+ mounted.current = false;
+ };
+ }, [refresh]);
+ useEffect(() => {
+ if (!downloading) return;
+ const id = window.setInterval(refresh, 1000);
+ return () => window.clearInterval(id);
+ }, [downloading, refresh]);
+ // Re-read once a download ends, so the notice clears without a reload.
+ const wasDownloading = useRef(false);
+ useEffect(() => {
+ if (wasDownloading.current && !downloading) refresh();
+ wasDownloading.current = downloading;
+ }, [downloading, refresh]);
+ return { status, refresh };
+}
+
+function SegmenterDownloadNotice({
+ status,
+ onChanged,
+}: {
+ status: SegmenterStatus;
+ onChanged: () => void;
+}) {
+ const [busy, setBusy] = useState(false);
+ // The download and cancel routes are owner-only; a managed account is told who can fetch it instead.
+ const owner = useIsAccountOwner();
+ const { download } = status;
+ const fraction = segmenterDownloadFraction(download);
+ const size = status.size_bytes ? formatBytes(status.size_bytes) : null;
+ const start = async () => {
+ setBusy(true);
+ try {
+ await startSegmenterDownload(hfApiToken(getHfToken()));
+ } catch (err) {
+ toast.error("Could not download SAM 3", {
+ description: err instanceof Error ? err.message : undefined,
+ });
+ } finally {
+ setBusy(false);
+ onChanged();
+ }
+ };
+ const cancel = async () => {
+ try {
+ await cancelSegmenterDownload();
+ } catch (err) {
+ toast.error(err instanceof Error ? err.message : "Could not cancel the download");
+ } finally {
+ onChanged();
+ }
+ };
+ return (
+ <div
+ data-testid="mask-segmenter-download"
+ className="flex flex-col gap-1.5 rounded-md border border-border/60 bg-muted/30 px-2.5 py-2 text-ui-11"
+ >
+ {download.downloading ? (
+ <>
+ <div className="flex items-center justify-between gap-2">
+ <span className="text-muted-foreground">
+ {download.bytes_total
+ ? `Downloading SAM 3: ${formatBytes(download.bytes_done ?? 0)} of ${formatBytes(download.bytes_total)}`
+ : "Downloading SAM 3..."}
+ </span>
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className={cn("h-6 gap-1 px-1.5 text-xs", !owner && "hidden")}
+ onClick={() => void cancel()}
+ >
+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />
+ Cancel
+ </Button>
+ </div>
+ <Progress
+ className="h-1.5"
+ value={fraction === null ? undefined : fraction * 100}
+ indeterminate={fraction === null}
+ />
+ </>
+ ) : (
+ <>
+ <span className="text-muted-foreground">
+ {`Mask from text uses SAM 3${size ? ` (${size})` : ""}, which is not downloaded yet.`}
+ {status.gated ? (
+ <>
+ {" Its license must be accepted on Hugging Face first"}
+ {status.license_url ? (
+ <>
+ {": "}
+ <a
+ href={status.license_url}
+ target="_blank"
+ rel="noreferrer"
+ className="underline underline-offset-2"
+ >
+ {status.repo || "the model page"}
+ </a>
+ </>
+ ) : null}
+ {", and your Hugging Face token must be set."}
+ </>
+ ) : null}
+ </span>
+ {download.error && !download.cancelled ? (
+ <span className="text-destructive">{download.error}</span>
+ ) : null}
+ {owner ? (
+ <Button
+ type="button"
+ size="sm"
+ variant="secondary"
+ className="h-7 gap-1.5 self-start text-xs"
+ disabled={busy}
+ onClick={() => void start()}
+ >
+ {busy ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Download04Icon} className="size-3.5" />}
+ {size ? `Download (${size})` : "Download"}
+ </Button>
+ ) : (
+ <span className="text-muted-foreground">Ask the owner of this Unsloth install to download it.</span>
+ )}
+ </>
+ )}
+ </div>
+ );
+}
+
+/** "Mask from text": finds what the text names in ``image`` and hands the mask to the painter. */
+export function MaskFromTextPanel({
+ image,
+ onMask,
+}: {
+ image: string;
+ onMask: (mask: { dataUrl: string; additive: boolean; empty?: boolean }) => void;
+}) {
+ const { status, refresh } = useSegmenterStatus();
+ const [prompt, setPrompt] = useState("");
+ const [additive, setAdditive] = useState(false);
+ const [running, setRunning] = useState(false);
+ // The last outcome, kept with the source it was for: a new source hides it instead of an effect clearing it.
+ const [outcome, setOutcome] = useState<{ image: string; error: string | null; summary: string | null } | null>(
+ null,
+ );
+ const error = outcome?.image === image ? outcome.error : null;
+ const summary = outcome?.image === image ? outcome.summary : null;
+ const controllerRef = useRef<AbortController | null>(null);
+ const latestImage = useRef(image);
+
+ // A new source drops a run still in flight for the old one, so its mask never lands on the new image.
+ useEffect(() => {
+ latestImage.current = image;
+ controllerRef.current?.abort();
+ }, [image]);
+
+ // Closing the painter frees the segmenter instead of holding it for its keep-alive.
+ useEffect(
+ () => () => {
+ controllerRef.current?.abort();
+ releaseSegmenter().catch(() => {});
+ },
+ [],
+ );
+
+ const ready = segmenterReady(status);
+ const concepts = parseMaskPrompt(prompt);
+ const canRun = ready && !running && concepts.length > 0;
+
+ const run = async () => {
+ if (!canRun) return;
+ controllerRef.current?.abort();
+ const controller = new AbortController();
+ controllerRef.current = controller;
+ const source = image;
+ setRunning(true);
+ setOutcome(null);
+ try {
+ const result = await segmentImage({ image: source, prompt }, controller.signal);
+ if (controller.signal.aborted || latestImage.current !== source) return;
+ setOutcome({ image: source, error: null, summary: segmentSummary(result) });
+ if (result.instances.length > 0 || additive === false) {
+ // Nothing found with Replace still clears the mask: the painter shows exactly what the text selected.
+ onMask({ dataUrl: result.mask, additive, empty: result.instances.length === 0 });
+ }
+ } catch (err) {
+ if (controller.signal.aborted) return;
+ setOutcome({
+ image: source,
+ error: err instanceof Error ? err.message : "Could not make a mask from the text.",
+ summary: null,
+ });
+ // A 409 can mean the weights went missing; re-read so the download notice comes back.
+ refresh();
+ } finally {
+ if (controllerRef.current === controller) {
+ controllerRef.current = null;
+ setRunning(false);
+ }
+ }
+ };
+
+ return (
+ <div data-testid="mask-from-text" className="flex flex-col gap-1.5">
+ <div className="flex items-center gap-1">
+ <label htmlFor="mask-from-text-input" className="text-xs font-medium text-muted-foreground">
+ {MASK_FROM_TEXT_LABEL}
+ </label>
+ <InfoHint>{MASK_FROM_TEXT_HINT}</InfoHint>
+ </div>
+ {status && !status.supported ? (
+ <p className="text-ui-11 leading-snug text-muted-foreground">
+ {status.reason || "Mask from text is not available on this install."}
+ </p>
+ ) : null}
+ {segmenterNeedsDownload(status) && status ? (
+ <SegmenterDownloadNotice status={status} onChanged={refresh} />
+ ) : null}
+ <div className="flex gap-2">
+ <Input
+ id="mask-from-text-input"
+ value={prompt}
+ placeholder={MASK_FROM_TEXT_PLACEHOLDER}
+ disabled={!ready}
+ onChange={(e) => setPrompt(e.target.value)}
+ onKeyDown={(e) => {
+ if (e.key === "Enter") {
+ e.preventDefault();
+ void run();
+ }
+ }}
+ className="h-8 flex-1 text-xs"
+ />
+ <Button
+ type="button"
+ size="sm"
+ variant="secondary"
+ className="h-8 gap-1.5 text-xs"
+ disabled={!canRun}
+ onClick={() => void run()}
+ >
+ {running ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Search01Icon} className="size-3.5" />}
+ {running ? "Finding" : "Find"}
+ </Button>
+ </div>
+ <div className="flex items-center gap-2">
+ <Switch
+ size="sm"
+ checked={additive}
+ onCheckedChange={setAdditive}
+ disabled={!ready}
+ aria-label="Add to current mask"
+ />
+ <span className="text-ui-11 text-muted-foreground">Add to current mask</span>
+ </div>
+ {error ? <p className="text-ui-11 leading-snug text-destructive">{error}</p> : null}
+ {summary && !error ? (
+ <p data-testid="mask-from-text-summary" className="text-ui-11 leading-snug text-muted-foreground">
+ {summary}
+ </p>
+ ) : null}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/images/mask-segment-api.ts b/studio/frontend/src/features/images/mask-segment-api.ts
new file mode 100644
index 00000000000..df17fcb484e
--- /dev/null
+++ b/studio/frontend/src/features/images/mask-segment-api.ts
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// The "Mask from text" segmenter routes, kept out of api.ts like the prompt upsampler's: only the lazily loaded Images
+// page uses them.
+
+import { authFetch } from "@/features/auth";
+import { readFastApiError } from "@/lib/format-fastapi-error";
+
+import {
+ MASK_SEGMENT_ROUTES,
+ parseSegmenterStatus,
+ parseSegmentResult,
+ type SegmenterStatus,
+ type SegmentRequest,
+ type SegmentResult,
+} from "./lib/mask-from-text";
+
+async function parseJson<T>(response: Response): Promise<T> {
+ if (!response.ok) {
+ throw new Error(await readFastApiError(response));
+ }
+ return (await response.json()) as T;
+}
+
+export async function getSegmenterStatus(signal?: AbortSignal): Promise<SegmenterStatus> {
+ return parseSegmenterStatus(
+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.status, { signal })),
+ );
+}
+
+/** Start the background download of the SAM 3 weights. Progress is read back from the status route. */
+export async function startSegmenterDownload(hfToken?: string): Promise<void> {
+ await parseJson<unknown>(
+ await authFetch(MASK_SEGMENT_ROUTES.download, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(hfToken ? { hf_token: hfToken } : {}),
+ }),
+ );
+}
+
+/** Stop the download. Partial files stay cached, so a later download resumes. */
+export async function cancelSegmenterDownload(): Promise<void> {
+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.cancel, { method: "POST" }));
+}
+
+/** The mask for ``prompt`` on ``image``, at the image's own size. */
+export async function segmentImage(body: SegmentRequest, signal?: AbortSignal): Promise<SegmentResult> {
+ return parseSegmentResult(
+ await parseJson<unknown>(
+ await authFetch(MASK_SEGMENT_ROUTES.segment, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ signal,
+ }),
+ ),
+ );
+}
+
+/** The painter closed: free the segmenter now instead of holding it for its keep-alive. Best effort. */
+export function releaseSegmenter(): Promise<void> {
+ return (async () => {
+ await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.release, { method: "POST" }));
+ })();
+}
diff --git a/studio/frontend/tests/image-mask-from-text.test.ts b/studio/frontend/tests/image-mask-from-text.test.ts
new file mode 100644
index 00000000000..d7ed33aeae3
--- /dev/null
+++ b/studio/frontend/tests/image-mask-from-text.test.ts
@@ -0,0 +1,265 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { readSrc } from "./helpers/kit.ts";
+import {
+ MASK_OVERLAY_RGBA,
+ MASK_SEGMENT_ROUTES,
+ parseMaskPrompt,
+ parseSegmenterStatus,
+ parseSegmentResult,
+ segmenterDownloadFraction,
+ segmenterNeedsDownload,
+ segmenterReady,
+ segmentSummary,
+ tintMaskPixels,
+} from "../src/features/images/lib/mask-from-text.ts";
+
+const MASK = "data:image/png;base64,iVBORw0KGgo=";
+
+test("every segmenter route sits under the image inference prefix", () => {
+ assert.deepEqual(MASK_SEGMENT_ROUTES, {
+ status: "/api/inference/images/segment/status",
+ download: "/api/inference/images/segment/download",
+ cancel: "/api/inference/images/segment/cancel",
+ release: "/api/inference/images/segment/release",
+ segment: "/api/inference/images/segment",
+ });
+});
+
+test("the status parse tolerates missing and malformed fields", () => {
+ for (const raw of [null, undefined, 3, "x", [], {}]) {
+ const s = parseSegmenterStatus(raw);
+ assert.equal(s.supported, false);
+ assert.equal(s.downloaded, false);
+ assert.equal(s.size_bytes, null);
+ assert.equal(s.download.downloading, false);
+ assert.equal(s.download.bytes_total, null);
+ assert.equal(segmenterNeedsDownload(s), false);
+ assert.equal(segmenterReady(s), false);
+ }
+ const s = parseSegmenterStatus({
+ supported: true,
+ reason: null,
+ downloaded: false,
+ loaded: false,
+ device: null,
+ repo: "facebook/sam3",
+ gated: true,
+ license_url: "https://huggingface.co/facebook/sam3",
+ size_bytes: 3_439_938_512,
+ download: { downloading: true, download_id: "abc", error: null, cancelled: false, bytes_total: 100, bytes_done: 25 },
+ });
+ assert.equal(s.repo, "facebook/sam3");
+ assert.equal(s.gated, true);
+ assert.equal(s.size_bytes, 3_439_938_512);
+ assert.equal(s.download.download_id, "abc");
+ assert.equal(segmenterDownloadFraction(s.download), 0.25);
+ assert.equal(segmenterNeedsDownload(s), true);
+ // A download in flight is not ready even once the files report downloaded.
+ assert.equal(segmenterReady({ ...s, downloaded: true }), false);
+ assert.equal(segmenterReady({ ...s, downloaded: true, download: { ...s.download, downloading: false } }), true);
+ // Negative or non-numeric sizes read as unknown.
+ assert.equal(parseSegmenterStatus({ size_bytes: -1 }).size_bytes, null);
+ assert.equal(segmenterDownloadFraction({ ...s.download, bytes_total: null }), null);
+ assert.equal(segmenterDownloadFraction({ ...s.download, bytes_total: 10, bytes_done: 50 }), 1);
+});
+
+test("an unsupported install never offers the download", () => {
+ const s = parseSegmenterStatus({ supported: false, reason: "Needs transformers 5", downloaded: false });
+ assert.equal(s.reason, "Needs transformers 5");
+ assert.equal(segmenterNeedsDownload(s), false);
+ assert.equal(segmenterReady(s), false);
+ assert.equal(segmenterNeedsDownload(null), false);
+});
+
+test("the segment result parse keeps well-formed instances and requires a mask", () => {
+ const r = parseSegmentResult({
+ mask: MASK,
+ width: 640,
+ height: 480,
+ instances: [
+ { prompt: "sofa", score: 0.91, box: [1, 2, 300, 200] },
+ { prompt: "bad", score: "high", box: [1, 2, 3, 4] },
+ { prompt: "bad box", score: 0.5, box: [1, 2] },
+ ],
+ prompts: [
+ { text: "sofa", max_detections: 1, found: 1 },
+ { text: "pillow", max_detections: 2, found: 0 },
+ { text: "", max_detections: 1, found: 0 },
+ ],
+ device: "cuda:0",
+ elapsed_ms: 412,
+ });
+ assert.equal(r.width, 640);
+ assert.equal(r.instances.length, 1);
+ assert.deepEqual(r.instances[0].box, [1, 2, 300, 200]);
+ assert.equal(r.prompts.length, 2);
+ assert.equal(r.device, "cuda:0");
+ assert.throws(() => parseSegmentResult({}), /no mask/);
+ assert.throws(() => parseSegmentResult({ mask: "not a data url" }), /no mask/);
+});
+
+test("the prompt parser matches ComfyUI's SAM 3 syntax", () => {
+ assert.deepEqual(parseMaskPrompt("Sofa:2"), [["Sofa", 2]]);
+ assert.deepEqual(parseMaskPrompt("(red lamp), window, pillow:3"), [
+ ["red lamp", 1],
+ ["window", 1],
+ ["pillow", 3],
+ ]);
+ assert.deepEqual(parseMaskPrompt("a:2.6"), [["a", 3]]);
+ assert.deepEqual(parseMaskPrompt("a:0"), [["a", 1]]);
+ assert.deepEqual(parseMaskPrompt("a : 4 "), [["a", 4]]);
+ assert.deepEqual(parseMaskPrompt(" , sofa,, ,"), [["sofa", 1]]);
+ assert.deepEqual(parseMaskPrompt(""), []);
+ // A colon not followed by a number stays part of the concept.
+ assert.deepEqual(parseMaskPrompt("note: red"), [["note: red", 1]]);
+ // Python's round() (half to even), as the backend parses it.
+ assert.deepEqual(parseMaskPrompt("a:2.5, b:3.5, c:0.5"), [["a", 2], ["b", 4], ["c", 1]]);
+ // Not a number: the whole part is the concept.
+ assert.deepEqual(parseMaskPrompt("v:2.5.1"), [["v:2.5.1", 1]]);
+ assert.deepEqual(parseMaskPrompt("grain:5000"), [["grain", 200]]);
+});
+
+test("the summary names what was found and what was not", () => {
+ assert.equal(
+ segmentSummary({
+ prompts: [
+ { text: "sofa", max_detections: 1, found: 1 },
+ { text: "pillow", max_detections: 3, found: 2 },
+ { text: "lamp", max_detections: 1, found: 0 },
+ ],
+ instances: [],
+ }),
+ "Found 2 of 3: sofa, pillow x2. Not found: lamp.",
+ );
+ assert.equal(
+ segmentSummary({ prompts: [{ text: "cat", max_detections: 1, found: 0 }], instances: [] }),
+ "Nothing found for cat.",
+ );
+ assert.equal(segmentSummary({ prompts: [], instances: [] }), "Nothing found.");
+ for (const text of ["x"]) {
+ const line = segmentSummary({ prompts: [{ text, max_detections: 1, found: 1 }], instances: [] });
+ assert.ok(!/—/.test(line));
+ }
+});
+
+test("the overlay tint keeps black transparent and colours white", () => {
+ const layer = new Uint8ClampedArray([0, 0, 0, 255, 255, 255, 255, 255, 128, 128, 128, 255]);
+ const out = tintMaskPixels(layer, MASK_OVERLAY_RGBA.mask);
+ assert.deepEqual([...out.slice(0, 4)], [0, 0, 0, 0]);
+ assert.deepEqual([...out.slice(4, 8)], [244, 114, 114, 140]);
+ assert.equal(out[11], 70);
+ assert.deepEqual([...tintMaskPixels(layer, MASK_OVERLAY_RGBA.paint).slice(4, 8)], [255, 255, 255, 255]);
+});
+
+test("both painters take a text mask and the page renders the panel for inpaint and mask or paint edits", () => {
+ const page = readSrc("features/images/images-page.tsx");
+ const localized = readSrc("features/images/localized-edit-canvas.tsx");
+ const apply = readSrc("features/images/mask-apply.ts");
+ assert.match(page, /function MaskCanvas\([\s\S]*?applyMask\?: AppliedMask \| null;/);
+ assert.match(localized, /applyMask\?: AppliedMask \| null;/);
+ // Not for annotate: its layer is coloured outlines, not a mask.
+ assert.match(localized, /mode === "annotate"\) return;/);
+ assert.match(page, /onMaskChange=\{setMaskImage\}\s*applyMask=\{textMask\}\s*\/>\s*<\/Field>\s*<MaskFromTextPanel image=\{initImage\}/);
+ assert.match(page, /applyMask=\{textMask\}\s*\/>\s*\{localizedMode !== "annotate" && \(\s*<MaskFromTextPanel/);
+ // Replace clears to black first; additive unions with "lighten".
+ assert.match(apply, /if \(!additive\) \{[\s\S]*?fillRect/);
+ assert.match(apply, /globalCompositeOperation = "lighten"/);
+ // Each painter replays only results that arrive after it mounted.
+ assert.equal((page.match(/useRef<number \| null>\(applyMask\?\.key \?\? null\)/g) ?? []).length, 1);
+ assert.equal((localized.match(/useRef<number \| null>\(applyMask\?\.key \?\? null\)/g) ?? []).length, 1);
+});
+
+test("the panel releases the segmenter on unmount and uses the shared download idioms", () => {
+ const panel = readSrc("features/images/mask-from-text-panel.tsx");
+ const api = readSrc("features/images/mask-segment-api.ts");
+ assert.match(panel, /releaseSegmenter\(\)\.catch/);
+ assert.match(panel, /useIsAccountOwner/);
+ assert.match(panel, /hfApiToken\(getHfToken\(\)\)/);
+ assert.match(api, /MASK_SEGMENT_ROUTES\.segment/);
+ assert.match(api, /readFastApiError/);
+ for (const src of [panel, api]) assert.ok(!/—/.test(src), "no em dashes in UI copy");
+});
+
+test("a text mask that went stale while it decoded never touches the reused canvases", async () => {
+ const { applyMaskToLayer } = await import("../src/features/images/mask-apply.ts");
+ const g = globalThis as Record<string, unknown>;
+ const saved = { Image: g.Image, ImageData: g.ImageData };
+ // The decode resolves on a later tick, as a real <img> does.
+ g.Image = class {
+ onload: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ set src(_v: string) {
+ setTimeout(() => this.onload?.(), 0);
+ }
+ };
+ g.ImageData = class {
+ data: Uint8ClampedArray;
+ constructor(data: Uint8ClampedArray) {
+ this.data = data;
+ }
+ };
+ const calls: string[] = [];
+ const ctx = new Proxy(
+ {},
+ {
+ get: (_t, key) =>
+ key === "getImageData"
+ ? () => ({ data: new Uint8ClampedArray(4) })
+ : typeof key === "string" && /^[a-z]+[A-Z]|^(save|restore|fillRect|drawImage)$/.test(key)
+ ? () => calls.push(key)
+ : undefined,
+ set: (_t, key) => {
+ calls.push(`set:${String(key)}`);
+ return true;
+ },
+ },
+ );
+ const canvas = () =>
+ ({ width: 1, height: 1, getContext: () => ctx, toDataURL: () => "data:image/png;base64,AA==" }) as unknown as HTMLCanvasElement;
+ try {
+ let live = true;
+ const stale = applyMaskToLayer({
+ layer: canvas(),
+ display: canvas(),
+ dataUrl: MASK,
+ additive: false,
+ overlay: "mask",
+ isCurrent: () => live,
+ });
+ live = false; // Clear, a new source or a newer mask before the decode finished
+ assert.equal(await stale, null);
+ assert.deepEqual(calls, []);
+ const fresh = await applyMaskToLayer({
+ layer: canvas(),
+ display: canvas(),
+ dataUrl: MASK,
+ additive: false,
+ overlay: "mask",
+ isCurrent: () => true,
+ });
+ assert.equal(fresh, "data:image/png;base64,AA==");
+ assert.ok(calls.includes("drawImage") && calls.includes("putImageData"));
+ } finally {
+ g.Image = saved.Image;
+ g.ImageData = saved.ImageData;
+ }
+ // Both painters pass their effect's liveness.
+ assert.match(readSrc("features/images/images-page.tsx"), /overlay: "mask",\s*isCurrent: \(\) => live,/);
+ assert.match(readSrc("features/images/localized-edit-canvas.tsx"), /isCurrent: \(\) => live,/);
+});
+
+test("a Replace that found nothing leaves both painters holding no mask", () => {
+ const panel = readSrc("features/images/mask-from-text-panel.tsx");
+ const page = readSrc("features/images/images-page.tsx");
+ const localized = readSrc("features/images/localized-edit-canvas.tsx");
+ // The canvas still clears (the black PNG is applied), but the result is flagged as selecting nothing...
+ assert.match(panel, /onMask\(\{ dataUrl: result\.mask, additive, empty: result\.instances\.length === 0 \}\)/);
+ // ...so the logical value stays null and the page's !maskImage / !localizedLayer checks still refuse Generate.
+ assert.match(page, /onMaskChange\(applyMask\.empty \? null : url\)/);
+ assert.match(localized, /dirty\.current = !applyMask\.empty;\s*onLayerChange\(applyMask\.empty \? null : url\)/);
+});