Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Images: Detailer (face fix) pass with SAM 3 and inpaint (#14616)

Daniel Hanedd613689240
diff --git a/studio/backend/core/inference/diffusion_detailer.py b/studio/backend/core/inference/diffusion_detailer.py
new file mode 100644
index 00000000000..7493269c7f1
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_detailer.py
@@ -0,0 +1,813 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Detailer (face fix) post-pass, after ADetailer and ComfyUI-Impact-Pack's FaceDetailer: SAM 3 finds each target,
+its padded crop is enlarged, inpainted at a moderate denoise and pasted back through a feathered mask (pixels outside
+the mask stay bit-identical). Defaults follow FaceDetailer; prompts, [SEP]/[PROMPT] and infotext keys follow ADetailer.
+Departures: the mask is SAM 3's dilated instance mask, not the box, and SAM 3 is parked in host RAM before the first
+inpaint so it never sits on the GPU beside the image model.
+"""
+
+from __future__ import annotations
+
+import math
+
+import base64
+import contextlib
+import io
+import json
+import re
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+# Defaults: FaceDetailer, modules/impact/impact_pack.py:743-769; utils.py:454.
+DEFAULT_TARGETS = "face"
+DEFAULT_CONFIDENCE = 0.5 # bbox_threshold
+DEFAULT_DENOISE = 0.5 # denoise
+DEFAULT_FEATHER = 5 # feather (kernel 2 * feather + 1)
+FEATHER_SIGMA = 10.0 # tensor_gaussian_blur_mask(mask, kernel_size, sigma = 10.0)
+DEFAULT_DILATION = 10 # bbox_dilation
+DEFAULT_CROP_FACTOR = 3.0 # bbox_crop_factor
+DEFAULT_GUIDE_SIZE = 512 # guide_size
+DEFAULT_GUIDE_SIZE_FOR = "bbox" # guide_size_for: True = "bbox", False = "crop_region"
+DEFAULT_MAX_SIZE = 1024 # max_size
+DEFAULT_DROP_SIZE = 10 # drop_size
+DEFAULT_PER_TARGET = 8
+MAX_PER_TARGET = 20
+MAX_DETECTIONS = 32
+# Matches diffusion.py _snap_to_multiple so inpaint never resizes.
+SIZE_MULTIPLE = 16
+_SEED_SPACE = 2**53
+
+DETAILER_UNAVAILABLE_INPAINT = (
+ "The Detailer needs a model with an inpaint pipeline; this model has none."
+)
+
+_MAX_SUFFIX = re.compile(r"^(.+?)\s*:\s*([\d.]+)\s*$")
+_SEP = re.compile(r"\s*\[SEP\]\s*")
+
+
+class DetailerCancelled(RuntimeError):
+ """Stop pressed between two inpaints."""
+
+
+# Running passes' stop events -> owning account (None: no account scope).
+_active: dict[threading.Event, Optional[str]] = {}
+_active_lock = threading.Lock()
+
+
+def cancel_active(account: Optional[str] = None) -> bool:
+ """Stop running passes (all when ``account`` is None); True when one was stopped."""
+ with _active_lock:
+ hit = [event for event, owner in _active.items() if account is None or owner == account]
+ for event in hit:
+ event.set()
+ return bool(hit)
+
+
+@dataclass(frozen = True)
+class DetailerSettings:
+ targets: str = DEFAULT_TARGETS
+ prompt: Optional[str] = None
+ negative_prompt: Optional[str] = None
+ confidence: float = DEFAULT_CONFIDENCE
+ denoise: float = DEFAULT_DENOISE
+ crop_factor: float = DEFAULT_CROP_FACTOR
+ guide_size: int = DEFAULT_GUIDE_SIZE
+ guide_size_for: str = DEFAULT_GUIDE_SIZE_FOR
+ max_size: int = DEFAULT_MAX_SIZE
+ dilation: int = DEFAULT_DILATION
+ feather: int = DEFAULT_FEATHER
+ drop_size: int = DEFAULT_DROP_SIZE
+ steps: Optional[int] = None
+ guidance: Optional[float] = None
+
+ @classmethod
+ def from_request(cls, spec: Any) -> "DetailerSettings":
+ values = {}
+ for name in cls.__dataclass_fields__:
+ value = getattr(spec, name, None)
+ if value is not None:
+ values[name] = value
+ settings = cls(**values)
+ if not target_concepts(settings.targets):
+ raise ValueError("Name what the Detailer should fix, for example: face")
+ from core.inference import diffusion_segment
+
+ # Validate SAM 3 limits before the render, not after.
+ diffusion_segment.parse_prompt(segment_prompt(settings.targets))
+ return settings
+
+
+@dataclass
+class Detection:
+ concept: str
+ score: float
+ box: tuple[float, float, float, float]
+ # Full-size bool mask, or None with ``plane`` (low-res logits) expanded lazily to bound batch memory.
+ mask: Any = None
+ plane: Any = None
+
+ def full_mask(self, height: int, width: int) -> Any:
+ if self.mask is not None:
+ return self.mask
+ return _plane_to_mask(self.plane, height, width)
+
+
+@dataclass
+class DetailResult:
+ image: Any
+ record: dict = field(default_factory = dict)
+
+
+def target_concepts(targets: Optional[str]) -> list[tuple[str, int]]:
+ """``face, hand:2`` -> [("face", DEFAULT_PER_TARGET), ("hand", 2)]."""
+ out: list[tuple[str, int]] = []
+ text = (targets or "").replace("(", "").replace(")", "")
+ for part in (p.strip() for p in text.split(",")):
+ if not part:
+ continue
+ match = _MAX_SUFFIX.match(part)
+ label, count = part, DEFAULT_PER_TARGET
+ if match:
+ try:
+ label, count = match.group(1).strip(), max(1, round(float(match.group(2))))
+ except (ValueError, OverflowError):
+ label, count = part, DEFAULT_PER_TARGET
+ if label:
+ out.append((label, min(count, MAX_PER_TARGET)))
+ return out
+
+
+def segment_prompt(targets: Optional[str]) -> str:
+ return ", ".join(f"{label}:{count}" for label, count in target_concepts(targets))
+
+
+def _normalize_region(limit: int, start: int, size: float) -> tuple[int, int]:
+ """Impact-Pack utils.normalize_region."""
+ if start < 0:
+ return 0, int(min(limit, size))
+ if start + size > limit:
+ return int(max(0, limit - size)), int(limit)
+ return int(start), int(min(limit, start + size))
+
+
+def crop_region(
+ width: int, height: int, box: tuple[float, float, float, float], crop_factor: float
+) -> tuple[int, int, int, int]:
+ """Impact-Pack utils.make_crop_region."""
+ x1, y1, x2, y2 = box
+ bbox_w, bbox_h = x2 - x1, y2 - y1
+ crop_w, crop_h = bbox_w * crop_factor, bbox_h * crop_factor
+ new_x1 = int(x1 + bbox_w / 2 - crop_w / 2)
+ new_y1 = int(y1 + bbox_h / 2 - crop_h / 2)
+ cx1, cx2 = _normalize_region(width, new_x1, crop_w)
+ cy1, cy2 = _normalize_region(height, new_y1, crop_h)
+ return cx1, cy1, cx2, cy2
+
+
+def upscaled_size(
+ bbox_w: float,
+ bbox_h: float,
+ crop_w: int,
+ crop_h: int,
+ guide_size: int,
+ guide_size_for: str,
+ max_size: int,
+) -> tuple[float, int, int]:
+ """Impact-Pack core.enhance_detail sizing with force_inpaint: never shrinks."""
+ short = min(bbox_w, bbox_h) if guide_size_for == "bbox" else min(crop_w, crop_h)
+ upscale = guide_size / short if short > 0 else 1.0
+ new_w, new_h = int(crop_w * upscale), int(crop_h * upscale)
+ if new_w > max_size or new_h > max_size:
+ upscale *= max_size / max(new_w, new_h)
+ new_w, new_h = int(crop_w * upscale), int(crop_h * upscale)
+ if upscale <= 1.0 or new_w == 0 or new_h == 0:
+ return 1.0, crop_w, crop_h
+ return upscale, new_w, new_h
+
+
+def snap_size(
+ width: int,
+ height: int,
+ multiple: int = SIZE_MULTIPLE,
+) -> tuple[int, int]:
+ return max(multiple, width // multiple * multiple), max(multiple, height // multiple * multiple)
+
+
+def dilate(mask: Any, pixels: int) -> Any:
+ """cv2.dilate / cv2.erode (Impact-Pack utils.dilate_masks), OpenCV border semantics."""
+ import numpy as np
+
+ mask = np.asarray(mask, dtype = bool)
+ size = abs(int(pixels))
+ if size <= 1:
+ return mask.copy()
+ grow = pixels > 0
+ anchor = size // 2
+ out = mask.copy()
+ for axis in (0, 1):
+ src, acc = out, out.copy()
+ length = src.shape[axis]
+ for k in range(size):
+ shift = k - anchor
+ if shift == 0:
+ continue
+ moved = np.full_like(src, not grow)
+ lo, hi = max(0, -shift), min(length, length - shift)
+ if lo < hi:
+ dst = [slice(None)] * 2
+ srcs = [slice(None)] * 2
+ dst[axis] = slice(lo, hi)
+ srcs[axis] = slice(lo + shift, hi + shift)
+ moved[tuple(dst)] = src[tuple(srcs)]
+ acc = acc | moved if grow else acc & moved
+ out = acc
+ return out
+
+
+def _gaussian_kernel(size: int, sigma: float) -> Any:
+ """torchvision's GaussianBlur kernel (transforms.functional._get_gaussian_kernel1d)."""
+ import numpy as np
+
+ half = (size - 1) * 0.5
+ x = np.linspace(-half, half, size, dtype = np.float32)
+ pdf = np.exp(-0.5 * (x / np.float32(sigma)) ** 2)
+ return (pdf / pdf.sum()).astype(np.float32)
+
+
+def feather_mask(
+ mask: Any,
+ feather: int,
+ sigma: float = FEATHER_SIGMA,
+) -> Any:
+ """Impact-Pack utils.tensor_gaussian_blur_mask."""
+ import numpy as np
+
+ alpha = np.asarray(mask, dtype = np.float32)
+ if feather <= 0:
+ return alpha
+ size = feather * 2 + 1
+ shortest = min(alpha.shape)
+ if shortest <= size:
+ size = int(shortest / 2)
+ if size % 2 == 0:
+ size += 1
+ if size < 3:
+ return alpha
+ kernel = _gaussian_kernel(size, sigma)
+ pad = size // 2
+ for axis in (0, 1):
+ padding = [(0, 0), (0, 0)]
+ padding[axis] = (pad, pad)
+ padded = np.pad(alpha, padding, mode = "reflect")
+ acc = np.zeros_like(alpha)
+ for k in range(size):
+ index = [slice(None)] * 2
+ index[axis] = slice(k, k + alpha.shape[axis])
+ acc += kernel[k] * padded[tuple(index)]
+ alpha = acc
+ return np.clip(alpha, 0.0, 1.0)
+
+
+def composite(base: Any, patch: Any, alpha: Any) -> Any:
+ """Impact-Pack utils.tensor_paste; alpha 0 keeps ``base`` byte for byte."""
+ import numpy as np
+
+ base = np.asarray(base, dtype = np.uint8)
+ patch = np.asarray(patch, dtype = np.uint8)
+ a = np.asarray(alpha, dtype = np.float32)[..., None]
+ mixed = base.astype(np.float32) * (1.0 - a) + patch.astype(np.float32) * a
+ mixed = np.clip(np.rint(mixed), 0, 255).astype(np.uint8)
+ return np.where(a > 0, mixed, base)
+
+
+def detection_seed(base_seed: int, index: int) -> int:
+ return (int(base_seed) + int(index)) % _SEED_SPACE
+
+
+def detection_prompt(template: Optional[str], main: str, index: int) -> Optional[str]:
+ """ADetailer [SEP]/[PROMPT] rule (last part repeats); None means [SKIP]."""
+ parts = _SEP.split(template or "")
+ part = parts[min(index, len(parts) - 1)].strip()
+ if part == "[SKIP]":
+ return None
+ if not part:
+ return main
+ return part.replace("[PROMPT]", main)
+
+
+def inherits_main(template: Optional[str]) -> bool:
+ if not template:
+ return True
+ return any(not part.strip() or "[PROMPT]" in part for part in _SEP.split(template))
+
+
+def resolve_inherited(template: str, main: str) -> str:
+ return " [SEP] ".join(
+ main if not part.strip() else part.strip().replace("[PROMPT]", main)
+ for part in _SEP.split(template)
+ )
+
+
+def schedulable_steps(steps: int, denoise: float) -> int:
+ """Smallest steps >= ``steps`` with int(steps * denoise) >= 1; inpaint pipelines refuse zero timesteps."""
+ n = max(1, int(steps))
+ denoise = float(denoise)
+ if not (0.0 < denoise < 1.0): # also NaN
+ return n
+ # Closed form plus a bounded nudge for float rounding; counting up would loop forever at tiny denoise.
+ m = max(n, math.ceil(1.0 / denoise))
+ for _ in range(4):
+ if m > n and int((m - 1) * denoise) >= 1:
+ m -= 1
+ elif int(m * denoise) < 1:
+ m += 1
+ else:
+ break
+ return m
+
+
+InpaintFn = Callable[[Any, Any, int, str, Optional[str]], Any]
+
+
+def _big_enough(box: tuple[float, float, float, float], drop_size: int) -> bool:
+ x1, y1, x2, y2 = box
+ return x2 - x1 > drop_size and y2 - y1 > drop_size
+
+
+def detail_image(
+ image: Any,
+ detections: list[Detection],
+ settings: DetailerSettings,
+ *,
+ base_seed: int,
+ prompt: str,
+ negative_prompt: Optional[str],
+ inpaint: InpaintFn,
+ stop: Optional[Callable[[], bool]] = None,
+) -> DetailResult:
+ import numpy as np
+ from PIL import Image
+
+ started = time.perf_counter()
+ current = np.array(image.convert("RGB"), dtype = np.uint8)
+ height, width = current.shape[:2]
+ done: list[dict] = []
+ # Undersized boxes take no [SEP] prompt, seed or cap slot (Impact-Pack drops them at detection).
+ sized = [d for d in detections if _big_enough(d.box, settings.drop_size)]
+ skipped = len(detections) - len(sized)
+ for index, det in enumerate(sized[:MAX_DETECTIONS]):
+ if stop is not None and stop():
+ raise DetailerCancelled("cancelled")
+ x1, y1, x2, y2 = det.box
+ det_prompt = detection_prompt(settings.prompt, prompt, index)
+ if det_prompt is None:
+ skipped += 1
+ continue
+ det_negative = (
+ detection_prompt(settings.negative_prompt, negative_prompt or "", index)
+ if settings.negative_prompt
+ else negative_prompt
+ )
+ cx1, cy1, cx2, cy2 = crop_region(width, height, det.box, settings.crop_factor)
+ crop_w, crop_h = cx2 - cx1, cy2 - cy1
+ if crop_w <= 0 or crop_h <= 0:
+ skipped += 1
+ continue
+ mask = dilate(
+ np.asarray(det.full_mask(height, width), dtype = bool)[cy1:cy2, cx1:cx2],
+ settings.dilation,
+ )
+ if not mask.any():
+ skipped += 1
+ continue
+ _, new_w, new_h = upscaled_size(
+ x2 - x1,
+ y2 - y1,
+ crop_w,
+ crop_h,
+ settings.guide_size,
+ settings.guide_size_for,
+ settings.max_size,
+ )
+ new_w, new_h = snap_size(new_w, new_h)
+ crop = Image.fromarray(current[cy1:cy2, cx1:cx2])
+ crop_up = crop.resize((new_w, new_h), Image.LANCZOS)
+ mask_up = Image.fromarray(mask.astype(np.uint8) * 255, mode = "L").resize(
+ (new_w, new_h), Image.NEAREST
+ )
+ seed = detection_seed(base_seed, index)
+ rendered = inpaint(crop_up, mask_up, seed, det_prompt, det_negative)
+ patch = np.array(rendered.convert("RGB").resize((crop_w, crop_h), Image.LANCZOS))
+ alpha = feather_mask(mask, settings.feather)
+ current[cy1:cy2, cx1:cx2] = composite(current[cy1:cy2, cx1:cx2], patch, alpha)
+ entry = {
+ "target": det.concept,
+ "score": round(float(det.score), 4),
+ "box": [int(round(v)) for v in det.box],
+ "crop": [cx1, cy1, cx2, cy2],
+ "size": [new_w, new_h],
+ "seed": seed,
+ }
+ if det_prompt != prompt:
+ entry["prompt"] = det_prompt
+ done.append(entry)
+ record = {
+ **settings_record(settings),
+ "detections": done,
+ "found": len(detections),
+ "elapsed_ms": round((time.perf_counter() - started) * 1000.0, 1),
+ }
+ if skipped:
+ record["skipped"] = skipped
+ if len(sized) > MAX_DETECTIONS:
+ record["dropped"] = len(sized) - MAX_DETECTIONS
+ return DetailResult(Image.fromarray(current), record)
+
+
+def settings_record(settings: DetailerSettings) -> dict:
+ out = {
+ "targets": settings.targets,
+ "confidence": settings.confidence,
+ "denoise": settings.denoise,
+ "crop_factor": settings.crop_factor,
+ "guide_size": settings.guide_size,
+ "guide_size_for": settings.guide_size_for,
+ "max_size": settings.max_size,
+ "dilation": settings.dilation,
+ "feather": settings.feather,
+ "drop_size": settings.drop_size,
+ }
+ for name in ("prompt", "negative_prompt", "steps", "guidance"):
+ value = getattr(settings, name)
+ if value not in (None, ""):
+ out[name] = value
+ return out
+
+
+# Infotext keys: adetailer/args.py _all_args.
+def _quote(value: Any) -> str:
+ """AUTOMATIC1111 infotext_utils.quote."""
+ text = str(value)
+ if "," not in text and "\n" not in text and ":" not in text:
+ return text
+ return json.dumps(text, ensure_ascii = False)
+
+
+def infotext_fields(record: Optional[dict], model: Optional[str] = None) -> str:
+ """``, ADetailer model: ...`` for the parameters line, or "" when no detailer ran."""
+ if not record or not isinstance(record, dict) or record.get("error"):
+ return ""
+ pairs: list[tuple[str, Any]] = [
+ ("ADetailer model", model or record.get("model") or "sam3"),
+ ("ADetailer model classes", record.get("targets")),
+ ]
+ if record.get("prompt"):
+ pairs.append(("ADetailer prompt", record["prompt"]))
+ if record.get("negative_prompt"):
+ pairs.append(("ADetailer negative prompt", record["negative_prompt"]))
+ pairs += [
+ ("ADetailer confidence", record.get("confidence")),
+ ("ADetailer dilate erode", record.get("dilation")),
+ ("ADetailer mask blur", record.get("feather")),
+ ("ADetailer denoising strength", record.get("denoise")),
+ ("ADetailer inpaint only masked", True),
+ ]
+ if record.get("steps") is not None:
+ pairs += [("ADetailer use separate steps", True), ("ADetailer steps", record["steps"])]
+ if record.get("guidance") is not None:
+ pairs += [
+ ("ADetailer use separate CFG scale", True),
+ ("ADetailer CFG scale", record["guidance"]),
+ ]
+ return "".join(f", {key}: {_quote(value)}" for key, value in pairs if value is not None)
+
+
+def check_available(status: dict) -> None:
+ """Raise ValueError (400) without an inpaint pipeline (covers sd.cpp) or SAM 3."""
+ from core.inference import diffusion_segment
+
+ if not status.get("loaded"):
+ return # the render itself answers "no model loaded"
+ if "inpaint" not in (status.get("workflows") or []):
+ raise ValueError(DETAILER_UNAVAILABLE_INPAINT)
+ if not diffusion_segment.transformers_supports_sam3():
+ raise ValueError(
+ "The Detailer finds faces with SAM 3, which needs transformers 5 (Python 3.10 or newer)."
+ )
+ if diffusion_segment.model_dir() is None:
+ variant = diffusion_segment.active_variant()
+ raise ValueError(
+ f"The Detailer finds faces with {variant.label}, which is not downloaded. Download it from "
+ "Mask from text in the Inpaint tab first."
+ )
+
+
+def _plane_to_mask(plane: Any, height: int, width: int) -> Any:
+ import torch
+ import torch.nn.functional as F
+ with torch.inference_mode():
+ up = F.interpolate(
+ plane[None, None].float(), size = (height, width), mode = "bilinear", align_corners = False
+ )[0, 0]
+ return (up > 0).numpy()
+
+
+def _take_generation_slot(stop: Optional[Callable[[], bool]] = None) -> Any:
+ """Acquire the active engine's generation lock (None if it has none); cancellable while waiting."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ lock = getattr(get_active_diffusion_engine(), "_generate_lock", None)
+ except Exception: # noqa: BLE001 -- no engine: no render to collide with
+ return None
+ if lock is None or not hasattr(lock, "acquire"):
+ return None
+ while not lock.acquire(timeout = 0.1):
+ if stop is not None and stop():
+ raise DetailerCancelled("cancelled")
+ return lock
+
+
+def detect(
+ images: list,
+ settings: DetailerSettings,
+ stop: Optional[Callable[[], bool]] = None,
+) -> tuple[list[list[Detection]], dict]:
+ """SAM 3 on every image, then parked in host RAM; at most MAX_DETECTIONS sized planes per image."""
+ from core.inference import diffusion_segment
+
+ prompt = segment_prompt(settings.targets)
+ out: list[list[Detection]] = []
+ found: list[int] = []
+ undersized: list[int] = []
+ info: dict = {}
+ started = time.perf_counter()
+ # Hold the generation slot so a queued render waits for SAM 3 to leave the GPU.
+ slot = _take_generation_slot(stop)
+ try:
+ segmenter = diffusion_segment.get_segmenter()
+ hold = getattr(segmenter, "hold", None)
+ with contextlib.ExitStack() as held:
+ if slot is not None:
+ # Register as slot holder so this account's cancel is not refused as foreign.
+ from hub.services.models.account_access import media_generation_slot
+ held.enter_context(media_generation_slot("diffusion"))
+ if hold is not None:
+ held.enter_context(hold())
+ for image in images:
+ if stop is not None and stop():
+ raise DetailerCancelled("cancelled")
+ result = segmenter.segment(
+ image.convert("RGB"),
+ prompt,
+ threshold = settings.confidence,
+ instance_masks = True,
+ plane_filter = lambda box: _big_enough(box, settings.drop_size),
+ max_planes = MAX_DETECTIONS,
+ )
+ pairs = list(zip(result.instances, result.instance_logits))
+ found.append(len(pairs))
+ sized = [p for p in pairs if _big_enough(p[0].box, settings.drop_size)]
+ undersized.append(len(pairs) - len(sized))
+ out.append(
+ [
+ Detection(inst.concept, inst.score, inst.box, plane = plane)
+ for inst, plane in sized[:MAX_DETECTIONS]
+ ]
+ )
+ info = {"device": result.device, "precision": result.precision}
+ del result, pairs, sized
+ finally:
+ try:
+ diffusion_segment.release_segmenter("detailer", wait = True, park = True)
+ finally:
+ if slot is not None:
+ slot.release()
+ info["model"] = diffusion_segment.active_variant().id
+ info["found"] = found
+ info["undersized"] = undersized
+ info["detect_ms"] = round((time.perf_counter() - started) * 1000.0, 1)
+ return out, info
+
+
+# A same-repo reload differing in any of these is another model.
+BUILD_KEYS = (
+ "repo_id",
+ "model_kind",
+ "gguf_filename",
+ "transformer_quant",
+ "text_encoder_quant",
+ "baked_loras",
+)
+
+
+def same_build(render: dict, inpaint: dict) -> bool:
+ """Keys missing from either result are not compared."""
+ for key in BUILD_KEYS:
+ if key in render and key in inpaint:
+ a, b = render[key], inpaint[key]
+ if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
+ a, b = list(a), list(b)
+ if a != b:
+ return False
+ return True
+
+
+def _png_b64(image: Any) -> str:
+ buffer = io.BytesIO()
+ image.save(buffer, format = "PNG", compress_level = 1)
+ return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii")
+
+
+def detail_result(
+ generate: Callable[..., dict],
+ result: dict,
+ settings: DetailerSettings,
+ *,
+ prompt_for: Callable[[int], str],
+ negative_prompt: Optional[str],
+ steps: int,
+ guidance: float,
+ generate_kwargs: dict,
+ account: Optional[str] = None,
+) -> None:
+ """Detail a finished render in place; failures are recorded per image and the render is kept."""
+ images = list(result.get("images") or [])
+ if not images:
+ return
+ renders = list(images)
+ stop_event = threading.Event()
+ with _active_lock:
+ _active[stop_event] = account
+ finished = False
+ try:
+ _detail_images(
+ generate,
+ result,
+ images,
+ settings,
+ stop_event,
+ prompt_for = prompt_for,
+ negative_prompt = negative_prompt,
+ steps = steps,
+ guidance = guidance,
+ generate_kwargs = generate_kwargs,
+ )
+ finished = True
+ finally:
+ with _active_lock:
+ _active.pop(stop_event, None)
+ # Under the lock: a Stop cancel_active acknowledged is honoured even after the last inpaint.
+ stopped = stop_event.is_set()
+ if finished and stopped:
+ result["images"] = renders
+ result["detailer"] = [
+ record
+ if record and record.get("error")
+ else {**settings_record(settings), "error": "cancelled"}
+ for record in (result.get("detailer") or [None] * len(renders))
+ ]
+
+
+def _detail_images(
+ generate: Callable[..., dict],
+ result: dict,
+ images: list,
+ settings: DetailerSettings,
+ stop_event: threading.Event,
+ *,
+ prompt_for: Callable[[int], str],
+ negative_prompt: Optional[str],
+ steps: int,
+ guidance: float,
+ generate_kwargs: dict,
+) -> None:
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+
+ seeds = result.get("seeds") or []
+ records: list[Optional[dict]] = [None] * len(images)
+ try:
+ found, info = detect(images, settings, stop = stop_event.is_set)
+ except Exception as exc: # noqa: BLE001 -- the render stands; the recipe says why it was not detailed
+ cancelled = isinstance(exc, DetailerCancelled)
+ if not cancelled:
+ logger.warning("images.detailer: detection failed: %s", exc)
+ error = "cancelled" if cancelled else str(exc)
+ result["detailer"] = [{**settings_record(settings), "error": error} for _ in images]
+ return
+ totals = info.get("found") or []
+ tiny = info.get("undersized") or []
+ step_count = schedulable_steps(
+ settings.steps if settings.steps is not None else steps, settings.denoise
+ )
+ cfg = settings.guidance if settings.guidance is not None else guidance
+ cancelled = stop_event.is_set()
+ for index, image in enumerate(images):
+ base_seed = seeds[index] if index < len(seeds) else result.get("seed") or 0
+ if cancelled:
+ records[index] = {**settings_record(settings), "error": "cancelled"}
+ continue
+
+ inherited: dict = {}
+
+ def _inpaint(crop, mask, seed, det_prompt, det_negative):
+ out = generate(
+ prompt = det_prompt,
+ negative_prompt = det_negative,
+ width = crop.width,
+ height = crop.height,
+ steps = step_count,
+ guidance = cfg,
+ seed = seed,
+ batch_size = 1,
+ init_image = _png_b64(crop),
+ mask_image = _png_b64(mask),
+ strength = settings.denoise,
+ **generate_kwargs,
+ )
+ if not same_build(result, out):
+ raise ValueError("the model changed")
+ if det_negative and out.get("negative_prompt") and not result.get("negative_prompt"):
+ # Negative prompt engaged only at the Detailer's CFG: record it.
+ if not settings.negative_prompt:
+ inherited["negative_prompt"] = det_negative
+ elif negative_prompt and inherits_main(settings.negative_prompt):
+ inherited["negative_prompt"] = resolve_inherited(
+ settings.negative_prompt, negative_prompt
+ )
+ return out["images"][0]
+
+ try:
+ detailed = detail_image(
+ image,
+ found[index],
+ settings,
+ base_seed = int(base_seed),
+ prompt = prompt_for(index),
+ negative_prompt = negative_prompt,
+ inpaint = _inpaint,
+ stop = stop_event.is_set,
+ )
+ except Exception as exc: # noqa: BLE001
+ if isinstance(exc, DetailerCancelled) or str(exc) == DIFFUSION_CANCELLED_MSG:
+ cancelled = True
+ else:
+ logger.warning("images.detailer: image %d failed: %s", index, exc)
+ records[index] = {
+ **settings_record(settings),
+ "error": "cancelled" if cancelled else str(exc),
+ }
+ continue
+ images[index] = detailed.image
+ records[index] = {
+ **detailed.record,
+ **inherited,
+ "model": info.get("model"),
+ "detect_device": info.get("device"),
+ }
+ small = tiny[index] if index < len(tiny) else 0
+ if small:
+ records[index]["skipped"] = records[index].get("skipped", 0) + small
+ total = totals[index] if index < len(totals) else None
+ if total is not None and total > len(found[index]):
+ records[index]["found"] = total
+ if total - small > len(found[index]):
+ records[index]["dropped"] = total - small - len(found[index])
+ if index == 0:
+ records[index]["detect_ms"] = info.get("detect_ms")
+ result["images"] = images
+ result["detailer"] = records
+
+
+__all__ = [
+ "DEFAULT_CONFIDENCE",
+ "DEFAULT_CROP_FACTOR",
+ "DEFAULT_DENOISE",
+ "DEFAULT_DILATION",
+ "DEFAULT_DROP_SIZE",
+ "DEFAULT_FEATHER",
+ "DEFAULT_GUIDE_SIZE",
+ "DEFAULT_MAX_SIZE",
+ "DEFAULT_TARGETS",
+ "DetailerSettings",
+ "Detection",
+ "check_available",
+ "composite",
+ "crop_region",
+ "detail_image",
+ "detail_result",
+ "detect",
+ "dilate",
+ "feather_mask",
+ "infotext_fields",
+ "segment_prompt",
+ "target_concepts",
+ "upscaled_size",
+]
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 782a9aba80e..04ff43c63fa 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -1029,7 +1029,8 @@ def _metal_working_set_mib() -> tuple[Optional[int], int]:
def _graph_held_cache_bytes(device: Optional[int] = None) -> int:
"""Bytes inside ``memory_reserved() - memory_allocated()`` that ordinary allocations cannot reuse: the CUDA graph
- pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part)."""
+ pools (per-block pools whole, the whole-step graphs' shared pool's unallocated part). ``device`` None counts every
+ block pool and reads the whole-step pool on the current card."""
held = 0
try:
from .diffusion_block_graph import pool_bytes as _block_graph_pool_bytes
@@ -1037,13 +1038,49 @@ def _graph_held_cache_bytes(device: Optional[int] = None) -> int:
except Exception: # noqa: BLE001
pass
try:
+ # The whole-step pool reading is per current card, so a named card other than it is made current for the read.
from .diffusion_cuda_graph import live_pool_free_bytes
- held += int(live_pool_free_bytes())
+
+ scope = contextlib.nullcontext()
+ if device is not None:
+ import torch
+ try:
+ current = int(torch.cuda.current_device())
+ except Exception: # noqa: BLE001 - no current card to compare: switch to the named one
+ current = None
+ if current != int(device):
+ scope = torch.cuda.device(int(device))
+ with scope:
+ held += int(live_pool_free_bytes())
except Exception: # noqa: BLE001 -- no graph layer: nothing held
pass
return held
+def cuda_reclaimable_bytes(device: Optional[int] = None) -> Optional[int]:
+ """Bytes this process's caching allocator holds reserved-but-unallocated on CUDA ``device`` (the current card
+ when None) that an ordinary allocation can take back, minus the CUDA graph pools that only their graphs may reuse
+ (``_graph_held_cache_bytes``). None when the allocator cannot be read. May be negative; callers credit only a
+ positive figure."""
+ try:
+ import torch
+ if device is None:
+ reclaimable = int(torch.cuda.memory_reserved()) - int(torch.cuda.memory_allocated())
+ else:
+ reclaimable = int(torch.cuda.memory_reserved(device)) - int(
+ torch.cuda.memory_allocated(device)
+ )
+ except Exception: # noqa: BLE001
+ return None
+ pool_device = device
+ if pool_device is None:
+ try:
+ pool_device = torch.cuda.current_device()
+ except Exception: # noqa: BLE001 - no current CUDA device: count every pool
+ pool_device = None
+ return reclaimable - _graph_held_cache_bytes(pool_device)
+
+
def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
"""``snapshot_device_memory`` with the caching allocator's RECLAIMABLE bytes credited back as free,
without flushing it.
@@ -1074,16 +1111,9 @@ def reclaimable_snapshot_device_memory(target: Any) -> DeviceMemory:
snapshot = snapshot_device_memory(target)
if snapshot.free_mib is None:
return snapshot
- try:
- import torch
- reclaimable = int(torch.cuda.memory_reserved()) - int(torch.cuda.memory_allocated())
- except Exception: # noqa: BLE001 -- no allocator reading: the plain snapshot still stands
+ reclaimable = cuda_reclaimable_bytes()
+ if reclaimable is None: # no allocator reading: the plain snapshot still stands
return snapshot
- try:
- device = torch.cuda.current_device()
- except Exception: # noqa: BLE001 - no current CUDA device: count every pool
- device = None
- reclaimable -= _graph_held_cache_bytes(device)
if reclaimable <= 0:
return snapshot
free = int(snapshot.free_mib) + reclaimable // (1024 * 1024)
diff --git a/studio/backend/core/inference/diffusion_segment.py b/studio/backend/core/inference/diffusion_segment.py
index 9a3ba86db47..097809b5071 100644
--- a/studio/backend/core/inference/diffusion_segment.py
+++ b/studio/backend/core/inference/diffusion_segment.py
@@ -46,7 +46,7 @@ import time
from contextlib import contextmanager, suppress
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Any, Iterator, Optional
+from typing import Any, Callable, Iterator, Optional
from loggers import get_logger
@@ -152,6 +152,11 @@ ENV_VARIANT = "UNSLOTH_SAM3_VARIANT"
ENV_PATH_OVERRIDE = "UNSLOTH_SAM3_PATH"
ENV_KEEP_ALIVE = "UNSLOTH_SAM3_KEEP_ALIVE"
+# Seconds a host-RAM parked model is kept; 0 never parks.
+ENV_PARK_SECONDS = "UNSLOTH_SAM3_PARK_SECONDS"
+PARK_SECONDS = 600.0
+# "1" pins the parked copy (0.03 s unpark vs ~0.6 s); opt-in since pinning 1.7 GB took minutes on a loaded host.
+ENV_PARK_PIN = "UNSLOTH_SAM3_PARK_PIN"
# ── ComfyUI SAM3_Detect defaults ────────────────────────────────────────────────────────────────────────────────────
IMAGE_SIZE = 1008
@@ -815,12 +820,22 @@ def _resident_cuda_ordinal() -> Optional[int]:
def _cuda_free_mib(ordinal: int) -> tuple[Optional[int], Optional[int]]:
+ """(free, total) MiB on CUDA card ``ordinal``: driver free plus this process's reusable allocator cache, the same
+ credit the render's memory check gives (diffusion_memory.cuda_reclaimable_bytes). Parking leaves SAM's emptied
+ weights in that cache on purpose, so driver free alone would send the next detection to the CPU on a warm card
+ (ComfyUI's get_free_memory likewise adds reserved minus active bytes)."""
try:
import torch
free, total = torch.cuda.mem_get_info(ordinal)
- return int(free // (1024 * 1024)), int(total // (1024 * 1024))
except Exception: # noqa: BLE001
return None, None
+ try:
+ from core.inference.diffusion_memory import cuda_reclaimable_bytes
+ reclaimable = cuda_reclaimable_bytes(ordinal) or 0
+ except Exception: # noqa: BLE001 -- no allocator reading: driver free still stands
+ reclaimable = 0
+ free = min(int(free) + max(0, int(reclaimable)), int(total))
+ return int(free // (1024 * 1024)), int(total // (1024 * 1024))
def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:
@@ -903,6 +918,8 @@ class SegmentResult:
device: str = "cpu"
elapsed_ms: float = 0.0
precision: Optional[str] = None
+ # Per-instance CPU mask logits (instance_masks only, never in the route body); None for filtered planes.
+ instance_logits: list = field(default_factory = list)
def to_json(self) -> dict:
return {
@@ -1015,6 +1032,113 @@ def _free_device_cache(device: Optional[str]) -> None:
torch.mps.empty_cache()
+class _HostWeights:
+ """Parked weights as one host buffer and one device storage every weight views (via ``Tensor.set_``), so
+ park/unpark is one copy instead of ~1,500 allocations, which on a busy host is slower than rereading the file."""
+
+ _ALIGN = 256
+
+ def __init__(self, pin: bool) -> None:
+ self.pin = pin
+ self.buffer: Any = None
+ self.storage: Any = None
+ self.total = 0
+
+ @staticmethod
+ def _leaves(model: Any) -> list:
+ import torch
+
+ out: list = []
+
+ def _visit(t: Any) -> None:
+ if type(t) not in (torch.Tensor, torch.nn.Parameter):
+ flatten = getattr(t, "__tensor_flatten__", None)
+ if flatten is None:
+ raise TypeError(f"cannot park a {type(t).__name__} tensor")
+ names, _ctx = flatten()
+ for name in names:
+ _visit(getattr(t, name))
+ return
+ out.append(t)
+
+ # The registered objects themselves, not ``.data`` aliases: set_ has to re-point the tensor the module uses.
+ for param in model.parameters():
+ _visit(param)
+ for buffer in model.buffers():
+ _visit(buffer)
+ return out
+
+ def _bytes(
+ self,
+ storage: Any,
+ nbytes: int,
+ offset: int = 0,
+ ) -> Any:
+ import torch
+ view = torch.empty(0, dtype = torch.uint8, device = storage.device)
+ return view.set_(storage, offset, (nbytes,), (1,))
+
+ def _repack(self, model: Any) -> None:
+ import torch
+
+ leaves = self._leaves(model)
+ slots: dict = {}
+ offset = 0
+ for leaf in leaves:
+ storage = leaf.untyped_storage()
+ key = storage.data_ptr()
+ if key in slots or storage.nbytes() == 0:
+ continue
+ slots[key] = (storage, offset)
+ offset += -(-storage.nbytes() // self._ALIGN) * self._ALIGN
+ self.total = offset
+ self.buffer = torch.empty(self.total, dtype = torch.uint8, pin_memory = self.pin)
+ storages = [st for st, _off in slots.values()]
+ if storages:
+ torch._foreach_copy_(
+ [self.buffer[off : off + st.nbytes()] for st, off in slots.values()],
+ [self._bytes(st, st.nbytes()) for st in storages],
+ non_blocking = self.pin,
+ )
+ if storages[0].device.type == "cuda":
+ torch.cuda.synchronize(storages[0].device)
+ device = storages[0].device if storages else torch.device("cpu")
+ packed = torch.empty(self.total, dtype = torch.uint8, device = device).untyped_storage()
+ placed = []
+ for leaf in leaves:
+ storage = leaf.untyped_storage()
+ slot = slots.get(storage.data_ptr())
+ if slot is None:
+ continue
+ size, stride, start = leaf.size(), leaf.stride(), leaf.storage_offset()
+ placed.append((leaf, slot[1] // leaf.element_size() + start, size, stride))
+ # Re-pointed only once every offset is known; the old storages go with their last reference.
+ for leaf, start, size, stride in placed:
+ with torch.no_grad():
+ leaf.set_(packed, start, size, stride)
+ del storages, slots, placed
+ self.storage = packed
+
+ def park(self, model: Any) -> None:
+ import torch
+ if self.storage is None:
+ self._repack(model)
+ elif self.total:
+ self.buffer.copy_(self._bytes(self.storage, self.total), non_blocking = self.pin)
+ if self.storage.device.type == "cuda":
+ torch.cuda.synchronize(self.storage.device)
+ self.storage.resize_(0)
+
+ def unpark(self, model: Any, device: str) -> None:
+ import torch
+
+ self.storage.resize_(self.total)
+ if self.total:
+ self._bytes(self.storage, self.total).copy_(self.buffer, non_blocking = self.pin)
+ if device.startswith("cuda"):
+ torch.cuda.synchronize(device)
+
+
class MaskSegmenter:
"""One lazily loaded Sam3Model + its tokenizer. ``_run_lock`` serialises loads, calls and unloads."""
@@ -1042,6 +1166,8 @@ class MaskSegmenter:
# Planning reads the Hub cache only for the active variant's own snapshot, not an explicit folder.
self._use_cache = True
self._timer: Optional[threading.Timer] = None
+ # Batch holds (hold()): while one is open a call keeps the model loaded; the holder releases it at the end.
+ self._holds = 0
self._release_pending = False
self._release_epoch = 0
# Bumped by every GPU handover's release: a call that queued before one never runs on the accelerator.
@@ -1051,6 +1177,16 @@ class MaskSegmenter:
# Set (under _state_lock) once the call in flight has read its release flag: a release that times out after
# that is not honoured by the call, so it waits out the call's short tail itself.
self._call_settled = False
+ # A GPU model moved to host RAM by release(park = True), with what _load matches it against; the next call on
+ # the same card, weights and dtype moves it back instead of reading the files again. Never counts as loaded.
+ self._parked: Optional[dict] = None
+ self._park_timer: Optional[threading.Timer] = None
+ # The weights' host copy, kept while the same model is loaded so a later park copies nothing back.
+ self._host: Optional[_HostWeights] = None
+ try:
+ self.park_seconds = max(0.0, float(os.environ.get(ENV_PARK_SECONDS) or PARK_SECONDS))
+ except ValueError:
+ self.park_seconds = PARK_SECONDS
@property
def loaded(self) -> bool:
@@ -1059,6 +1195,10 @@ class MaskSegmenter:
def holds_vram(self) -> bool:
return self._model is not None and bool(self.device) and self.device != "cpu"
+ @property
+ def parked(self) -> bool:
+ return self._parked is not None
+
# -- lifecycle -------------------------------------------------------------------------------------------------
def _load(self, path: str, device: str, dtype: Any) -> None:
plan = plan_weights(path, device, dtype, use_cache = self._use_cache)
@@ -1073,6 +1213,8 @@ class MaskSegmenter:
return
if self._model is not None:
self._unload_locked("reload")
+ if self._unpark_locked(path, key, device, dtype):
+ return
from transformers import AutoTokenizer
started = time.perf_counter()
@@ -1185,6 +1327,137 @@ class MaskSegmenter:
model.eval()
return model, plan
+ def _unpark_locked(self, path: str, key: tuple, device: str, dtype: Any) -> bool:
+ """Unpark when weights, card and dtype match (else drop the parked copy); False when nothing was usable."""
+ parked = self._parked
+ if parked is None:
+ return False
+ if (parked["path"], parked["key"], parked["device"], parked["dtype"]) != (
+ path,
+ key,
+ device,
+ dtype,
+ ):
+ self._drop_parked_locked("another load")
+ return False
+ self._parked = None
+ self._cancel_park_timer()
+ started = time.perf_counter()
+ model = parked["model"]
+ try:
+ parked["host"].unpark(model, device)
+ except BaseException:
+ model = None
+ parked.clear()
+ self._host = None
+ _free_device_cache(device)
+ logger.warning(
+ "images.segment: the parked model did not move back to %s; loading again", device
+ )
+ return False
+ self._model, self._tokenizer = model, parked["tokenizer"]
+ self._host = parked["host"]
+ self._path, self.device, self._dtype, self._weights_key = path, device, dtype, key
+ self.precision, self.precision_reason, self.weights_file = (
+ parked["precision"],
+ parked["precision_reason"],
+ parked["weights_file"],
+ )
+ logger.info(
+ "images.segment: moved back from host RAM to %s in %.2fs",
+ device,
+ time.perf_counter() - started,
+ )
+ return True
+
+ def _park_locked(self, reason: str) -> bool:
+ """Move the GPU model to host RAM and free its VRAM, keeping it for the next call. Falls back to an unload
+ on a CPU / MPS model, with parking off, or when the move fails. True when the weights left the device."""
+ if self._model is None:
+ return False
+ device = self.device
+ if self.park_seconds <= 0 or not (device or "").startswith("cuda"):
+ return self._unload_locked(reason)
+ with self._state_lock:
+ if self._timer is not None:
+ self._timer.cancel()
+ self._timer = None
+ started = time.perf_counter()
+ model = self._model
+ host = self._host or _HostWeights(pin = (os.environ.get(ENV_PARK_PIN) or "").strip() == "1")
+ try:
+ host.park(model)
+ copied = time.perf_counter() - started
+ except Exception as exc: # noqa: BLE001 -- a module that will not move is simply unloaded
+ logger.warning("images.segment: could not park in host RAM (%s); unloading", exc)
+ self._host = None
+ return self._unload_locked(reason)
+ self._drop_parked_locked("replaced")
+ self._host = None
+ self._parked = {
+ "model": model,
+ "host": host,
+ "tokenizer": self._tokenizer,
+ "path": self._path,
+ "key": self._weights_key,
+ "device": device,
+ "dtype": self._dtype,
+ "precision": self.precision,
+ "precision_reason": self.precision_reason,
+ "weights_file": self.weights_file,
+ }
+ self._model = self._tokenizer = None
+ self._path = self._weights_key = self.device = self._dtype = None
+ self.precision = self.precision_reason = self.weights_file = None
+ del model
+ # No empty_cache: the flush took minutes on a busy card; the cache counts as free for the render's check.
+ self._arm_park_timer()
+ logger.info(
+ "images.segment: parked in host RAM from %s in %.2fs (copy %.2fs) (%s)",
+ device,
+ time.perf_counter() - started,
+ copied,
+ reason,
+ )
+ return True
+
+ def _drop_parked_locked(self, reason: str) -> bool:
+ self._cancel_park_timer()
+ if self._parked is None:
+ return False
+ device = self._parked.get("device")
+ self._parked = None
+ self._host = None
+ _free_device_cache(device)
+ logger.info("images.segment: dropped the host RAM copy (%s)", reason)
+ return True
+
+ def _cancel_park_timer(self) -> None:
+ with self._state_lock:
+ if self._park_timer is not None:
+ self._park_timer.cancel()
+ self._park_timer = None
+
+ def _arm_park_timer(self) -> None:
+ with self._state_lock:
+ if self._park_timer is not None:
+ self._park_timer.cancel()
+ timer = threading.Timer(self.park_seconds, self._park_expire)
+ timer.daemon = True
+ self._park_timer = timer
+ timer.start()
+
+ def _park_expire(self) -> None:
+ if not self._run_lock.acquire(blocking = False):
+ # A call is running: it either takes the parked copy or leaves it for a new timer.
+ if self._parked is not None:
+ self._arm_park_timer()
+ return
+ try:
+ self._drop_parked_locked("park expired")
+ finally:
+ self._run_lock.release()
+
def _unload_locked(self, reason: str) -> bool:
with self._state_lock:
if self._timer is not None:
@@ -1195,6 +1468,7 @@ class MaskSegmenter:
device = self.device
self._model = None
self._tokenizer = None
+ self._host = None
self._path = None
self._weights_key = None
self.device = None
@@ -1216,10 +1490,13 @@ class MaskSegmenter:
wait: bool = False,
timeout: float = 30.0,
handover: bool = False,
+ park: bool = False,
) -> bool:
"""Unload now if idle. A call in flight unloads when it ends (``wait`` waits up to ``timeout`` for it
instead). True when the weights left now. ``handover`` (the GPU is changing hands) raises
- SegmenterReleaseTimeoutError when the wait ran out on a call still on the accelerator."""
+ SegmenterReleaseTimeoutError when the wait ran out on a call still on the accelerator. ``park`` (an image
+ render takes the GPU back) moves a GPU model to host RAM for the next call instead of dropping it; without
+ it any parked copy is dropped too."""
with self._state_lock:
self._release_epoch += 1
if handover:
@@ -1248,10 +1525,23 @@ class MaskSegmenter:
try:
with self._state_lock:
self._release_pending = False
- return self._unload_locked(reason)
+ if park:
+ return self._park_locked(reason)
+ dropped = self._drop_parked_locked(reason)
+ return self._unload_locked(reason) or dropped
finally:
self._run_lock.release()
+ @contextmanager
+ def hold(self) -> Iterator[None]:
+ with self._state_lock:
+ self._holds += 1
+ try:
+ yield
+ finally:
+ with self._state_lock:
+ self._holds -= 1
+
def _arm_keep_alive(self) -> None:
with self._state_lock:
if self._timer is not None:
@@ -1282,6 +1572,10 @@ class MaskSegmenter:
if pending:
self._unload_locked("released during a call")
return
+ with self._state_lock:
+ held = self._holds > 0
+ if held:
+ return # a batch pass: its holder parks or unloads once, after the last image
if self.device and self.device.startswith("cuda"):
free_mib, total_mib = _cuda_free_mib(int(self.device.split(":", 1)[1]))
if vram_is_tight(free_mib, total_mib):
@@ -1301,7 +1595,11 @@ class MaskSegmenter:
threshold: float = DEFAULT_THRESHOLD,
allow_gpu: bool = True,
path: Optional[str] = None,
+ instance_masks: bool = False,
+ plane_filter: Optional[Callable[[tuple[float, float, float, float]], bool]] = None,
+ max_planes: Optional[int] = None,
) -> SegmentResult:
+ """With ``plane_filter`` / ``max_planes``, only those planes leave the device and no union mask is built."""
concepts = parse_prompt(prompt)
if not concepts:
raise ValueError("Describe what to mask, for example: sofa, red lamp, pillow:2")
@@ -1327,20 +1625,40 @@ class MaskSegmenter:
with self._state_lock:
self._call_device, self._call_settled = "", False
try:
- return self._segment_locked(image, concepts, threshold, path, allow_gpu)
+ return self._segment_locked(
+ image,
+ concepts,
+ threshold,
+ path,
+ allow_gpu,
+ instance_masks,
+ plane_filter,
+ max_planes,
+ )
finally:
with self._state_lock:
self._call_device, self._call_settled = None, False
def _segment_locked(
- self, image: Any, concepts: list[Concept], threshold: float, path: str, allow_gpu: bool
+ self,
+ image: Any,
+ concepts: list[Concept],
+ threshold: float,
+ path: str,
+ allow_gpu: bool,
+ instance_masks: bool = False,
+ plane_filter: Optional[Callable[[tuple[float, float, float, float]], bool]] = None,
+ max_planes: Optional[int] = None,
) -> SegmentResult:
started = time.perf_counter()
device, dtype = choose_device(allow_gpu)
self._call_device = device
+ extra: dict = {"instance_masks": True} if instance_masks else {}
+ if instance_masks and (plane_filter is not None or max_planes is not None):
+ extra.update(plane_filter = plane_filter, max_planes = max_planes)
oom = False
try:
- result = self._segment_on(image, concepts, threshold, path, device, dtype)
+ result = self._segment_on(image, concepts, threshold, path, device, dtype, **extra)
except Exception as exc:
if not _is_oom(exc) or device == "cpu":
self._after_call()
@@ -1357,7 +1675,7 @@ class MaskSegmenter:
device, dtype = "cpu", torch.float32
self._call_device = device
try:
- result = self._segment_on(image, concepts, threshold, path, device, dtype)
+ result = self._segment_on(image, concepts, threshold, path, device, dtype, **extra)
except Exception:
# The CPU copy and any release asked for mid-call are settled here too, not left resident.
self._after_call()
@@ -1374,6 +1692,9 @@ class MaskSegmenter:
path: str,
device: str,
dtype: Any,
+ instance_masks: bool = False,
+ plane_filter: Optional[Callable[[tuple[float, float, float, float]], bool]] = None,
+ max_planes: Optional[int] = None,
) -> SegmentResult:
import torch
@@ -1381,8 +1702,11 @@ class MaskSegmenter:
model, tokenizer = self._model, self._tokenizer
width, height = image.size
scale = torch.tensor([width, height, width, height], dtype = torch.float32)
- union = torch.zeros(height, width, dtype = torch.bool, device = device)
+ budgeted = instance_masks and (plane_filter is not None or max_planes is not None)
+ slots = 0
+ union = None if budgeted else torch.zeros(height, width, dtype = torch.bool, device = device)
instances: list[Instance] = []
+ logits_out: list = []
summary: list[dict] = []
with torch.inference_mode():
pixels = preprocess(image, device, dtype, model_image_size(model))
@@ -1403,7 +1727,7 @@ class MaskSegmenter:
caps[concept.text] = max(caps.get(concept.text, 0), concept.max_detections)
# Per text only its kept scores and boxes outlive the forward: the union of a text's best-``cap`` masks
# covers every occurrence's best-``k`` (k <= cap), so the masks are OR-ed in at once and dropped.
- kept: dict[str, tuple[Any, Any]] = {}
+ kept: dict[str, tuple[Any, Any, list]] = {}
for concept in concepts:
if concept.text not in kept:
index = texts.index(concept.text)
@@ -1422,25 +1746,38 @@ class MaskSegmenter:
presence_logit = None if presence is None else presence[0],
)
del out, presence
- union |= union_mask(masks, height, width)
- del masks
- kept[concept.text] = (scores, boxes)
- scores, boxes = (t[: concept.max_detections] for t in kept[concept.text])
+ if budgeted:
+ kept[concept.text] = (scores, boxes, (masks, {}))
+ else:
+ union |= union_mask(masks, height, width)
+ planes = [m.float().cpu() for m in masks] if instance_masks else []
+ del masks
+ kept[concept.text] = (scores, boxes, planes)
+ scores, boxes, planes = kept[concept.text]
+ scores, boxes = scores[: concept.max_detections], boxes[: concept.max_detections]
+ if not budgeted:
+ logits_out.extend(planes[: concept.max_detections])
boxes_px = (boxes.float().cpu() * scale).tolist()
- for score, box in zip(scores.tolist(), boxes_px):
+ for j, (score, box) in enumerate(zip(scores.tolist(), boxes_px)):
x1, y1, x2, y2 = (float(v) for v in box)
- instances.append(
- Instance(
- concept.text,
- float(score),
- (
- max(0.0, x1),
- max(0.0, y1),
- min(float(width), x2),
- min(float(height), y2),
- ),
- )
+ clipped = (
+ max(0.0, x1),
+ max(0.0, y1),
+ min(float(width), x2),
+ min(float(height), y2),
)
+ instances.append(Instance(concept.text, float(score), clipped))
+ if budgeted:
+ device_masks, moved = planes
+ plane = None
+ if (max_planes is None or slots < max_planes) and (
+ plane_filter is None or plane_filter(clipped)
+ ):
+ slots += 1
+ if j not in moved:
+ moved[j] = device_masks[j].float().cpu()
+ plane = moved[j]
+ logits_out.append(plane)
summary.append(
{
"text": concept.text,
@@ -1450,13 +1787,14 @@ class MaskSegmenter:
)
del kept, vision, pixels
return SegmentResult(
- mask_png = mask_to_png(union),
+ mask_png = b"" if union is None else mask_to_png(union),
width = width,
height = height,
instances = instances,
concepts = summary,
device = device,
precision = self.precision,
+ instance_logits = logits_out,
)
@@ -1496,11 +1834,16 @@ def release_segmenter(
*,
wait: bool = False,
handover: bool = False,
+ park: bool = False,
) -> bool:
- """Drop the weights if they were ever loaded. Never creates the segmenter. ``handover``: see
+ """Drop the weights if they were ever loaded. Never creates the segmenter. ``handover`` / ``park``: see
MaskSegmenter.release."""
segmenter = _segmenter
- return segmenter is not None and segmenter.release(reason, wait = wait, handover = handover)
+ if segmenter is None:
+ return False
+ if park:
+ return segmenter.release(reason, wait = wait, handover = handover, park = True)
+ return segmenter.release(reason, wait = wait, handover = handover)
_renders_pending = 0
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index 8327c7c3ba0..817edb9df74 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -103,6 +103,10 @@ def _params_text(meta: dict[str, Any]) -> str:
# Static flow shift, read back by the paste parser; base / max shift stay in the recipe chunk only.
if meta.get("shift") is not None:
fields += f", Shift: {meta['shift']}"
+ # ADetailer's own keys (adetailer/args.py), so a paste into Studio or a webui with ADetailer restores the pass.
+ if meta.get("detailer"):
+ from core.inference.diffusion_detailer import infotext_fields
+ fields += infotext_fields(meta["detailer"])
if meta.get("generator"):
fields += f", Version: {meta['generator']}"
lines.append(fields)
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 8a0563275af..df1172a1a01 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5001,6 +5001,70 @@ class LocalizedEditSpec(BaseModel):
_MAX_CONDITION_PAYLOAD = 128 * 1024 * 1024
+class DiffusionDetailerSpec(BaseModel):
+ """Detailer (face fix) post-pass, after ADetailer and ComfyUI-Impact-Pack's FaceDetailer: SAM 3 finds the targets,
+ each is inpainted in an enlarged crop and pasted back through a feathered mask. Defaults follow FaceDetailer
+ (core/inference/diffusion_detailer.py). Needs a model whose status `workflows` has 'inpaint' and a downloaded SAM 3."""
+
+ targets: str = Field(
+ "face",
+ min_length = 1,
+ max_length = 500,
+ description = "What to fix, as Mask from text takes it: comma-separated, `:N` caps a target "
+ "(default 8 each, at most 20), e.g. 'face' or 'face, hand:2'",
+ )
+ prompt: Optional[str] = Field(
+ None,
+ max_length = 20000,
+ description = "Prompt for the detail pass; omitted = the image's prompt. [PROMPT] inserts it, [SEP] gives "
+ "later detections their own text (ADetailer).",
+ )
+ negative_prompt: Optional[str] = Field(
+ None, max_length = 20000, description = "Negative prompt for the pass; omitted = the image's"
+ )
+ confidence: float = Field(
+ 0.5, ge = 0.0, le = 1.0, description = "SAM 3 score a detection must exceed"
+ )
+ denoise: float = Field(
+ 0.5,
+ ge = 0.01,
+ le = 1.0,
+ description = "Inpaint strength of the pass (the UI request builder's floor)",
+ )
+ crop_factor: float = Field(
+ 3.0, ge = 1.0, le = 10.0, description = "Crop around each detection, times its box"
+ )
+ guide_size: int = Field(
+ 512, ge = 64, le = 2048, description = "The box's (or crop's) short side is enlarged to this"
+ )
+ guide_size_for: Literal["bbox", "crop_region"] = Field(
+ "bbox", description = "What guide_size measures: the detection box or the whole crop"
+ )
+ max_size: int = Field(
+ 1024,
+ ge = 64,
+ le = 2048,
+ description = "Caps the enlargement: an enlarged crop's sides stay within this. A crop "
+ "already larger renders at its own size, never shrunk (FaceDetailer's force_inpaint)",
+ )
+ dilation: int = Field(
+ 10, ge = -128, le = 128, description = "Grow (or, negative, shrink) each mask by this many px"
+ )
+ feather: int = Field(5, ge = 0, le = 100, description = "Paste-back blur (px) at the mask's edge")
+ drop_size: int = Field(
+ 10,
+ ge = 1,
+ le = 1024,
+ description = "Detections with a box side at or below this (px) are skipped",
+ )
+ steps: Optional[int] = Field(
+ None, ge = 1, le = 200, description = "Steps for the pass; omitted = the image's"
+ )
+ guidance: Optional[float] = Field(
+ None, ge = 0.0, le = 100.0, description = "Guidance for the pass; omitted = the image's"
+ )
+
+
class DiffusionGenerateRequest(BaseModel):
"""Request to generate one image from the loaded diffusion model."""
@@ -5265,6 +5329,11 @@ class DiffusionGenerateRequest(BaseModel):
le = 256,
description = "Padding (px) around the mask when `inpaint_only_masked` is set; omitted = 32.",
)
+ detailer: Optional[DiffusionDetailerSpec] = Field(
+ None,
+ description = "Detailer (face fix) post-pass on every image of the run. Omitted = off. Refused (400) "
+ "when the loaded model has no inpaint pipeline or SAM 3 is not downloaded.",
+ )
hires: Optional[HiresSpec] = Field(
None,
description = "Hires fix for text to image: upscale the render and refine it in the same request. "
@@ -5525,6 +5594,11 @@ class GalleryImage(BaseModel):
inpaint_padding: Optional[int] = Field(
None, description = "Padding (px) around the mask for an only-masked inpaint"
)
+ detailer: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "The Detailer pass this image ran: its settings, each detection (target, score, box, crop, "
+ "size, seed) and timing, or `error` when it did not run; null = no pass",
+ )
created_at: float = Field(..., description = "Creation time (epoch seconds)")
# Library state, not recipe: stored beside the PNG, so older files simply read as unset.
pinned: bool = Field(False, description = "Pinned to the front of the gallery")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d29936e2c71..4eef1eeaff8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -52552,6 +52552,7 @@ async def _external_image_generation(
or body.inpaint_padding is not None
or body.hires is not None
or body.upscale_model
+ or body.detailer is not None
):
raise HTTPException(
status_code = 400,
@@ -52792,6 +52793,12 @@ async def _generate_diffusion_image_run(
backend = get_active_diffusion_engine()
if account_access.managed_account():
await asyncio.to_thread(account_access.require_media_adapters, request)
+ # Validate before the render so an unrunnable pass never costs one.
+ detailer = (
+ await asyncio.to_thread(_detailer_settings, backend, request.detailer)
+ if request.detailer is not None
+ else None
+ )
# An edit that names no size matches Image 1 in the backend instead of the schema's 1024 square default.
size_omitted = request.workflow == "edit" and not (
{"width", "height"} & request.model_fields_set
@@ -52924,6 +52931,19 @@ async def _generate_diffusion_image_run(
logger.error("diffusion.generate_failed: %s", exc, exc_info = True)
raise HTTPException(status_code = 500, detail = _generate_failure_detail(str(exc)))
+ global _diffusion_persist_active
+ if detailer is not None:
+ # The render's active marker is down: hold "active" through the persist so recovery never sees it finished.
+ _diffusion_persist_active += 1
+ try:
+ with account_access.media_generation("diffusion"):
+ await asyncio.to_thread(
+ _run_detailer, backend, request, result, enhancement, detailer
+ )
+ except BaseException:
+ _diffusion_persist_active -= 1
+ raise
+
# Persist each image with its full recipe. BOTH engines batch with a distinct seed per image, returned in ``seeds``, so each is individually reproducible.
created_at = time.time()
per_image_seeds = result.get("seeds")
@@ -53038,6 +53058,11 @@ async def _generate_diffusion_image_run(
)
if result.get(key) not in (None, [])
},
+ **(
+ {"detailer": result["detailer"][index]}
+ if result.get("detailer") and index < len(result["detailer"])
+ else {}
+ ),
"created_at": created_at,
},
)
@@ -53045,7 +53070,6 @@ async def _generate_diffusion_image_run(
return records
# Hold generate-progress "active" across the persist so a reload mount probe cannot refresh the gallery before these records exist.
- global _diffusion_persist_active
_diffusion_persist_active += 1
try:
with account_access.media_generation("diffusion"):
@@ -53054,7 +53078,7 @@ async def _generate_diffusion_image_run(
logger.error("diffusion.persist_failed: %s", exc)
raise HTTPException(status_code = 500, detail = "Failed to save the generated image.")
finally:
- _diffusion_persist_active -= 1
+ _diffusion_persist_active -= 2 if detailer is not None else 1
return DiffusionGenerateResponse(images = [GalleryImage(**r) for r in records])
@@ -54004,13 +54028,16 @@ async def cancel_diffusion_generation(
# Both: another tab's request may be enhancing while this one denoises.
upsampling = cancel_upsampling()
+ # A Detailer pass between two inpaints has no denoise for the engine to stop.
+ detailer = sys.modules.get("core.inference.diffusion_detailer")
# The slot can change hands before the executor callback, so the engine rechecks under the lock.
expected = account_access.tracked_generation_account()
+ detailing = bool(detailer is not None and detailer.cancel_active(expected))
cancel = get_active_diffusion_engine().cancel_generate
if expected is not None:
cancel = functools.partial(cancel, expected_account = expected)
cancelled = await asyncio.get_running_loop().run_in_executor(_CANCEL_EXECUTOR, cancel)
- return {"cancelled": bool(cancelled or upsampling)}
+ return {"cancelled": bool(cancelled or upsampling or detailing)}
# ── FLUX.3 prompt upsampler (llama.cpp sidecar) ──────────────────────────────────────────────────────────
@@ -54075,13 +54102,78 @@ def _cancel_upsampler_warm_start() -> None:
logger.warning("flux3.prompt_upsampler: could not cancel a warm start: %s", exc)
+def _detailer_settings(backend: Any, spec: Any) -> Any:
+ from core.inference import diffusion_detailer
+ try:
+ settings = diffusion_detailer.DetailerSettings.from_request(spec)
+ diffusion_detailer.check_available(backend.status())
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc))
+ return settings
+
+
+def _run_detailer(
+ backend: Any, request: Any, result: dict, enhancement: Any, settings: Any
+) -> None:
+ """Detailer pass on the same load, LoRAs and sampler; no live preview (it would show a crop)."""
+ from types import SimpleNamespace
+
+ from core.inference import diffusion_detailer
+ from core.inference.diffusion_families import load_identity
+
+ status = backend.status()
+ if result.get("repo_id") != (status.get("display_repo_id") or status.get("repo_id")):
+ result["detailer"] = [
+ {**diffusion_detailer.settings_record(settings), "error": "the model changed"}
+ for _ in result.get("images") or []
+ ]
+ return
+
+ def _prompt_for(index: int) -> str:
+ typed = (
+ request.prompts[index]
+ if request.prompts and index < len(request.prompts)
+ else request.prompt
+ )
+ return enhancement.encoded(typed)
+
+ sd_controls = SimpleNamespace(
+ prompt_weighting = getattr(request, "prompt_weighting", None),
+ clip_skip = getattr(request, "clip_skip", None),
+ cfg_rescale = getattr(request, "cfg_rescale", None),
+ )
+ diffusion_detailer.detail_result(
+ backend.generate,
+ result,
+ settings,
+ prompt_for = _prompt_for,
+ # The submitted one: the render's is None when the base CFG left it off, but the Detailer's CFG may use it.
+ negative_prompt = request.negative_prompt,
+ account = account_access.tracked_generation_account(),
+ steps = request.steps,
+ guidance = request.guidance,
+ generate_kwargs = {
+ "expected_load": load_identity(
+ status.get("repo_id"), status.get("base_repo"), status.get("family")
+ ),
+ "allow_oversized": request.allow_oversized and not account_access.managed_account(),
+ "loras": [(l.id, l.weight) for l in request.loras] if request.loras else None,
+ **_live_preview_kwarg(backend.generate, False),
+ **_sampler_kwargs(backend.generate, request.sampler, request.schedule_type),
+ **_sd_controls_kwargs(backend.generate, sd_controls),
+ **_shift_kwargs(backend.generate, request),
+ },
+ )
+
+
def _release_mask_segmenter_for_render() -> None:
"""Drop the text-to-mask segmenter's weights before a render: a resident image model renders without an arbiter
acquire, so no handover would unload it. Only if Images ever loaded it; a Find in flight finishes first."""
module = sys.modules.get("core.inference.diffusion_segment")
if module is not None:
try:
- module.release_segmenter("image generation", wait = True)
+ # Park, not drop: unparking takes well under a second.
+ module.release_segmenter("image generation", wait = True, park = True)
except Exception as exc: # noqa: BLE001 -- never fail a render on this
logger.warning("images.segment: could not release before a render: %s", exc)
diff --git a/studio/backend/tests/test_diffusion_detailer.py b/studio/backend/tests/test_diffusion_detailer.py
new file mode 100644
index 00000000000..ab14f30e39a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_detailer.py
@@ -0,0 +1,1381 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The Images Detailer (face fix) pass: FaceDetailer's crop / enlarge / feather / paste math, ADetailer's prompt and
+infotext rules, the per-detection seeds, the recipe record, and the gating that refuses a model with no inpaint path
+or a missing SAM 3. The inpaint render and SAM 3 are fakes here; the real ones run in the GPU end-to-end script."""
+
+from __future__ import annotations
+
+import types
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from core.inference import diffusion_detailer as det
+
+
+def test_defaults_follow_facedetailer():
+ s = det.DetailerSettings()
+ assert (s.targets, s.confidence, s.denoise, s.crop_factor) == ("face", 0.5, 0.5, 3.0)
+ assert (s.guide_size, s.guide_size_for, s.max_size) == (512, "bbox", 1024)
+ assert (s.dilation, s.feather, s.drop_size) == (10, 5, 10)
+ assert s.steps is None and s.guidance is None and s.prompt is None
+
+
+def test_request_defaults_match_the_module():
+ from models.inference import DiffusionDetailerSpec
+ spec = DiffusionDetailerSpec()
+ assert det.DetailerSettings.from_request(spec) == det.DetailerSettings()
+
+
+def test_crop_region_is_the_box_times_crop_factor_about_its_centre():
+ # 100x80 box at (400, 300): 300x240 crop centred on (450, 340).
+ assert det.crop_region(1024, 1024, (400, 300, 500, 380), 3.0) == (300, 220, 600, 460)
+
+
+def test_crop_region_is_shifted_back_inside_the_image():
+ # Near the top-left corner the crop keeps its size and starts at 0 (normalize_region).
+ assert det.crop_region(1024, 768, (10, 5, 110, 85), 3.0) == (0, 0, 300, 240)
+ assert det.crop_region(1024, 768, (950, 700, 1020, 760), 3.0) == (814, 588, 1024, 768)
+ assert det.crop_region(200, 100, (50, 20, 150, 80), 3.0) == (0, 0, 200, 100)
+
+
+def test_upscale_puts_the_box_short_side_at_guide_size():
+ # 100x80 box in a 300x240 crop: 512 / 80 = 6.4 -> 1920x1536, capped at 1024 on the long side.
+ upscale, w, h = det.upscaled_size(100, 80, 300, 240, 512, "bbox", 1024)
+ assert (w, h) == (1024, 819) and upscale == pytest.approx(6.4 * 1024 / 1920)
+ # A loose cap: the box's short side lands on guide_size.
+ upscale, w, h = det.upscaled_size(100, 80, 300, 240, 512, "bbox", 4096)
+ assert upscale == pytest.approx(6.4) and (w, h) == (1920, 1536)
+ upscale, w, h = det.upscaled_size(100, 80, 300, 240, 512, "crop_region", 4096)
+ assert upscale == pytest.approx(512 / 240) and h == 512
+
+
+def test_a_big_target_renders_at_its_own_size():
+ # force_inpaint: factor <= 1 never shrinks the crop.
+ assert det.upscaled_size(600, 600, 1000, 1000, 512, "bbox", 1024) == (1.0, 1000, 1000)
+
+
+def test_max_size_caps_the_enlargement_but_never_shrinks_a_large_crop():
+ """FaceDetailer parity (Impact-Pack core.enhance_detail, force_inpaint): when the capped factor is at or below 1,
+ the crop renders at its own size, so a 2048 px crop with a large face is not shrunk to max_size."""
+ assert det.upscaled_size(800, 800, 2048, 2048, 512, "bbox", 1024) == (1.0, 2048, 2048)
+
+
+def test_snap_size_floors_to_the_inpaint_grid():
+ assert det.snap_size(1024, 819) == (1024, 816)
+ assert det.snap_size(5, 40) == (16, 32)
+
+
+def _cv2():
+ return pytest.importorskip("cv2")
+
+
[email protected]("pixels", [10, 3, 4, -4, -7])
+def test_dilate_matches_opencv(pixels):
+ cv2 = _cv2()
+ rng = np.random.default_rng(0)
+ mask = rng.random((37, 53)) > 0.93
+ kernel = np.ones((abs(pixels), abs(pixels)), np.uint8)
+ op = cv2.dilate if pixels > 0 else cv2.erode
+ expected = op(mask.astype(np.uint8), kernel, iterations = 1) > 0
+ assert (det.dilate(mask, pixels) == expected).all()
+
+
+def test_dilate_by_zero_or_one_is_a_copy():
+ mask = np.zeros((5, 5), bool)
+ mask[2, 2] = True
+ for px in (0, 1, -1):
+ out = det.dilate(mask, px)
+ assert (out == mask).all() and out is not mask
+
+
[email protected]("feather,shape", [(5, (64, 80)), (3, (40, 33)), (5, (9, 30))])
+def test_feather_matches_torchvision_gaussian_blur(feather, shape):
+ torch = pytest.importorskip("torch")
+ tv = pytest.importorskip("torchvision")
+ mask = np.zeros(shape, np.float32)
+ mask[shape[0] // 4 : shape[0] * 3 // 4, shape[1] // 3 : shape[1] * 2 // 3] = 1.0
+ size = feather * 2 + 1
+ shortest = min(shape)
+ if shortest <= size:
+ size = int(shortest / 2)
+ size += size % 2 == 0
+ expected = (
+ tv.transforms.GaussianBlur(kernel_size = size, sigma = 10.0)(
+ torch.from_numpy(mask)[None, None]
+ )[0, 0]
+ .numpy()
+ .clip(0, 1)
+ )
+ assert np.abs(det.feather_mask(mask, feather) - expected).max() < 1e-5
+
+
+def test_feather_skips_a_tiny_crop_and_zero():
+ # A 3 px short side halves to a 1 px kernel, under 3: Impact-Pack skips the blur.
+ mask = np.zeros((3, 9), np.float32)
+ mask[1, 4] = 1.0
+ assert (det.feather_mask(mask, 5) == mask).all()
+ assert (det.feather_mask(np.ones((40, 40), np.float32), 0) == 1.0).all()
+
+
+def test_composite_keeps_zero_alpha_pixels_byte_for_byte():
+ rng = np.random.default_rng(1)
+ base = rng.integers(0, 256, (16, 16, 3), dtype = np.uint8)
+ patch = rng.integers(0, 256, (16, 16, 3), dtype = np.uint8)
+ alpha = np.zeros((16, 16), np.float32)
+ alpha[4:12, 4:12] = 1.0
+ alpha[3, 3:13] = 0.25
+ out = composite = det.composite(base, patch, alpha)
+ assert (composite[alpha == 0] == base[alpha == 0]).all()
+ assert (out[alpha == 1] == patch[alpha == 1]).all()
+ blended = np.rint(base[3, 5] * 0.75 + patch[3, 5] * 0.25)
+ assert (out[3, 5] == blended).all()
+
+
+def test_targets_take_the_detailer_count_unless_named():
+ assert det.target_concepts("face") == [("face", det.DEFAULT_PER_TARGET)]
+ assert det.target_concepts("face:2, (hand), eyes:99") == [
+ ("face", 2),
+ ("hand", det.DEFAULT_PER_TARGET),
+ ("eyes", det.MAX_PER_TARGET),
+ ]
+ assert det.segment_prompt(" face , hand:3 ") == f"face:{det.DEFAULT_PER_TARGET}, hand:3"
+ assert det.target_concepts(" , ") == []
+ with pytest.raises(ValueError):
+ det.DetailerSettings.from_request(types.SimpleNamespace(targets = " , "))
+
+
+def test_detection_prompt_follows_adetailer():
+ assert det.detection_prompt(None, "a man", 0) == "a man"
+ assert det.detection_prompt("", "a man", 3) == "a man"
+ assert det.detection_prompt("detailed face, [PROMPT]", "a man", 0) == "detailed face, a man"
+ template = "smiling [SEP] [SKIP] [SEP] [SEP] old face"
+ assert [det.detection_prompt(template, "p", i) for i in range(6)] == [
+ "smiling",
+ None,
+ "p",
+ "old face",
+ "old face",
+ "old face",
+ ]
+
+
+def test_detection_seed_is_seed_plus_index():
+ assert [det.detection_seed(42, i) for i in range(3)] == [42, 43, 44]
+ assert det.detection_seed(2**53 - 1, 1) == 0
+
+
+def _scene(width = 256, height = 192):
+ rng = np.random.default_rng(7)
+ return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype = np.uint8))
+
+
+def _face(
+ box,
+ width = 256,
+ height = 192,
+):
+ x1, y1, x2, y2 = box
+ mask = np.zeros((height, width), bool)
+ yy, xx = np.ogrid[:height, :width]
+ cx, cy, rx, ry = (x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1) / 2, (y2 - y1) / 2
+ mask[((xx - cx) / rx) ** 2 + ((yy - cy) / ry) ** 2 <= 1] = True
+ return det.Detection("face", 0.9, box, mask)
+
+
+class _FakeInpaint:
+ """Paints the masked area of the enlarged crop a colour derived from the seed, as a real inpaint would only change
+ the masked area; records every call."""
+
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, crop, mask, seed, prompt, negative):
+ self.calls.append(
+ {
+ "size": crop.size,
+ "mask_size": mask.size,
+ "seed": seed,
+ "prompt": prompt,
+ "negative": negative,
+ }
+ )
+ arr = np.array(crop)
+ m = np.array(mask) > 127
+ arr[m] = [(seed * 37) % 256, (seed * 91) % 256, (seed * 13) % 256]
+ return Image.fromarray(arr)
+
+
+def test_pass_changes_only_inside_the_feathered_masks():
+ image = _scene()
+ detections = [_face((40, 30, 80, 80)), _face((150, 60, 200, 120))]
+ settings = det.DetailerSettings()
+ fake = _FakeInpaint()
+ out = det.detail_image(
+ image,
+ detections,
+ settings,
+ base_seed = 100,
+ prompt = "two people",
+ negative_prompt = "blurry",
+ inpaint = fake,
+ )
+ before, after = np.array(image), np.array(out.image)
+ # Where each feathered paste mask can reach: the dilated mask, blurred, inside its crop.
+ reach = np.zeros(before.shape[:2], bool)
+ for d in detections:
+ x1, y1, x2, y2 = det.crop_region(256, 192, d.box, settings.crop_factor)
+ local = det.feather_mask(
+ det.dilate(d.mask[y1:y2, x1:x2], settings.dilation), settings.feather
+ )
+ reach[y1:y2, x1:x2] |= local > 0
+ changed = (before != after).any(axis = 2)
+ assert changed.any()
+ assert not changed[~reach].any()
+ # Sequential seeds, the image's prompts, sizes on the 16 px grid, the enlarged box short side near guide_size.
+ assert [c["seed"] for c in fake.calls] == [100, 101]
+ assert all(c["prompt"] == "two people" and c["negative"] == "blurry" for c in fake.calls)
+ for call in fake.calls:
+ w, h = call["size"]
+ assert w % 16 == 0 and h % 16 == 0 and call["mask_size"] == call["size"]
+ assert max(w, h) <= settings.max_size
+ record = out.record
+ assert [d["seed"] for d in record["detections"]] == [100, 101]
+ assert record["found"] == 2 and record["denoise"] == 0.5 and record["targets"] == "face"
+ assert record["detections"][0]["crop"] == list(det.crop_region(256, 192, (40, 30, 80, 80), 3.0))
+ assert "prompt" not in record["detections"][0]
+
+
+def test_pass_is_deterministic():
+ image = _scene()
+ detections = [_face((40, 30, 80, 80)), _face((150, 60, 200, 120))]
+ runs = [
+ det.detail_image(
+ image,
+ detections,
+ det.DetailerSettings(),
+ base_seed = 5,
+ prompt = "p",
+ negative_prompt = None,
+ inpaint = _FakeInpaint(),
+ )
+ for _ in range(2)
+ ]
+ assert np.array_equal(np.array(runs[0].image), np.array(runs[1].image))
+ assert runs[0].record["detections"] == runs[1].record["detections"]
+
+
+def test_later_detections_see_earlier_results():
+ """Overlapping crops: the second crop is read from the image the first paste left (Impact-Pack re-crops)."""
+ image = _scene()
+ seen = []
+
+ def _inpaint(crop, mask, seed, prompt, negative):
+ seen.append(np.array(crop).copy())
+ return _FakeInpaint()(crop, mask, seed, prompt, negative)
+
+ detections = [_face((60, 40, 100, 90)), _face((90, 50, 130, 100))]
+ det.detail_image(
+ image,
+ detections,
+ det.DetailerSettings(),
+ base_seed = 1,
+ prompt = "p",
+ negative_prompt = None,
+ inpaint = _inpaint,
+ )
+ first_colour = np.array([(1 * 37) % 256, (1 * 91) % 256, (1 * 13) % 256], np.uint8)
+ assert (seen[1].reshape(-1, 3) == first_colour).all(axis = 1).any()
+
+
+def test_small_skipped_and_empty_detections():
+ image = _scene()
+ tiny = _face((10, 10, 20, 40)) # 10 px wide: drop_size 10 drops it (the side must be > 10)
+ empty = det.Detection("face", 0.8, (100, 100, 150, 150), np.zeros((192, 256), bool))
+ keep = _face((150, 60, 200, 120))
+ fake = _FakeInpaint()
+ out = det.detail_image(
+ image,
+ [tiny, empty, keep],
+ det.DetailerSettings(prompt = "a [SEP] b"),
+ base_seed = 9,
+ prompt = "p",
+ negative_prompt = None,
+ inpaint = fake,
+ )
+ # Undersized boxes are dropped before indexing (Impact-Pack's detectors); an empty mask still takes its index.
+ assert [c["seed"] for c in fake.calls] == [10]
+ assert fake.calls[0]["prompt"] == "b"
+ assert out.record["skipped"] == 2 and out.record["detections"][0]["prompt"] == "b"
+
+
+def test_undersized_detections_take_no_prompt_or_seed():
+ fake = _FakeInpaint()
+ det.detail_image(
+ _scene(),
+ [_face((10, 10, 20, 40)), _face((150, 60, 200, 120))],
+ det.DetailerSettings(prompt = "a [SEP] b"),
+ base_seed = 9,
+ prompt = "p",
+ negative_prompt = None,
+ inpaint = fake,
+ )
+ assert [(c["seed"], c["prompt"]) for c in fake.calls] == [(9, "a")]
+
+
+def test_too_many_targets_are_refused_before_the_render():
+ from models.inference import DiffusionDetailerSpec
+
+ targets = ", ".join(f"thing{i}" for i in range(17))
+ with pytest.raises(ValueError, match = "At most 16"):
+ det.DetailerSettings.from_request(DiffusionDetailerSpec(targets = targets))
+ sixteen = ", ".join(f"thing{i}" for i in range(16))
+ assert (
+ det.DetailerSettings.from_request(DiffusionDetailerSpec(targets = sixteen)).targets == sixteen
+ )
+
+
+def test_an_inpaint_on_another_build_of_the_same_repo_is_discarded(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ image = _scene()
+
+ def _generate(**kwargs):
+ # Another tab reloaded the same repo with a different GGUF file while SAM 3 ran.
+ return {"images": [_scene(64, 64)], "repo_id": "org/m", "gguf_filename": "q8.gguf"}
+
+ result = {"images": [image], "seeds": [1], "repo_id": "org/m", "gguf_filename": "q4.gguf"}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert result["images"][0] is image
+ assert result["detailer"][0]["error"] == "the model changed"
+
+
+def test_same_build_compares_every_build_field():
+ base = {"repo_id": "r", "model_kind": "gguf", "gguf_filename": "a", "baked_loras": ["x"]}
+ assert det.same_build(base, {**base, "baked_loras": ("x",), "seed": 4})
+ assert det.same_build(base, {"images": []})
+ for key, value in [("gguf_filename", "b"), ("transformer_quant", "int8"), ("baked_loras", [])]:
+ assert not det.same_build(
+ {**base, "transformer_quant": None}, {**base, "transformer_quant": None, key: value}
+ )
+
+
+def test_detection_holds_the_engine_generation_slot(monkeypatch):
+ import threading
+
+ from core.inference import diffusion_engine_router as router
+ from core.inference import diffusion_segment
+
+ lock = threading.Lock()
+ monkeypatch.setattr(
+ router, "get_active_diffusion_engine", lambda: types.SimpleNamespace(_generate_lock = lock)
+ )
+ held = []
+
+ class _Seg:
+ def segment(self, *a, **k):
+ held.append(("segment", lock.locked()))
+ return types.SimpleNamespace(
+ instances = [], instance_logits = [], device = "cpu", precision = "fp32"
+ )
+
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: _Seg())
+ monkeypatch.setattr(
+ diffusion_segment,
+ "release_segmenter",
+ lambda reason, wait = False, park = False: held.append(("park", lock.locked())),
+ )
+ monkeypatch.setattr(
+ diffusion_segment, "active_variant", lambda: types.SimpleNamespace(id = "sam3.1")
+ )
+ det.detect([_scene()], det.DetailerSettings())
+ # A queued render cannot take the slot until SAM 3 is parked.
+ assert held == [("segment", True), ("park", True)] and not lock.locked()
+
+
+def test_detection_registers_as_the_generation_slot_holder(monkeypatch):
+ # Without a registered slot holder, generation_is_foreign() refuses the owner's cancel.
+ import threading
+
+ from core.inference import diffusion_engine_router as router
+ from core.inference import diffusion_segment
+ from hub.services.models import account_access
+ from utils.account_context import AccountContext, bind_account, reset_account
+
+ lock = threading.Lock()
+ monkeypatch.setattr(
+ router, "get_active_diffusion_engine", lambda: types.SimpleNamespace(_generate_lock = lock)
+ )
+ monkeypatch.setattr(account_access, "account_scope", lambda: "acct-a")
+ seen = []
+
+ class _Seg:
+ def segment(self, *a, **k):
+ seen.append(
+ (
+ list(account_access._generation_holders.get("diffusion", [])),
+ account_access.generation_is_foreign("diffusion"),
+ )
+ )
+ return types.SimpleNamespace(
+ instances = [], instance_logits = [], device = "cpu", precision = "fp32"
+ )
+
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: _Seg())
+ monkeypatch.setattr(diffusion_segment, "release_segmenter", lambda *a, **k: None)
+ monkeypatch.setattr(
+ diffusion_segment, "active_variant", lambda: types.SimpleNamespace(id = "sam3.1")
+ )
+ token = bind_account(AccountContext("acct-a", "a"))
+ try:
+ with account_access.media_generation("diffusion"):
+ # Account B is queued on the engine lock: counted, not holding.
+ other = bind_account(AccountContext("acct-b", "b"))
+ try:
+ with account_access.media_generation("diffusion"):
+ reset_account(other)
+ other = None
+ det.detect([_scene()], det.DetailerSettings())
+ finally:
+ if other is not None:
+ reset_account(other)
+ finally:
+ reset_account(token)
+ assert seen == [(["acct-a"], False)]
+ assert "diffusion" not in account_access._generation_holders
+
+
+def test_detection_waiting_for_the_slot_honours_stop(monkeypatch):
+ import threading
+
+ from core.inference import diffusion_engine_router as router
+ from core.inference import diffusion_segment
+
+ lock = threading.Lock()
+ lock.acquire() # a queued render denoises
+ monkeypatch.setattr(
+ router, "get_active_diffusion_engine", lambda: types.SimpleNamespace(_generate_lock = lock)
+ )
+ monkeypatch.setattr(
+ diffusion_segment, "get_segmenter", lambda: pytest.fail("SAM 3 loaded beside a render")
+ )
+ with pytest.raises(det.DetailerCancelled):
+ det.detect([_scene()], det.DetailerSettings(), stop = lambda: True)
+ lock.release()
+
+
+def test_detections_beyond_the_cap_are_dropped():
+ image = _scene()
+ detections = [_face((150, 60, 200, 120))] * (det.MAX_DETECTIONS + 3)
+ fake = _FakeInpaint()
+ out = det.detail_image(
+ image,
+ detections,
+ det.DetailerSettings(),
+ base_seed = 0,
+ prompt = "p",
+ negative_prompt = None,
+ inpaint = fake,
+ )
+ assert len(fake.calls) == det.MAX_DETECTIONS and out.record["dropped"] == 3
+
+
+def test_overrides_reach_the_record_and_the_negative_prompt():
+ settings = det.DetailerSettings(
+ negative_prompt = "[SKIP]x", steps = 12, guidance = 4.5, prompt = "sharp face"
+ )
+ fake = _FakeInpaint()
+ out = det.detail_image(
+ _scene(),
+ [_face((150, 60, 200, 120))],
+ settings,
+ base_seed = 0,
+ prompt = "p",
+ negative_prompt = "n",
+ inpaint = fake,
+ )
+ assert fake.calls[0]["negative"] == "[SKIP]x"
+ record = out.record
+ assert record["steps"] == 12 and record["guidance"] == 4.5 and record["prompt"] == "sharp face"
+
+
+def _patch_detect(monkeypatch, per_image):
+ monkeypatch.setattr(
+ det,
+ "detect",
+ lambda images, settings, stop = None: (
+ per_image[: len(images)],
+ {"model": "sam3.1", "device": "cpu", "detect_ms": 1.0},
+ ),
+ )
+
+
+def test_detail_result_runs_every_image_with_its_own_seed(monkeypatch):
+ _patch_detect(
+ monkeypatch,
+ [[_face((150, 60, 200, 120))], [_face((40, 30, 80, 80)), _face((150, 60, 200, 120))]],
+ )
+ calls = []
+
+ def _generate(**kwargs):
+ calls.append(kwargs)
+ crop = Image.open(
+ __import__("io").BytesIO(
+ __import__("base64").b64decode(kwargs["init_image"].split(",", 1)[1])
+ )
+ )
+ return {"images": [crop.convert("RGB")]}
+
+ result = {"images": [_scene(), _scene()], "seeds": [10, 20], "seed": 10}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(steps = 6),
+ prompt_for = lambda i: f"prompt {i}",
+ negative_prompt = None,
+ steps = 9,
+ guidance = 0.0,
+ generate_kwargs = {"expected_load": "L", "loras": None},
+ )
+ assert [c["seed"] for c in calls] == [10, 20, 21]
+ assert [c["prompt"] for c in calls] == ["prompt 0", "prompt 1", "prompt 1"]
+ assert all(
+ c["steps"] == 6 and c["strength"] == 0.5 and c["expected_load"] == "L" for c in calls
+ )
+ assert all(c["mask_image"].startswith("data:image/png;base64,") for c in calls)
+ assert [len(r["detections"]) for r in result["detailer"]] == [1, 2]
+ assert result["detailer"][0]["model"] == "sam3.1"
+
+
+def test_a_failed_pass_keeps_the_render(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ image = _scene()
+
+ def _generate(**kwargs):
+ raise RuntimeError("CUDA out of memory")
+
+ result = {"images": [image], "seeds": [1]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert result["images"][0] is image
+ assert result["detailer"][0]["error"] == "CUDA out of memory"
+
+
+def test_cancel_stops_the_pass(monkeypatch):
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))], [_face((150, 60, 200, 120))]])
+ calls = []
+
+ def _generate(**kwargs):
+ calls.append(1)
+ raise RuntimeError(DIFFUSION_CANCELLED_MSG)
+
+ result = {"images": [_scene(), _scene()], "seeds": [1, 2]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert len(calls) == 1
+ assert [r["error"] for r in result["detailer"]] == ["cancelled", "cancelled"]
+
+
+def test_detection_failure_is_recorded(monkeypatch):
+ def _boom(
+ images,
+ settings,
+ stop = None,
+ ):
+ raise RuntimeError("SAM 3 failed")
+
+ monkeypatch.setattr(det, "detect", _boom)
+ result = {"images": [_scene()], "seeds": [1]}
+ det.detail_result(
+ lambda **k: None,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert result["detailer"][0]["error"] == "SAM 3 failed"
+
+
+def test_detect_releases_sam3_even_on_failure(monkeypatch):
+ from core.inference import diffusion_segment
+
+ released = []
+
+ class _Seg:
+ def segment(self, *a, **k):
+ raise RuntimeError("boom")
+
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: _Seg())
+ monkeypatch.setattr(
+ diffusion_segment,
+ "release_segmenter",
+ lambda reason, wait = False, park = False: released.append((reason, wait, park)),
+ )
+ with pytest.raises(RuntimeError):
+ det.detect([_scene()], det.DetailerSettings())
+ assert released == [("detailer", True, True)]
+
+
+def test_detect_builds_full_size_instance_masks(monkeypatch):
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_segment
+
+ plane = torch.full((8, 8), -1.0)
+ plane[2:6, 2:6] = 1.0
+ seen = {}
+
+ class _Seg:
+ def segment(
+ self,
+ image,
+ prompt,
+ threshold,
+ instance_masks,
+ plane_filter = None,
+ max_planes = None,
+ ):
+ seen.update(prompt = prompt, threshold = threshold, instance_masks = instance_masks)
+ return types.SimpleNamespace(
+ instances = [diffusion_segment.Instance("face", 0.9, (64.0, 48.0, 192.0, 144.0))],
+ instance_logits = [plane],
+ device = "cpu",
+ precision = "bf16",
+ )
+
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: _Seg())
+ monkeypatch.setattr(
+ diffusion_segment, "release_segmenter", lambda reason, wait = False, park = False: True
+ )
+ found, info = det.detect(
+ [_scene()], det.DetailerSettings(targets = "face, hand:2", confidence = 0.3)
+ )
+ assert seen == {
+ "prompt": f"face:{det.DEFAULT_PER_TARGET}, hand:2",
+ "threshold": 0.3,
+ "instance_masks": True,
+ }
+ # Kept at SAM 3's resolution until the detection is processed, then expanded to the image's size.
+ assert found[0][0].mask is None
+ mask = found[0][0].full_mask(192, 256)
+ assert mask.shape == (192, 256) and mask.dtype == bool
+ assert mask[96, 128] and not mask[5, 5]
+ assert info["device"] == "cpu" and info["found"] == [1]
+
+
+def _many_detections_segmenter(count, calls):
+ import torch
+
+ from core.inference import diffusion_segment
+
+ class _Seg:
+ def segment(
+ self,
+ image,
+ prompt,
+ threshold,
+ instance_masks,
+ plane_filter = None,
+ max_planes = None,
+ ):
+ calls.append(image)
+ return types.SimpleNamespace(
+ instances = [
+ diffusion_segment.Instance("face", 0.9, (10.0 + i, 10.0, 60.0 + i, 70.0))
+ for i in range(count)
+ ],
+ instance_logits = [torch.zeros(8, 8) for _ in range(count)],
+ device = "cpu",
+ precision = "bf16",
+ )
+
+ return _Seg()
+
+
+def test_detection_keeps_only_the_pass_cap_and_no_full_size_masks(monkeypatch):
+ """16 targets x 20 detections is 320 masks; only the first MAX_DETECTIONS are kept, none expanded yet."""
+ pytest.importorskip("torch")
+ from core.inference import diffusion_segment
+
+ calls = []
+ monkeypatch.setattr(
+ diffusion_segment, "get_segmenter", lambda: _many_detections_segmenter(320, calls)
+ )
+ monkeypatch.setattr(
+ diffusion_segment, "release_segmenter", lambda reason, wait = False, park = False: True
+ )
+ found, info = det.detect([_scene(), _scene()], det.DetailerSettings())
+ assert [len(f) for f in found] == [det.MAX_DETECTIONS] * 2
+ assert info["found"] == [320, 320]
+ assert all(d.mask is None and d.plane is not None for f in found for d in f)
+
+
+def test_detail_result_records_what_detection_dropped(monkeypatch):
+ found = [[_face((150, 60, 200, 120))]]
+ monkeypatch.setattr(
+ det,
+ "detect",
+ lambda images, settings, stop = None: (found, {"model": "sam3.1", "found": [40]}),
+ )
+ result = {"images": [_scene()], "seeds": [1]}
+ det.detail_result(
+ lambda **k: {"images": [Image.new("RGB", (k["width"], k["height"]))]},
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ record = result["detailer"][0]
+ assert record["found"] == 40 and record["dropped"] == 39
+
+
+def test_stop_during_detection_skips_the_remaining_images(monkeypatch):
+ """Stop pressed while SAM 3 runs on a batch: no further image is segmented and no inpaint runs."""
+ pytest.importorskip("torch")
+ from core.inference import diffusion_segment
+
+ calls, released = [], []
+ segmenter = _many_detections_segmenter(1, calls)
+ real_segment = segmenter.segment
+
+ def _segment(*a, **k):
+ out = real_segment(*a, **k)
+ assert det.cancel_active() is True # Stop arrives during the first image's detection
+ return out
+
+ segmenter.segment = _segment
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: segmenter)
+ monkeypatch.setattr(
+ diffusion_segment,
+ "release_segmenter",
+ lambda reason, wait = False, park = False: released.append(park),
+ )
+ inpaints = []
+ result = {"images": [_scene() for _ in range(4)], "seeds": [1, 2, 3, 4]}
+ det.detail_result(
+ lambda **k: inpaints.append(1),
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert len(calls) == 1 and inpaints == [] and released == [True]
+ assert [r["error"] for r in result["detailer"]] == ["cancelled"] * 4
+
+
+def test_infotext_uses_adetailer_keys():
+ record = {
+ **det.settings_record(det.DetailerSettings(prompt = "detailed face, smile", steps = 12)),
+ "model": "sam3.1",
+ }
+ text = det.infotext_fields(record)
+ assert text == (
+ ", ADetailer model: sam3.1, ADetailer model classes: face"
+ ', ADetailer prompt: "detailed face, smile"'
+ ", ADetailer confidence: 0.5, ADetailer dilate erode: 10, ADetailer mask blur: 5"
+ ", ADetailer denoising strength: 0.5, ADetailer inpaint only masked: True"
+ ", ADetailer use separate steps: True, ADetailer steps: 12"
+ )
+ assert det.infotext_fields(None) == ""
+ assert det.infotext_fields({**record, "error": "cancelled"}) == ""
+
+
+def test_gallery_parameters_carry_the_detailer_only_when_it_ran():
+ from core.inference import image_gallery
+
+ meta = {
+ "prompt": "p",
+ "steps": 9,
+ "guidance": 0.0,
+ "seed": 1,
+ "width": 512,
+ "height": 512,
+ "model": "m",
+ }
+ plain = image_gallery._params_text(meta)
+ assert "ADetailer" not in plain
+ record = {**det.settings_record(det.DetailerSettings()), "model": "sam3.1", "detections": []}
+ text = image_gallery._params_text({**meta, "detailer": record})
+ assert text.startswith(plain.split(", Version")[0])
+ assert "ADetailer model: sam3.1" in text and "ADetailer denoising strength: 0.5" in text
+
+
+def test_gallery_image_accepts_the_detailer_record():
+ from models.inference import GalleryImage
+
+ record = {**det.settings_record(det.DetailerSettings()), "detections": [{"seed": 1}]}
+ image = GalleryImage(
+ id = "x",
+ url = "/u",
+ prompt = "p",
+ width = 64,
+ height = 64,
+ steps = 1,
+ guidance = 0.0,
+ seed = 1,
+ created_at = 0.0,
+ detailer = record,
+ )
+ assert image.detailer["targets"] == "face"
+
+
+class _Backend:
+ def __init__(self, workflows):
+ self._status = {"loaded": True, "workflows": workflows, "repo_id": "r"}
+
+ def status(self):
+ return self._status
+
+
+def _gate(
+ monkeypatch,
+ workflows,
+ *,
+ sam3 = True,
+ downloaded = True,
+):
+ from fastapi import HTTPException
+
+ from core.inference import diffusion_segment
+ from models.inference import DiffusionDetailerSpec
+ from routes import inference as routes
+
+ monkeypatch.setattr(diffusion_segment, "transformers_supports_sam3", lambda: sam3)
+ monkeypatch.setattr(diffusion_segment, "model_dir", lambda: "/sam" if downloaded else None)
+ try:
+ return routes._detailer_settings(_Backend(workflows), DiffusionDetailerSpec()), None
+ except HTTPException as exc:
+ return None, exc
+
+
+def test_gate_admits_an_inpaint_model_with_sam3(monkeypatch):
+ settings, error = _gate(monkeypatch, ["txt2img", "inpaint"])
+ assert error is None and settings == det.DetailerSettings()
+
+
+ "workflows,sam3,downloaded,needle",
+ [
+ (["txt2img", "img2img"], True, True, "inpaint pipeline"),
+ (["edit"], True, True, "inpaint pipeline"),
+ (["txt2img", "inpaint"], False, True, "transformers 5"),
+ (["txt2img", "inpaint"], True, False, "not downloaded"),
+ ],
+)
+def test_gate_refuses_with_400(monkeypatch, workflows, sam3, downloaded, needle):
+ _, error = _gate(monkeypatch, workflows, sam3 = sam3, downloaded = downloaded)
+ assert error is not None and error.status_code == 400 and needle in error.detail
+
+
+def test_request_without_detailer_is_unchanged():
+ from models.inference import DiffusionGenerateRequest
+
+ request = DiffusionGenerateRequest(prompt = "p")
+ assert request.detailer is None
+ assert "detailer" not in request.model_dump(exclude_none = True)
+
+
+def test_stop_between_two_inpaints_ends_the_pass(monkeypatch):
+ """Stop pressed while no denoise runs (between two face inpaints): the next detection is not inpainted."""
+ _patch_detect(
+ monkeypatch,
+ [[_face((40, 30, 80, 80)), _face((150, 60, 200, 120))], [_face((150, 60, 200, 120))]],
+ )
+ calls = []
+
+ def _generate(**kwargs):
+ import base64
+ import io
+
+ calls.append(kwargs["seed"])
+ # The first inpaint finishes, then the user presses Stop before the second starts.
+ assert det.cancel_active() is True
+ crop = Image.open(io.BytesIO(base64.b64decode(kwargs["init_image"].split(",", 1)[1])))
+ return {"images": [crop.convert("RGB")]}
+
+ first, second = _scene(), _scene()
+ result = {"images": [first, second], "seeds": [1, 2]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert calls == [1]
+ assert [r["error"] for r in result["detailer"]] == ["cancelled", "cancelled"]
+ assert result["images"][0] is first and result["images"][1] is second
+ assert det.cancel_active() is False
+
+
+def test_cancel_route_stops_a_detailer_pass(monkeypatch):
+ import asyncio
+ import sys
+
+ import core.inference.diffusion_engine_router as router
+ import core.inference.flux3_prompt_upsampler as pup
+ from routes import inference as routes
+
+ stopped = []
+ fake = types.SimpleNamespace(cancel_active = lambda account = None: stopped.append(1) or True)
+ monkeypatch.setitem(sys.modules, "core.inference.diffusion_detailer", fake)
+ monkeypatch.setattr(routes.account_access, "generation_is_foreign", lambda kind: False)
+ monkeypatch.setattr(routes.account_access, "generation_is_mine", lambda kind: True)
+ monkeypatch.setattr(routes.account_access, "tracked_generation_account", lambda: None)
+ monkeypatch.setattr(pup, "cancel_active", lambda: False)
+ monkeypatch.setattr(
+ router,
+ "get_active_diffusion_engine",
+ lambda: types.SimpleNamespace(cancel_generate = lambda: False),
+ )
+ out = asyncio.run(routes.cancel_diffusion_generation(target = None, current_subject = "u"))
+ assert out == {"cancelled": True} and stopped == [1]
+
+
+def test_route_runs_the_pass_on_a_display_id_load_and_stays_in_flight(monkeypatch, tmp_path):
+ """A load shown under a display id (a local snapshot under its Hub id) is the same model, and generate-progress
+ / liveness stay active from the end of the render through the Detailer and the persist."""
+ import asyncio
+
+ from core.inference import diffusion_engine_router as router
+ from core.inference import image_gallery
+ from models.inference import DiffusionDetailerSpec, DiffusionGenerateRequest
+ from routes import inference as routes
+
+ class _Backend:
+ def status(self):
+ return {
+ "loaded": True,
+ "workflows": ["txt2img", "inpaint"],
+ "repo_id": "/models/snap",
+ "display_repo_id": "org/model",
+ "family": "z-image",
+ "base_repo": None,
+ }
+
+ def generate(self, **kwargs):
+ return {"images": [_scene()], "seed": 3, "seeds": [3], "repo_id": "org/model"}
+
+ seen = {}
+
+ def _detail_result(generate, result, settings, **kwargs):
+ seen["in_flight"] = routes.generation_in_flight()
+ result["detailer"] = [{"targets": "face", "detections": []}]
+
+ monkeypatch.setattr(router, "get_active_diffusion_engine", lambda: _Backend())
+ monkeypatch.setattr(det, "check_available", lambda status: None)
+ monkeypatch.setattr(det, "detail_result", _detail_result)
+ monkeypatch.setattr(routes, "_release_mask_segmenter_for_render", lambda: None)
+ monkeypatch.setattr(
+ image_gallery,
+ "save",
+ lambda image, meta: {**meta, "id": "x", "url": "/u", "created_at": 0.0},
+ )
+ request = DiffusionGenerateRequest(
+ prompt = "p", steps = 4, guidance = 0.0, seed = 3, detailer = DiffusionDetailerSpec()
+ )
+ response = asyncio.run(routes._generate_diffusion_image_core(request))
+ assert seen == {"in_flight": True}
+ assert response.images[0].detailer == {"targets": "face", "detections": []}
+ assert routes.generation_in_flight() is False
+
+
+def _crop_back(kwargs):
+ import base64
+ import io
+ return Image.open(io.BytesIO(base64.b64decode(kwargs["init_image"].split(",", 1)[1]))).convert(
+ "RGB"
+ )
+
+
+def test_cancel_stops_only_the_callers_pass(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((40, 30, 80, 80)), _face((150, 60, 200, 120))]])
+ calls = []
+
+ def _generate(**kwargs):
+ calls.append(kwargs["seed"])
+ # Another account presses Stop: this account's pass carries on.
+ assert det.cancel_active("someone-else") is False
+ return {"images": [_crop_back(kwargs)]}
+
+ result = {"images": [_scene()], "seeds": [1]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ account = "me",
+ )
+ assert calls == [1, 2] and "error" not in result["detailer"][0]
+
+ calls.clear()
+
+ def _generate_own(**kwargs):
+ calls.append(kwargs["seed"])
+ assert det.cancel_active("me") is True
+ return {"images": [_crop_back(kwargs)]}
+
+ result = {"images": [_scene()], "seeds": [1]}
+ det.detail_result(
+ _generate_own,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ account = "me",
+ )
+ assert calls == [1] and result["detailer"][0]["error"] == "cancelled"
+
+
+def test_stop_after_the_last_inpaint_returns_keeps_the_render(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ render = _scene()
+
+ def _generate(**kwargs):
+ out = {"images": [_crop_back(kwargs).point(lambda v: 255 - v)]}
+ assert det.cancel_active() is True # Stop lands while the patch is still being composited
+ return out
+
+ result = {"images": [render], "seeds": [1]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert result["images"][0] is render
+ assert result["detailer"][0]["error"] == "cancelled"
+ assert det.cancel_active() is False
+
+
+def test_undersized_detections_take_no_cap_slot(monkeypatch):
+ torch = pytest.importorskip("torch")
+ from core.inference import diffusion_segment
+
+ tiny = [
+ diffusion_segment.Instance("eye", 0.9, (1.0, 1.0, 5.0, 5.0))
+ for _ in range(det.MAX_DETECTIONS)
+ ]
+ big = [diffusion_segment.Instance("face", 0.8, (10.0, 10.0, 60.0, 70.0)) for _ in range(20)]
+
+ class _Seg:
+ def segment(
+ self,
+ image,
+ prompt,
+ threshold,
+ instance_masks,
+ plane_filter = None,
+ max_planes = None,
+ ):
+ return types.SimpleNamespace(
+ instances = tiny + big,
+ instance_logits = [torch.zeros(8, 8) for _ in tiny + big],
+ device = "cpu",
+ precision = "bf16",
+ )
+
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: _Seg())
+ monkeypatch.setattr(
+ diffusion_segment, "release_segmenter", lambda reason, wait = False, park = False: True
+ )
+ found, info = det.detect([_scene()], det.DetailerSettings())
+ assert [d.concept for d in found[0]] == ["face"] * 20
+ assert info["found"] == [det.MAX_DETECTIONS + 20] and info["undersized"] == [det.MAX_DETECTIONS]
+
+
+def test_detect_keeps_sam_loaded_across_the_batch(monkeypatch):
+ from core.inference import diffusion_segment
+
+ events = []
+ seg = diffusion_segment.MaskSegmenter(
+ keep_alive_seconds = 0
+ ) # keep-alive off: every call would unload
+ seg.device = "cpu"
+ monkeypatch.setattr(seg, "_unload_locked", lambda reason: events.append(("unload", reason)))
+
+ def _segment(
+ image,
+ prompt,
+ threshold,
+ instance_masks,
+ plane_filter = None,
+ max_planes = None,
+ ):
+ seg._after_call() # what every real call ends with
+ events.append(("segment", None))
+ return types.SimpleNamespace(
+ instances = [], instance_logits = [], device = "cpu", precision = "bf16"
+ )
+
+ seg.segment = _segment
+ monkeypatch.setattr(diffusion_segment, "get_segmenter", lambda: seg)
+ monkeypatch.setattr(
+ diffusion_segment,
+ "release_segmenter",
+ lambda reason, wait = False, park = False: events.append(("park", reason)),
+ )
+ det.detect([_scene(), _scene(), _scene()], det.DetailerSettings())
+ assert events == [("segment", None)] * 3 + [("park", "detailer")]
+ seg._after_call()
+ assert events[-1] == ("unload", "no keep-alive")
+
+
+def test_hold_still_honours_a_release_asked_mid_call(monkeypatch):
+ from core.inference import diffusion_segment
+
+ seg = diffusion_segment.MaskSegmenter()
+ unloads = []
+ monkeypatch.setattr(seg, "_unload_locked", lambda reason: unloads.append(reason))
+ with seg.hold():
+ seg._release_pending = True
+ seg._after_call()
+ assert unloads == ["released during a call"]
+
+
+def test_inherited_negative_prompt_is_recorded_when_the_detailer_cfg_engages_it(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ seen = []
+
+ def _generate(**kwargs):
+ seen.append(kwargs["negative_prompt"])
+ return {"images": [_crop_back(kwargs)], "negative_prompt": kwargs["negative_prompt"]}
+
+ # The base render's CFG was too low for the negative prompt, so its result records none.
+ result = {"images": [_scene()], "seeds": [1], "negative_prompt": None}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(guidance = 5.0),
+ prompt_for = lambda i: "p",
+ negative_prompt = "blurry",
+ steps = 4,
+ guidance = 1.0,
+ generate_kwargs = {},
+ )
+ assert seen == ["blurry"] and result["detailer"][0]["negative_prompt"] == "blurry"
+
+
+def test_route_passes_sd_controls_submitted_negative_and_account(monkeypatch):
+ from core.inference.diffusion_sd_controls import SdControls
+ from models.inference import DiffusionGenerateRequest
+ from routes import inference as routes
+
+ captured = {}
+
+ class _Backend:
+ def status(self):
+ return {"loaded": True, "repo_id": "org/m", "family": "sdxl", "base_repo": None}
+
+ def generate(
+ self,
+ sd_controls = None,
+ **kwargs,
+ ):
+ raise AssertionError("not called")
+
+ def _detail_result(generate, result, settings, **kwargs):
+ captured.update(kwargs)
+
+ monkeypatch.setattr(det, "detail_result", _detail_result)
+ monkeypatch.setattr(routes.account_access, "tracked_generation_account", lambda: "acct-1")
+ request = DiffusionGenerateRequest(
+ prompt = "p",
+ negative_prompt = "blurry",
+ steps = 4,
+ guidance = 1.0,
+ prompt_weighting = False,
+ clip_skip = 2,
+ cfg_rescale = 0.7,
+ variation_seed = 5,
+ variation_strength = 0.3,
+ )
+ result = {"images": [_scene()], "repo_id": "org/m", "negative_prompt": None}
+ enhancement = types.SimpleNamespace(encoded = lambda text: text)
+ routes._run_detailer(_Backend(), request, result, enhancement, det.DetailerSettings())
+ controls = captured["generate_kwargs"]["sd_controls"]
+ assert isinstance(controls, SdControls)
+ assert (controls.prompt_weighting, controls.clip_skip, controls.cfg_rescale) == (False, 2, 0.7)
+ assert controls.variation_seed is None and not controls.variation_strength
+ assert captured["negative_prompt"] == "blurry" and captured["account"] == "acct-1"
+
+
+def test_a_low_denoise_still_schedules_one_inpaint_timestep(monkeypatch):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ calls = []
+
+ def _generate(**kwargs):
+ calls.append(kwargs)
+ # SDXL inpaint keeps int(steps * strength) timesteps and refuses zero.
+ if int(kwargs["steps"] * kwargs["strength"]) < 1:
+ raise ValueError("num_inference_steps < 1")
+ return {"images": [_crop_back(kwargs)]}
+
+ result = {"images": [_scene()], "seeds": [1]}
+ det.detail_result(
+ _generate,
+ result,
+ det.DetailerSettings(denoise = 0.25),
+ prompt_for = lambda i: "p",
+ negative_prompt = None,
+ steps = 3, # SDXL Turbo's default
+ guidance = 0.0,
+ generate_kwargs = {},
+ )
+ assert not result["detailer"][0].get("error")
+ assert calls[0]["steps"] == 4
+ assert det.schedulable_steps(1, 0.5) == 2
+ assert det.schedulable_steps(9, 0.5) == 9
+ assert det.schedulable_steps(3, 1.0) == 3
+
+
+def _schedulable_steps_reference(steps, denoise):
+ """The original count-up search, kept as the oracle for the closed form."""
+ n = max(1, int(steps))
+ if denoise <= 0.0 or denoise >= 1.0:
+ return n
+ while int(n * denoise) < 1:
+ n += 1
+ return n
+
+
+def test_schedulable_steps_closed_form_matches_the_search_over_a_sweep():
+ denoises = [i / 1000 for i in range(10, 1001)] + [
+ 0.01,
+ 0.05,
+ 1 / 3,
+ 2 / 3,
+ 0.1 + 0.2,
+ 0.999999,
+ 1.0,
+ ]
+ for steps in (0, 1, 2, 3, 4, 7, 20, 50, 99, 100, 101, 150):
+ for d in denoises:
+ assert det.schedulable_steps(steps, d) == _schedulable_steps_reference(steps, d), (
+ steps,
+ d,
+ )
+
+
+def test_schedulable_steps_is_bounded_for_a_tiny_denoise():
+ # The count-up search would loop 10M times here; the closed form must not depend on 1 / denoise.
+ import sys
+
+ lines = [0]
+
+ def tracer(frame, event, arg):
+ if frame.f_code is det.schedulable_steps.__code__:
+ if event == "line":
+ lines[0] += 1
+ if lines[0] > 200:
+ raise AssertionError("schedulable_steps is not bounded")
+ return tracer
+ return None
+
+ sys.settrace(tracer)
+ try:
+ result = det.schedulable_steps(1, 1e-7)
+ finally:
+ sys.settrace(None)
+ assert result == 10_000_000
+ assert det.schedulable_steps(1, 1e-300) >= 1
+ assert det.schedulable_steps(5, float("nan")) == 5
+
+
[email protected]("denoise", [1e-7, 0.001, 0.0099, 0.0, -0.5])
+def test_request_rejects_a_denoise_below_the_ui_floor(denoise):
+ import pydantic
+
+ from models.inference import DiffusionDetailerSpec
+
+ with pytest.raises(pydantic.ValidationError):
+ DiffusionDetailerSpec(denoise = denoise)
+ assert DiffusionDetailerSpec(denoise = 0.01).denoise == 0.01
+
+
+ "template, expected_seen, expected_saved",
+ [
+ ("[PROMPT], bad teeth", ["blurry, bad teeth"], "blurry, bad teeth"),
+ (" [SEP] bad teeth", ["blurry"], "blurry [SEP] bad teeth"),
+ ("bad teeth", ["bad teeth"], "bad teeth"),
+ ],
+)
+def test_a_template_inheriting_the_negative_saves_it_resolved(
+ monkeypatch, template, expected_seen, expected_saved
+):
+ _patch_detect(monkeypatch, [[_face((150, 60, 200, 120))]])
+ seen = []
+
+ def _generate(**kwargs):
+ seen.append(kwargs["negative_prompt"])
+ return {"images": [_crop_back(kwargs)], "negative_prompt": kwargs["negative_prompt"]}
+
+ result = {"images": [_scene()], "seeds": [1], "negative_prompt": None}
+ settings = det.DetailerSettings(guidance = 5.0, negative_prompt = template)
+ det.detail_result(
+ _generate,
+ result,
+ settings,
+ prompt_for = lambda i: "p",
+ negative_prompt = "blurry",
+ steps = 4,
+ guidance = 0.0,
+ generate_kwargs = {},
+ )
+ assert seen == expected_seen
+ saved = result["detailer"][0]["negative_prompt"]
+ assert saved == expected_saved
+ # Restored with no main negative (the base recorded none), the saved template expands the same.
+ assert det.detection_prompt(saved, "", 0) == expected_seen[0]
diff --git a/studio/backend/tests/test_diffusion_inpaint_mask.py b/studio/backend/tests/test_diffusion_inpaint_mask.py
index 7966e32106c..c609f6e5042 100644
--- a/studio/backend/tests/test_diffusion_inpaint_mask.py
+++ b/studio/backend/tests/test_diffusion_inpaint_mask.py
@@ -276,7 +276,13 @@ def test_status_flag_false_when_nothing_loaded():
@pytest.mark.parametrize(
- "field", [{"mask_blur": 4}, {"inpaint_only_masked": True}, {"inpaint_padding": 48}]
+ "field",
+ [
+ {"mask_blur": 4},
+ {"inpaint_only_masked": True},
+ {"inpaint_padding": 48},
+ {"detailer": {"targets": "face"}},
+ ],
)
def test_external_connection_refuses_inpaint_mask_fields(client, monkeypatch, field):
import routes.inference as inference_routes
diff --git a/studio/backend/tests/test_diffusion_segment.py b/studio/backend/tests/test_diffusion_segment.py
index 6da2f56252f..af39b63406f 100644
--- a/studio/backend/tests/test_diffusion_segment.py
+++ b/studio/backend/tests/test_diffusion_segment.py
@@ -332,6 +332,242 @@ def test_tiny_model_segments_at_source_size(tiny_sam3):
assert torch.is_grad_enabled()
+def test_instance_masks_are_the_union_split_per_detection(tiny_sam3):
+ """The Detailer's per-detection masks: one plane per instance in ``instances`` order, whose OR at the source
+ size is exactly the union mask; a plain Find keeps none."""
+ import numpy as np
+ from PIL import Image
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ image = Image.new("RGB", (64, 40), (120, 80, 40))
+ plain = segmenter.segment(
+ image, "sofa, pillow:3", threshold = 0.0, allow_gpu = False, path = tiny_sam3
+ )
+ assert plain.instance_logits == []
+ result = segmenter.segment(
+ image,
+ "sofa, pillow:3",
+ threshold = 0.0,
+ allow_gpu = False,
+ path = tiny_sam3,
+ instance_masks = True,
+ )
+ assert len(result.instance_logits) == len(result.instances) == 4
+ assert all(p.device.type == "cpu" and p.ndim == 2 for p in result.instance_logits)
+ import torch
+
+ union = seg.union_mask(torch.stack(result.instance_logits), 40, 64).numpy()
+ png = np.array(Image.open(io.BytesIO(result.mask_png))) > 0
+ assert (union == png).all()
+ assert "instance_logits" not in result.to_json()
+
+
+def test_detailer_budget_moves_only_the_planes_it_keeps(tiny_sam3, monkeypatch):
+ """The Detailer's call (``plane_filter`` + ``max_planes``): the same instances as a plain instance-mask call,
+ bit-identical planes for exactly the detections it keeps (filter first, then the cap, in ``instances`` order),
+ no union mask, and at most ``max_planes`` planes moved off the device."""
+ import torch
+ from PIL import Image
+
+ from core.inference import diffusion_detailer as det
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ image = Image.new("RGB", (64, 40), (120, 80, 40))
+ # A repeat with a larger cap after a smaller one: its extra planes are only picked on the second occurrence.
+ prompt = "pillow:1, sofa:2, pillow:4"
+ full = segmenter.segment(
+ image, prompt, threshold = 0.0, allow_gpu = False, path = tiny_sam3, instance_masks = True
+ )
+ assert len(full.instances) == 7
+ dropped = {full.instances[1].box, full.instances[4].box}
+
+ def keep(box):
+ return box not in dropped
+
+ # What the Detailer kept before: every plane moved, then filtered, then capped.
+ cap = 3
+ before = [p for p in zip(full.instances, full.instance_logits) if keep(p[0].box)][:cap]
+
+ def _no_union(*a, **k):
+ raise AssertionError("the Detailer's call built the union mask")
+
+ monkeypatch.setattr(seg, "union_mask", _no_union)
+ monkeypatch.setattr(seg, "mask_to_png", _no_union)
+ plane_shape = full.instance_logits[0].shape
+ real_float = torch.Tensor.float
+ moved = []
+
+ def _counting_float(self, *a, **k):
+ out = real_float(self, *a, **k)
+ if self.shape == plane_shape: # a mask plane, not the (N, 4) boxes
+ moved.append(out)
+ return out
+
+ monkeypatch.setattr(torch.Tensor, "float", _counting_float)
+ budgeted = segmenter.segment(
+ image,
+ prompt,
+ threshold = 0.0,
+ allow_gpu = False,
+ path = tiny_sam3,
+ instance_masks = True,
+ plane_filter = keep,
+ max_planes = cap,
+ )
+ monkeypatch.setattr(torch.Tensor, "float", real_float)
+ assert budgeted.mask_png == b""
+ assert budgeted.instances == full.instances and budgeted.concepts == full.concepts
+ assert len(budgeted.instance_logits) == len(budgeted.instances)
+ after = [p for p in zip(budgeted.instances, budgeted.instance_logits) if p[1] is not None]
+ assert len(after) == cap and len(moved) <= cap
+ assert [i for i, _ in after] == [i for i, _ in before]
+ assert all(torch.equal(a, b) for (_, a), (_, b) in zip(after, before))
+ sized = [p for p in zip(budgeted.instances, budgeted.instance_logits) if keep(p[0].box)]
+ assert all(plane is not None for _, plane in sized[:cap])
+ assert det.MAX_DETECTIONS >= cap
+
+
+def _small_model():
+ """Stands in for a loaded Sam3Model: real parameters and a buffer, one of them tied (shared storage)."""
+ import torch
+
+ torch.manual_seed(0)
+ model = torch.nn.Sequential(torch.nn.Linear(8, 8), torch.nn.LayerNorm(8))
+ model.register_buffer("table", torch.arange(16.0))
+ model.register_buffer("tied", model.table)
+ return model
+
+
+def _gpu_loaded(
+ monkeypatch,
+ segmenter,
+ model,
+ path = "/sam3",
+):
+ """``segmenter`` as if ``model`` were loaded on cuda:0 (its tensors stay on the CPU), with the weights plan, the
+ cache release and the device sync faked, and the host buffer pageable (no CUDA on the test host)."""
+ import torch
+
+ plan = seg.WeightsPlan("int8", path, "/sam3/w.safetensors", None, "test")
+ monkeypatch.setattr(seg, "plan_weights", lambda *a, **k: plan)
+ monkeypatch.setattr(seg, "_free_device_cache", lambda device: None)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ monkeypatch.setenv(seg.ENV_PARK_PIN, "0")
+ segmenter._model, segmenter._tokenizer = model, "tok"
+ segmenter._path, segmenter.device, segmenter._dtype = path, "cuda:0", torch.bfloat16
+ segmenter._weights_key = (plan.model_dir, plan.weights, plan.dequantize)
+ segmenter.precision, segmenter.weights_file = "int8", "w.safetensors"
+ return plan
+
+
+def _weights(model):
+ return {k: v.detach().clone() for k, v in model.state_dict().items()}
+
+
+def test_park_keeps_the_model_in_host_ram_and_the_next_load_reuses_it(monkeypatch):
+ """The render (or the Detailer) parks SAM 3 instead of dropping it: its weight storages are emptied, and the next
+ call on the same card gives the same module its exact bytes back with no second load."""
+ import torch
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ model = _small_model()
+ before = _weights(model)
+ _gpu_loaded(monkeypatch, segmenter, model)
+ loads = []
+ monkeypatch.setattr(segmenter, "_load_weights", lambda *a, **k: loads.append(1))
+ monkeypatch.setattr(
+ "transformers.AutoTokenizer.from_pretrained",
+ lambda *a, **k: loads.append("tok"),
+ raising = False,
+ )
+ assert segmenter.release("image generation", wait = True, park = True) is True
+ assert segmenter.parked and not segmenter.loaded and not segmenter.holds_vram()
+ # Every weight storage gave its bytes up (the tied buffer once); the host buffer holds them.
+ assert all(t.untyped_storage().nbytes() == 0 for t in model.state_dict().values())
+ host = segmenter._parked["host"]
+ # One device storage behind every weight, emptied; one host buffer holds them (each slot 256-byte aligned).
+ assert host.storage.nbytes() == 0
+ assert host.total >= sum(v.numel() * v.element_size() for k, v in before.items() if k != "tied")
+ # A render's release while only the parked copy exists keeps it.
+ assert segmenter.release("image generation", wait = True, park = True) is False
+ assert segmenter.parked
+ segmenter._load("/sam3", "cuda:0", torch.bfloat16)
+ assert loads == []
+ assert segmenter._model is model and segmenter.device == "cuda:0" and not segmenter.parked
+ assert (segmenter.precision, segmenter.weights_file) == ("int8", "w.safetensors")
+ after = _weights(model)
+ assert all(torch.equal(before[k], after[k]) for k in before)
+ assert model.tied.data_ptr() == model.table.data_ptr()
+ # The second park reuses the same host buffer (no new allocation) and round-trips the bytes again.
+ buffer = host.buffer
+ segmenter.release("detailer", wait = True, park = True)
+ assert segmenter._parked["host"] is host and host.buffer is buffer
+ assert all(t.untyped_storage().nbytes() == 0 for t in model.state_dict().values())
+ segmenter._load("/sam3", "cuda:0", torch.bfloat16)
+ assert all(torch.equal(before[k], v) for k, v in _weights(model).items())
+ segmenter._cancel_park_timer()
+
+
+def test_a_full_release_or_another_load_drops_the_parked_copy(monkeypatch):
+ import torch
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ _gpu_loaded(monkeypatch, segmenter, _small_model())
+ segmenter.release("detailer", wait = True, park = True)
+ # Training start / a GPU handover: everything goes, the host copy too.
+ assert segmenter.release("training start", wait = True) is True
+ assert not segmenter.parked and not segmenter.loaded and segmenter._host is None
+
+ _gpu_loaded(monkeypatch, segmenter, _small_model())
+ segmenter.release("detailer", wait = True, park = True)
+ fresh = []
+ monkeypatch.setattr(
+ segmenter,
+ "_load_weights",
+ lambda plan, path, device, dtype: (fresh.append(device) or _small_model(), plan),
+ )
+ monkeypatch.setattr(
+ "transformers.AutoTokenizer.from_pretrained", lambda *a, **k: "tok", raising = False
+ )
+ # The CPU fallback (fp32) is a different load: the parked GPU copy is dropped and the files are read.
+ segmenter._load("/sam3", "cpu", torch.float32)
+ assert fresh == ["cpu"] and not segmenter.parked
+
+
+def test_park_expires_and_a_cpu_model_is_never_parked(monkeypatch):
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ segmenter.park_seconds = 0.05
+ _gpu_loaded(monkeypatch, segmenter, _small_model())
+ segmenter.release("detailer", wait = True, park = True)
+ deadline = time.time() + 5
+ while segmenter.parked and time.time() < deadline:
+ time.sleep(0.02)
+ assert not segmenter.parked
+
+ model = _small_model()
+ _gpu_loaded(monkeypatch, segmenter, model)
+ segmenter.device = "cpu"
+ assert segmenter.release("detailer", wait = True, park = True) is True
+ assert not segmenter.parked and not segmenter.loaded
+ assert model[0].weight.untyped_storage().nbytes() > 0
+
+
+def test_a_model_that_cannot_park_is_unloaded(monkeypatch):
+ import torch
+
+ class _Opaque(torch.Tensor):
+ """A tensor subclass with no __tensor_flatten__: nothing knows its storages."""
+
+ segmenter = seg.MaskSegmenter(keep_alive_seconds = 0)
+ model = _small_model()
+ model.register_buffer("odd", torch.zeros(2).as_subclass(_Opaque))
+ _gpu_loaded(monkeypatch, segmenter, model)
+ assert segmenter.release("detailer", wait = True, park = True) is True
+ assert not segmenter.parked and not segmenter.loaded
+ # Nothing was emptied before the refusal.
+ assert model[0].weight.untyped_storage().nbytes() > 0
+
+
def test_mask_planes_do_not_outlive_their_text(tiny_sam3, monkeypatch):
import weakref
@@ -528,6 +764,41 @@ def test_choose_device_needs_room_on_the_card(monkeypatch):
assert seg.choose_device(True) == ("cpu", torch.float32)
+def test_choose_device_credits_the_allocator_cache_parking_leaves(monkeypatch):
+ # Reusable CUDA cache counts as free, so the next detection must not fall back to the CPU.
+ import contextlib
+
+ import torch
+
+ from core.inference import diffusion_block_graph as bg
+ from core.inference import diffusion_cuda_graph as cg
+
+ gib = 1024**3
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 2)
+ monkeypatch.setattr(torch.cuda, "is_bf16_supported", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device", lambda ordinal: contextlib.nullcontext())
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda ordinal: (2 * gib, 24 * gib))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a: 12 * gib)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda *a: 4 * gib)
+ monkeypatch.setattr(bg, "pool_bytes", lambda device = None: 0)
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 0)
+ monkeypatch.setattr(seg, "_resident_cuda_ordinal", lambda: 1)
+ monkeypatch.setattr(seg, "_other_owner_on", lambda ordinal: None)
+ if getattr(torch.backends, "mps", None) is not None:
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False)
+ assert seg._cuda_free_mib(1) == (10 * 1024, 24 * 1024)
+ assert seg.choose_device(True) == ("cuda:1", torch.bfloat16)
+ # A live CUDA graph pool is reserved but not reusable: without it the card really is short.
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 7 * gib)
+ assert seg._cuda_free_mib(1) == (3 * 1024, 24 * 1024)
+ assert seg.choose_device(True) == ("cpu", torch.float32)
+ # The credit never claims more than the card.
+ monkeypatch.setattr(cg, "live_pool_free_bytes", lambda: 0)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda *a: 90 * gib)
+ assert seg._cuda_free_mib(1) == (24 * 1024, 24 * 1024)
+
+
def test_choose_device_never_shares_a_card_another_owner_holds(monkeypatch):
import contextlib
@@ -1043,14 +1314,17 @@ def test_start_download_uses_the_active_variant(monkeypatch):
class _Recorder:
def __init__(self):
self.calls = []
+ self.parks = []
def release_segmenter(
self,
reason,
wait = False,
handover = False,
+ park = False,
):
self.calls.append((reason, wait))
+ self.parks.append(park)
return True
@@ -1421,6 +1695,8 @@ def test_a_render_releases_the_segmenter_first(monkeypatch):
monkeypatch.setitem(sys.modules, "core.inference.diffusion_segment", recorder)
inference_routes._release_mask_segmenter_for_render()
assert recorder.calls == [("image generation", True)]
+ # Parked in host RAM for the next Find or Detailer pass, not dropped.
+ assert recorder.parks == [True]
# Never loaded: nothing is imported.
monkeypatch.delitem(sys.modules, "core.inference.diffusion_segment")
inference_routes._release_mask_segmenter_for_render()
diff --git a/studio/backend/tests/test_liveness_reports_inference_active.py b/studio/backend/tests/test_liveness_reports_inference_active.py
index 598365a4bee..b154c6d79a7 100644
--- a/studio/backend/tests/test_liveness_reports_inference_active.py
+++ b/studio/backend/tests/test_liveness_reports_inference_active.py
@@ -282,16 +282,192 @@ def test_liveness_covers_the_image_persist_tail():
assert routes_inference.generation_in_flight() is False
+_PERSIST_COUNTER = "_diffusion_persist_active"
+# The two image routes' run bodies: the Unsloth /images/generate and the OpenAI-compatible /images/generations.
+_PERSIST_ROUTES = {"_generate_diffusion_image_run", "_generate_openai_images"}
+
+
+def _persist_marker_violations(source: str) -> list[str]:
+ """Each image route raises the persist marker, and every raise is dropped on every exit.
+
+ A raise is balanced when the statement after it is a ``try`` whose ``finally`` drops it, or (a hold handed to a
+ later persist) when it sits under ``if C:`` and its ``try`` drops it in an ``except BaseException`` that re-raises,
+ and a later ``try`` in the same block drops ``2 if C else 1`` in its ``finally``, with nothing that can return or
+ await in between."""
+ import ast
+
+ def is_counter(node, op):
+ return (
+ isinstance(node, ast.AugAssign)
+ and isinstance(node.target, ast.Name)
+ and node.target.id == _PERSIST_COUNTER
+ and isinstance(node.op, op)
+ )
+
+ def is_one(value):
+ return isinstance(value, ast.Constant) and value.value == 1
+
+ def two_if(value):
+ """The condition C of ``2 if C else 1``, else None."""
+ if (
+ isinstance(value, ast.IfExp)
+ and isinstance(value.body, ast.Constant)
+ and value.body.value == 2
+ and is_one(value.orelse)
+ ):
+ return ast.dump(value.test)
+ return None
+
+ def finally_drop(node):
+ drops = [s for s in node.finalbody if is_counter(s, ast.Sub)]
+ return drops[0] if len(drops) == 1 else None
+
+ def reraising_drop(node):
+ for handler in node.handlers:
+ catches_all = handler.type is None or (
+ isinstance(handler.type, ast.Name) and handler.type.id == "BaseException"
+ )
+ body = handler.body
+ if (
+ catches_all
+ and body
+ and is_counter(body[0], ast.Sub)
+ and is_one(body[0].value)
+ and isinstance(body[-1], ast.Raise)
+ and body[-1].exc is None
+ ):
+ return body[0]
+ return None
+
+ tree = ast.parse(source)
+ problems: list[str] = []
+ mutators: set[str] = set()
+ for func in tree.body:
+ if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ name = func.name
+ parents = {}
+ for node in ast.walk(func):
+ for field, value in ast.iter_fields(node):
+ if isinstance(value, list):
+ for index, child in enumerate(value):
+ if isinstance(child, ast.AST):
+ parents[id(child)] = (node, value, index)
+ raises = [n for n in ast.walk(func) if is_counter(n, ast.Add)]
+ drops = [n for n in ast.walk(func) if is_counter(n, ast.Sub)]
+ if not raises and not drops:
+ continue
+ mutators.add(name)
+ accounted: set[int] = set()
+ handoffs: dict[str, tuple] = {}
+ releases: dict[str, tuple] = {}
+ for stmt in raises:
+ owner, body, index = parents[id(stmt)]
+ follow = body[index + 1] if index + 1 < len(body) else None
+ if not is_one(stmt.value) or not isinstance(follow, ast.Try):
+ problems.append(
+ f"{name}:{stmt.lineno}: the raise is not followed by the try that drops it"
+ )
+ continue
+ drop = finally_drop(follow)
+ if drop is not None and is_one(drop.value):
+ accounted.add(id(drop))
+ continue
+ if drop is not None and two_if(drop.value) is not None:
+ accounted.add(id(drop))
+ releases[two_if(drop.value)] = (stmt, body, index)
+ continue
+ handed = reraising_drop(follow)
+ if (
+ drop is None
+ and handed is not None
+ and isinstance(owner, ast.If)
+ and body is owner.body
+ ):
+ accounted.add(id(handed))
+ handoffs[ast.dump(owner.test)] = (stmt, owner)
+ continue
+ problems.append(
+ f"{name}:{stmt.lineno}: a raise is not dropped on every exit of its try"
+ )
+ for drop in drops:
+ if id(drop) not in accounted:
+ problems.append(f"{name}:{drop.lineno}: a drop with no matching raise")
+ for cond, (stmt, guard) in handoffs.items():
+ if cond not in releases:
+ problems.append(f"{name}:{stmt.lineno}: a handed-off hold no later persist drops")
+ continue
+ _, release_body, release_index = releases.pop(cond)
+ _, guard_body, guard_index = parents[id(guard)]
+ if guard_body is not release_body or guard_index >= release_index:
+ problems.append(
+ f"{name}:{stmt.lineno}: the hold's release is not later in its block"
+ )
+ continue
+ for between in guard_body[guard_index + 1 : release_index]:
+ if isinstance(between, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue # defined here, run inside the release's try
+ for node in ast.walk(between):
+ if isinstance(node, (ast.Return, ast.Raise, ast.Await, ast.Yield)):
+ problems.append(
+ f"{name}:{node.lineno}: an exit between a held marker and its release"
+ )
+ for cond, (stmt, _, _) in releases.items():
+ problems.append(f"{name}:{stmt.lineno}: a persist drops a hold nothing raised")
+ for route in sorted(_PERSIST_ROUTES - mutators):
+ problems.append(f"{route}: persists gallery records without publishing the marker")
+ for other in sorted(mutators - _PERSIST_ROUTES):
+ problems.append(f"{other}: moves the persist marker outside the image routes")
+ return problems
+
+
def test_both_image_routes_publish_the_persist_marker():
"""The Unsloth route and the OpenAI-compatible one write the gallery on separate paths.
Only the first used to count, so an OpenAI client's persist was invisible to both
- generate-progress and liveness."""
+ generate-progress and liveness. Each must raise the marker, and drop every raise on every exit
+ (the Detailer's hold included), or liveness reads idle mid-request or busy forever."""
import inspect
import importlib
source = inspect.getsource(importlib.import_module("routes.inference"))
- assert source.count("_diffusion_persist_active += 1") == 2, (
- "an image route persists gallery records without publishing the marker, so liveness "
- "reports idle while that request is still in flight"
+ assert _persist_marker_violations(source) == [], (
+ "an image route persists gallery records without a balanced marker, so liveness "
+ "misreports that request"
)
- assert source.count("_diffusion_persist_active -= 1") == 2
+
+
+def test_the_persist_marker_check_catches_each_imbalance():
+ """The check above fails on each way the marker can go wrong, so its pass means something."""
+ import inspect
+ import importlib
+
+ source = inspect.getsource(importlib.import_module("routes.inference"))
+ detailer_drop = "_diffusion_persist_active -= 2 if detailer is not None else 1"
+ handoff_drop = " except BaseException:\n _diffusion_persist_active -= 1\n raise\n"
+ openai_raise = ' global _diffusion_persist_active\n _diffusion_persist_active += 1\n try:\n with account_access.media_generation("diffusion"):\n data ='
+ assert source.count(detailer_drop) == 1
+ assert source.count(handoff_drop) == 1
+ assert source.count(openai_raise) == 1
+ mutants = {
+ "the persist no longer drops the Detailer's hold": source.replace(
+ detailer_drop, "_diffusion_persist_active -= 1"
+ ),
+ "a failed Detailer pass leaks its hold": source.replace(
+ handoff_drop, " except BaseException:\n raise\n"
+ ),
+ "the OpenAI route stops publishing": source.replace(
+ openai_raise,
+ openai_raise.replace(" _diffusion_persist_active += 1\n", " pass\n"),
+ ),
+ "the OpenAI route never drops its marker": source.replace(
+ " finally:\n _diffusion_persist_active -= 1\n\n return ImageGen",
+ " finally:\n pass\n\n return ImageGen",
+ ),
+ "a return between the hold and its release": source.replace(
+ " created_at = time.time()\n per_image_seeds",
+ " if not result:\n return None\n created_at = time.time()\n per_image_seeds",
+ ),
+ }
+ for why, mutant in mutants.items():
+ assert mutant != source, why
+ assert _persist_marker_violations(mutant), why
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index c1ae15d7100..d9828de5351 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -11,6 +11,7 @@ import {
MEMORY_REFUSAL_HEADER,
MemoryEstimateRefusalError,
} from "./lib/memory-refusal";
+import type { DetailerRecord, DetailerRequest } from "./lib/detailer";
import type { PromptEnhancement } from "./lib/prompt-enhance";
import type { HiresCapability, HiresRequest } from "./lib/hires-choice";
import type { SdControlsCapability } from "./lib/sd-controls";
@@ -263,6 +264,7 @@ export interface DiffusionGenerateRequest {
mask_blur?: number;
inpaint_only_masked?: boolean;
inpaint_padding?: number;
+ detailer?: DetailerRequest;
// Upscale (hires fix): factor > 1 with an init_image enlarges the source and re-denoises at low strength.
upscale?: number;
allow_oversized?: boolean;
@@ -401,6 +403,8 @@ export interface GalleryImage {
mask_blur?: number | null;
inpaint_only_masked?: boolean | null;
inpaint_padding?: number | null;
+ // The Detailer pass this image ran (settings, detections) or why it did not; absent = none.
+ detailer?: DetailerRecord | null;
upscale_model?: string | null;
// Hires fix; `width` / `height` stay the output size, hires_base_* the base pass.
hires_scale?: number | null;
diff --git a/studio/frontend/src/features/images/detailer-field.tsx b/studio/frontend/src/features/images/detailer-field.tsx
new file mode 100644
index 00000000000..1af1865ba32
--- /dev/null
+++ b/studio/frontend/src/features/images/detailer-field.tsx
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { useCallback, useEffect, useRef, useState, type ReactNode } from "react";
+
+import { Input } from "@/components/ui/input";
+import { InfoHint } from "@/components/ui/info-hint";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import { ParamSlider } from "@/features/chat";
+
+import {
+ DETAILER_DEFAULTS,
+ DETAILER_HINT,
+ DETAILER_LABEL,
+ DETAILER_PROMPT_HINT,
+ DETAILER_TARGETS_HINT,
+ type DetailerOptions,
+} from "./lib/detailer";
+import { SEGMENTER_STATUS_EVENT, type SegmenterStatus } from "./lib/mask-from-text";
+import { getSegmenterStatus } from "./mask-segment-api";
+
+/** SAM 3 status when the model can inpaint, else null; polled during a download so the control appears. */
+export function useDetailerSegmenter(modelCapable: boolean, modelKey: string): SegmenterStatus | null {
+ const [status, setStatus] = useState<SegmenterStatus | null>(null);
+ const seq = useRef(0);
+ const read = useCallback(() => {
+ const run = ++seq.current;
+ getSegmenterStatus()
+ .then((next) => {
+ if (run === seq.current) setStatus(next);
+ })
+ .catch(() => {
+ if (run === seq.current) setStatus(null);
+ });
+ }, []);
+ useEffect(() => {
+ if (!modelCapable) {
+ seq.current += 1;
+ setStatus(null);
+ return;
+ }
+ read();
+ const onStatus = (event: Event) => {
+ const next = (event as CustomEvent<SegmenterStatus>).detail;
+ if (next) {
+ seq.current += 1;
+ setStatus(next);
+ }
+ };
+ window.addEventListener(SEGMENTER_STATUS_EVENT, onStatus);
+ return () => window.removeEventListener(SEGMENTER_STATUS_EVENT, onStatus);
+ }, [modelCapable, modelKey, read]);
+ const downloading = modelCapable && status?.download.downloading === true;
+ useEffect(() => {
+ if (!downloading) return;
+ const id = window.setInterval(read, 2000);
+ return () => window.clearInterval(id);
+ }, [downloading, read]);
+ return status;
+}
+
+/** Empty means "the image's own"; keeps partial text like "4." while editing. */
+function OptionalNumberInput({
+ ariaLabel,
+ value,
+ onChange,
+}: {
+ ariaLabel: string;
+ value: number | null;
+ onChange: (next: number | null) => void;
+}) {
+ const [text, setText] = useState(value === null ? "" : String(value));
+ useEffect(() => {
+ setText((current) => {
+ const n = current.trim() === "" ? null : Number(current);
+ return n === value ? current : value === null ? "" : String(value);
+ });
+ }, [value]);
+ return (
+ <Input
+ aria-label={ariaLabel}
+ inputMode="decimal"
+ className="h-8 text-xs"
+ placeholder="Same as the image"
+ value={text}
+ onChange={(e) => {
+ setText(e.target.value);
+ const raw = e.target.value.trim();
+ const n = Number(raw);
+ onChange(raw === "" || !Number.isFinite(n) || n < 0 ? null : n);
+ }}
+ />
+ );
+}
+
+function Row({ label, hint, children }: { label: string; hint: string; children: ReactNode }) {
+ return (
+ <div className="flex flex-col gap-1.5">
+ <div className="flex items-center gap-1">
+ <label className="text-xs font-medium text-muted-foreground">{label}</label>
+ <InfoHint>{hint}</InfoHint>
+ </div>
+ {children}
+ </div>
+ );
+}
+
+export function DetailerField({
+ available,
+ enabled,
+ onEnabledChange,
+ options,
+ onOptionsChange,
+}: {
+ available: boolean;
+ enabled: boolean;
+ onEnabledChange: (on: boolean) => void;
+ options: DetailerOptions;
+ onOptionsChange: (next: DetailerOptions) => void;
+}) {
+ if (!available) return null;
+ const set = <K extends keyof DetailerOptions>(key: K, value: DetailerOptions[K]) =>
+ onOptionsChange({ ...options, [key]: value });
+ return (
+ <div className="flex flex-col gap-2 pt-2" data-testid="images-detailer">
+ <div className="flex items-center justify-between">
+ <span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">
+ {DETAILER_LABEL}
+ <InfoHint>{DETAILER_HINT}</InfoHint>
+ </span>
+ <Switch checked={enabled} onCheckedChange={onEnabledChange} aria-label={DETAILER_LABEL} />
+ </div>
+ {enabled && (
+ <div className="flex flex-col gap-2 rounded-md border border-border/60 px-2.5 py-2">
+ <Row label="Fix" hint={DETAILER_TARGETS_HINT}>
+ <Input
+ aria-label="Detailer targets"
+ className="h-8 text-xs"
+ placeholder={DETAILER_DEFAULTS.targets}
+ value={options.targets}
+ onChange={(e) => set("targets", e.target.value)}
+ />
+ </Row>
+ <Row label="Detailer prompt" hint={DETAILER_PROMPT_HINT}>
+ <Textarea
+ aria-label="Detailer prompt"
+ className="min-h-14 text-xs"
+ placeholder="Same as the image's prompt"
+ value={options.prompt}
+ onChange={(e) => set("prompt", e.target.value)}
+ />
+ </Row>
+ <Row label="Detailer negative prompt" hint="Negative prompt for the redraw. Empty uses the image's negative prompt; [SEP] works as in the prompt.">
+ <Textarea
+ aria-label="Detailer negative prompt"
+ className="min-h-10 text-xs"
+ placeholder="Same as the image's negative prompt"
+ value={options.negativePrompt}
+ onChange={(e) => set("negativePrompt", e.target.value)}
+ />
+ </Row>
+ <ParamSlider
+ inline={true}
+ label="Denoise"
+ info="How much each face is redrawn. Lower keeps it closer to the original; FaceDetailer's default is 0.5."
+ value={options.denoise}
+ min={0.05}
+ max={1}
+ step={0.05}
+ onChange={(v) => set("denoise", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Crop factor"
+ info="How much of the surroundings each redraw sees, times the face's box. More context blends better; less gives the face more pixels."
+ value={options.cropFactor}
+ min={1}
+ max={6}
+ step={0.1}
+ onChange={(v) => set("cropFactor", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Guide size"
+ info="The face's shorter side is enlarged to this many pixels before the redraw (never past Max size)."
+ value={options.guideSize}
+ min={256}
+ max={1536}
+ step={8}
+ onChange={(v) => set("guideSize", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Max size"
+ info="Caps the enlargement: an enlarged face's crop stays within this many pixels. A crop already larger is redrawn at its own size, never shrunk, as in ComfyUI's FaceDetailer."
+ value={options.maxSize}
+ min={256}
+ max={2048}
+ step={8}
+ onChange={(v) => set("maxSize", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Feather"
+ info="Blur, in pixels, where the redraw meets the original."
+ value={options.feather}
+ min={0}
+ max={32}
+ step={1}
+ onChange={(v) => set("feather", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Mask grow"
+ info="Grow each found mask by this many pixels (negative shrinks it)."
+ value={options.dilation}
+ min={-32}
+ max={64}
+ step={1}
+ onChange={(v) => set("dilation", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Drop size"
+ info="Faces whose box is this many pixels or fewer on a side are left alone."
+ value={options.dropSize}
+ min={1}
+ max={128}
+ step={1}
+ onChange={(v) => set("dropSize", v)}
+ />
+ <ParamSlider
+ inline={true}
+ label="Confidence"
+ info="SAM 3 score a face needs to be fixed."
+ value={options.confidence}
+ min={0.05}
+ max={0.95}
+ step={0.05}
+ onChange={(v) => set("confidence", v)}
+ />
+ <Row label="Detailer CFG" hint="Guidance for each redraw. Empty uses the image's guidance.">
+ <OptionalNumberInput
+ ariaLabel="Detailer CFG"
+ value={options.guidance}
+ onChange={(v) => set("guidance", v)}
+ />
+ </Row>
+ <Row label="Detailer steps" hint="Steps for each redraw. Empty uses the image's steps.">
+ <Input
+ aria-label="Detailer steps"
+ inputMode="numeric"
+ className="h-8 text-xs"
+ placeholder="Same as the image"
+ value={options.steps === null ? "" : String(options.steps)}
+ onChange={(e) => {
+ const raw = e.target.value.trim();
+ const n = Number(raw);
+ set("steps", raw === "" || !Number.isFinite(n) || n < 1 ? null : Math.round(n));
+ }}
+ />
+ </Row>
+ </div>
+ )}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 588402ba46a..d33cf2cc4c1 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -268,6 +268,21 @@ import { LocalizedEditCanvas } from "./localized-edit-canvas";
import type { AppliedMask } from "./lib/mask-from-text";
import { applyMaskToLayer } from "./mask-apply";
import { MaskFromTextPanel } from "./mask-from-text-panel";
+import { DetailerField, useDetailerSegmenter } from "./detailer-field";
+import {
+ detailerAvailable,
+ detailerNotApplied,
+ detailerEngagesNegative,
+ detailerErrors,
+ detailerModelCapable,
+ detailerRecordOf,
+ detailerRequestField,
+ detailerSummary,
+ readDetailerOptions,
+ restoredDetailer,
+ saveDetailerOptions,
+ type DetailerOptions,
+} from "./lib/detailer";
import {
type ControlNetSpecInput,
@@ -1467,6 +1482,9 @@ function RecipePopover({
<RecipeRow key={label} label={label} value={value} />
))}
{!viaConnection && <RecipeRow label="Seed" value={String(image.seed)} mono />}
+ {detailerSummary(detailerRecordOf(image.detailer)) ? (
+ <RecipeRow label="Detailer" value={detailerSummary(detailerRecordOf(image.detailer)) ?? ""} />
+ ) : null}
</div>
<div className="shrink-0 border-t border-border/60 px-3 py-2.5">
<Button size="sm" className="w-full gap-1.5" onClick={() => onRestore(image)}>
@@ -2285,12 +2303,6 @@ export function ImagesPage({
// A connection runs none of the local engine's Hires fix / model upscalers (the route refuses both).
const hiresCapability = !external && status?.loaded ? (status.hires ?? null) : null;
const upscaleModels = !external && status?.loaded ? status.upscale_models : undefined;
- // Hires CFG can turn CFG on for the refine pass alone; that pass then reads the negative prompt.
- const negativeEngaged =
- negativeCapable &&
- (negativeMinGuidance === null ||
- effectiveGuidance > negativeMinGuidance ||
- hiresEngagesNegative(hires, hiresCapability, workflow, negativeMinGuidance));
const livePreviewUnavailable =
!external && status?.loaded === true && status.supports_live_preview === false;
const samplerOptions = (!external && status?.loaded ? status.samplers : null) ?? NO_SAMPLERS;
@@ -2325,6 +2337,23 @@ export function ImagesPage({
[samplerModelKey],
);
const embeddingsEnabled = !external && status?.loaded === true && !!status.textual_inversion;
+ // Detailer (face fix): off by default; its options persist, the switch does not.
+ const [detailerOn, setDetailerOn] = useState(false);
+ const [detailerOptions, setDetailerOptionsState] = useState<DetailerOptions>(readDetailerOptions);
+ const setDetailerOptions = useCallback((next: DetailerOptions) => {
+ setDetailerOptionsState(next);
+ saveDetailerOptions(next);
+ }, []);
+ const detailerCapable = detailerModelCapable(status, Boolean(external));
+ const detailerSegmenter = useDetailerSegmenter(detailerCapable, loadedModelKey(status) ?? "");
+ const detailerReady = detailerAvailable(status, Boolean(external), detailerSegmenter);
+ // Hires CFG or the Detailer's CFG can turn CFG on for that pass alone; it then reads the negative prompt.
+ const negativeEngaged =
+ negativeCapable &&
+ (negativeMinGuidance === null ||
+ effectiveGuidance > negativeMinGuidance ||
+ hiresEngagesNegative(hires, hiresCapability, workflow, negativeMinGuidance) ||
+ detailerEngagesNegative(detailerOn, detailerReady, detailerOptions, negativeMinGuidance));
const insertEmbedding = useCallback((trigger: string, target: "prompt" | "negative") => {
if (target === "negative") setNegativePrompt((p) => withEmbeddingTrigger(p, trigger));
else setPrompt((p) => withEmbeddingTrigger(p, trigger));
@@ -2919,6 +2948,9 @@ export function ImagesPage({
else setStrength(image.strength);
}
if (typeof image.upscale === "number") setUpscaleFactor(image.upscale);
+ const detail = restoredDetailer(image);
+ setDetailerOn(detail.enabled);
+ if (detail.enabled) setDetailerOptions({ ...readDetailerOptions(), ...detail.options });
const maskOptions = restoredInpaintMaskOptions(image);
if (maskOptions.maskBlur !== undefined) setMaskBlur(maskOptions.maskBlur);
if (maskOptions.onlyMasked !== undefined) setInpaintOnlyMasked(maskOptions.onlyMasked);
@@ -2973,13 +3005,15 @@ export function ImagesPage({
const notApplied = [
...(literal ? [literal] : []),
...(via === "restore" && status?.loaded ? sdControlsNotRestored(image, sdCaps) : []),
+ // Restored into the hidden control, which Generate omits: say so, as an import does.
+ ...(via === "restore" ? detailerNotApplied(detail.enabled, detailerReady) : []),
];
if (notApplied.length > 0) {
toast.warning("Some settings in this image were not applied", {
description: `Not applied: ${notApplied.join(", ")}. This model or build does not support them.`,
});
}
- }, [changeSamplerChoice, samplerOptions.length, sdCaps, setEnhanceOverride, setPromptFor, setShiftValue, setWorkflow, sizeLimits, status]);
+ }, [changeSamplerChoice, samplerOptions.length, sdCaps, setDetailerOptions, setEnhanceOverride, setPromptFor, setShiftValue, setWorkflow, sizeLimits, status, detailerReady]);
// Like dropping an image on ComfyUI: Unsloth's recipe first, else AUTOMATIC1111 / ComfyUI metadata.
const importFileRef = useRef<HTMLInputElement>(null);
@@ -3106,6 +3140,8 @@ export function ImagesPage({
...hiresNotApplied(hiresFromRecipe(settings), status?.loaded ? status.hires : null),
...unknownRecipeKeys(recipe),
];
+ // Restored into the hidden control when this model cannot inpaint or SAM 3 is missing: say so.
+ notApplied.unshift(...detailerNotApplied(Boolean(settings.detailer), detailerReady));
if (tiMatch) {
const { installed, missing } = tiMatch;
const resolved = new Map<string, string | null>([
@@ -3167,7 +3203,7 @@ export function ImagesPage({
}
return true;
},
- [restoreSettings, width, height, steps, guidance, status, samplerOptions, availableLoras, loraCapable, loraCatalogFamily, sdCaps, embeddingsEnabled, shiftControl],
+ [restoreSettings, width, height, steps, guidance, status, samplerOptions, availableLoras, loraCapable, loraCatalogFamily, sdCaps, embeddingsEnabled, shiftControl, detailerReady],
);
const importSettingsFromFile = useCallback(
async (file: File | Uint8Array) => {
@@ -5470,6 +5506,7 @@ export function ImagesPage({
...shiftRequestFields(shiftControl, shiftValue),
...enhancePromptRequestField(upsamplerCapability, enhanceOn),
...condFields,
+ ...detailerRequestField(detailerOn, detailerReady, detailerOptions),
// Drop empty and zero-weight rows and trim hand-typed repo ids, so the recipe records only
// adapters that applied. Gated on loraCapable, since a restore can leave adapters in state.
loras: (() => {
@@ -5527,6 +5564,13 @@ export function ImagesPage({
if (notice) toast.info(notice.title, { id: "prompt-enhancement-notice", description: notice.description });
}
res.images.forEach((image) => void ensureSrc(image));
+ const detailFailed = detailerErrors(res.images);
+ if (detailFailed.length > 0) {
+ toast.warning("The Detailer did not run on this image", {
+ id: "images-detailer-error",
+ description: detailFailed.join("; "),
+ });
+ }
compare?.onImages(i, res.images);
setGenDone(i + 1);
}
@@ -5578,7 +5622,7 @@ export function ImagesPage({
setGenStep(null);
setStopping(false);
}
- }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeEngaged, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, sdCaps, sdChoice, shiftControl, shiftValue, width, height, steps, effectiveGuidance, seed, batchSize, count, workflow, initImage, maskImage, strength, maskBlur, inpaintOnlyMasked, inpaintPadding, status?.supports_inpaint_only_masked, extendPct, extendSides, upscaleFactor, upscaleStrength, upscaleModel, upscaleModels, hires, hiresCapability, referenceImages, loras, loraCapable, controlnetCapable, controlnetEngaged, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, multiImageEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
+ }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeEngaged, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, sdCaps, sdChoice, shiftControl, shiftValue, width, height, steps, effectiveGuidance, seed, batchSize, count, workflow, initImage, maskImage, strength, maskBlur, inpaintOnlyMasked, inpaintPadding, status?.supports_inpaint_only_masked, extendPct, extendSides, upscaleFactor, upscaleStrength, upscaleModel, upscaleModels, hires, hiresCapability, referenceImages, loras, loraCapable, controlnetCapable, controlnetEngaged, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, multiImageEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits, detailerOn, detailerReady, detailerOptions]);
// Stop the in-flight generation. Latch FIRST, so a multi-run request stops even if the POST
// races the run that is already finishing.
@@ -7091,6 +7135,13 @@ export function ImagesPage({
enabled={embeddingsEnabled}
onInsert={insertEmbedding}
/>
+ <DetailerField
+ available={detailerReady}
+ enabled={detailerOn}
+ onEnabledChange={setDetailerOn}
+ options={detailerOptions}
+ onOptionsChange={setDetailerOptions}
+ />
{!external && (
<>
<Field
diff --git a/studio/frontend/src/features/images/lib/detailer.ts b/studio/frontend/src/features/images/lib/detailer.ts
new file mode 100644
index 00000000000..5312f0478c5
--- /dev/null
+++ b/studio/frontend/src/features/images/lib/detailer.ts
@@ -0,0 +1,355 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Detailer (face fix) logic. No "@/" imports so node:test can load it directly.
+
+export const DETAILER_LABEL = "Detailer (face fix)";
+export const DETAILER_HINT =
+ "After each image, finds every face (or what you name) with SAM 3, redraws it at a higher resolution and pastes it back, like ADetailer or ComfyUI's FaceDetailer. Nothing outside the found areas changes. Each face costs one extra inpaint.";
+export const DETAILER_TARGETS_HINT =
+ "What to fix, separated by commas, as in Mask from text: face, hand, eyes. Add :N to cap a target (up to 8 each by default).";
+export const DETAILER_PROMPT_HINT =
+ "Prompt for the redraw. Empty uses the image's prompt; [PROMPT] inserts it; [SEP] gives the next faces their own text.";
+
+export type DetailerGuideSizeFor = "bbox" | "crop_region";
+
+const isGuideSizeFor = (v: unknown): v is DetailerGuideSizeFor => v === "bbox" || v === "crop_region";
+
+export interface DetailerOptions {
+ targets: string;
+ prompt: string;
+ denoise: number;
+ confidence: number;
+ cropFactor: number;
+ guideSize: number;
+ guideSizeFor: DetailerGuideSizeFor;
+ maxSize: number;
+ dilation: number;
+ feather: number;
+ dropSize: number;
+ steps: number | null;
+ negativePrompt: string;
+ /** null / empty = the image's own. */
+ guidance: number | null;
+}
+
+/** ComfyUI-Impact-Pack FaceDetailer's defaults (modules/impact/impact_pack.py:743-769). */
+export const DETAILER_DEFAULTS: DetailerOptions = {
+ targets: "face",
+ prompt: "",
+ denoise: 0.5,
+ confidence: 0.5,
+ cropFactor: 3.0,
+ guideSize: 512,
+ guideSizeFor: "bbox",
+ maxSize: 1024,
+ dilation: 10,
+ feather: 5,
+ dropSize: 10,
+ steps: null,
+ negativePrompt: "",
+ guidance: null,
+};
+
+export interface DetailerRequest {
+ targets: string;
+ prompt?: string;
+ denoise?: number;
+ confidence?: number;
+ crop_factor?: number;
+ guide_size?: number;
+ guide_size_for?: DetailerGuideSizeFor;
+ max_size?: number;
+ dilation?: number;
+ feather?: number;
+ drop_size?: number;
+ steps?: number;
+ negative_prompt?: string;
+ guidance?: number;
+}
+
+export interface DetailerDetection {
+ target?: string;
+ score?: number;
+ box?: number[];
+ crop?: number[];
+ size?: number[];
+ seed?: number;
+ prompt?: string;
+}
+
+export interface DetailerRecord {
+ targets?: string;
+ prompt?: string;
+ negative_prompt?: string;
+ confidence?: number;
+ denoise?: number;
+ crop_factor?: number;
+ guide_size?: number;
+ guide_size_for?: string;
+ max_size?: number;
+ dilation?: number;
+ feather?: number;
+ drop_size?: number;
+ steps?: number;
+ guidance?: number;
+ model?: string;
+ detections?: DetailerDetection[];
+ found?: number;
+ error?: string;
+ [key: string]: unknown;
+}
+
+interface DetailerModelStatus {
+ loaded?: boolean;
+ workflows?: string[] | null;
+}
+
+interface DetailerSegmenterStatus {
+ supported: boolean;
+ downloaded: boolean;
+}
+
+export function detailerModelCapable(status: DetailerModelStatus | null | undefined, external: boolean): boolean {
+ return !external && status?.loaded === true && (status.workflows ?? []).includes("inpaint");
+}
+
+export function detailerAvailable(
+ status: DetailerModelStatus | null | undefined,
+ external: boolean,
+ segmenter: DetailerSegmenterStatus | null | undefined,
+): boolean {
+ return detailerModelCapable(status, external) && segmenter?.supported === true && segmenter.downloaded === true;
+}
+
+const clamp = (n: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, n));
+
+export function detailerRequestField(
+ enabled: boolean,
+ available: boolean,
+ options: DetailerOptions,
+): { detailer?: DetailerRequest } {
+ if (!enabled || !available) return {};
+ const targets = options.targets.trim() || DETAILER_DEFAULTS.targets;
+ const out: DetailerRequest = { targets };
+ const prompt = options.prompt.trim();
+ if (prompt) out.prompt = prompt;
+ const numeric: Array<[keyof DetailerRequest, number, number, number, number]> = [
+ ["denoise", options.denoise, DETAILER_DEFAULTS.denoise, 0.01, 1],
+ ["confidence", options.confidence, DETAILER_DEFAULTS.confidence, 0, 1],
+ ["crop_factor", options.cropFactor, DETAILER_DEFAULTS.cropFactor, 1, 10],
+ ["guide_size", options.guideSize, DETAILER_DEFAULTS.guideSize, 64, 2048],
+ ["max_size", options.maxSize, DETAILER_DEFAULTS.maxSize, 64, 2048],
+ ["dilation", options.dilation, DETAILER_DEFAULTS.dilation, -128, 128],
+ ["feather", options.feather, DETAILER_DEFAULTS.feather, 0, 100],
+ ["drop_size", Math.round(options.dropSize), DETAILER_DEFAULTS.dropSize, 1, 1024],
+ ];
+ for (const [key, value, fallback, lo, hi] of numeric) {
+ if (!Number.isFinite(value)) continue;
+ const bounded = clamp(value, lo, hi);
+ if (bounded !== fallback) (out as unknown as Record<string, number>)[key] = bounded;
+ }
+ if (isGuideSizeFor(options.guideSizeFor) && options.guideSizeFor !== DETAILER_DEFAULTS.guideSizeFor)
+ out.guide_size_for = options.guideSizeFor;
+ if (options.steps !== null && Number.isFinite(options.steps)) out.steps = clamp(Math.round(options.steps), 1, 200);
+ const negative = (options.negativePrompt ?? "").trim();
+ if (negative) out.negative_prompt = negative;
+ if (options.guidance !== null && options.guidance !== undefined && Number.isFinite(options.guidance))
+ out.guidance = clamp(options.guidance, 0, 100);
+ return { detailer: out };
+}
+
+export function detailerTemplateInherits(template: string | undefined): boolean {
+ if (!template) return true;
+ return template.split(/\s*\[SEP\]\s*/).some((part) => !part.trim() || part.includes("[PROMPT]"));
+}
+
+/** The Detailer's CFG can engage the inherited negative prompt even when the base CFG leaves it unused. */
+export function detailerEngagesNegative(
+ enabled: boolean,
+ available: boolean,
+ options: DetailerOptions,
+ minGuidance: number | null,
+): boolean {
+ const req = detailerRequestField(enabled, available, options).detailer;
+ if (!req || !detailerTemplateInherits(req.negative_prompt) || req.guidance === undefined) return false;
+ return minGuidance === null || req.guidance > minGuidance;
+}
+
+const num = (v: unknown): number | undefined => (typeof v === "number" && Number.isFinite(v) ? v : undefined);
+const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined);
+
+export function detailerRecordOf(value: unknown): DetailerRecord | null {
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+ const v = value as Record<string, unknown>;
+ const out: DetailerRecord = {};
+ for (const key of ["targets", "prompt", "negative_prompt", "guide_size_for", "model", "error"] as const) {
+ const s = str(v[key]);
+ if (s !== undefined) out[key] = s;
+ }
+ for (const key of [
+ "confidence",
+ "denoise",
+ "crop_factor",
+ "guide_size",
+ "max_size",
+ "dilation",
+ "feather",
+ "drop_size",
+ "steps",
+ "guidance",
+ "found",
+ ] as const) {
+ const n = num(v[key]);
+ if (n !== undefined) out[key] = n;
+ }
+ if (Array.isArray(v.detections)) out.detections = v.detections.filter((d) => d && typeof d === "object") as DetailerDetection[];
+ return Object.keys(out).length > 0 ? out : null;
+}
+
+export function restoredDetailer(image: { detailer?: unknown }): {
+ enabled: boolean;
+ options: Partial<DetailerOptions>;
+} {
+ const record = detailerRecordOf(image.detailer);
+ if (!record) return { enabled: false, options: {} };
+ // Missing settings reset to defaults so a restore never keeps a stale value.
+ const options: Partial<DetailerOptions> = { ...DETAILER_DEFAULTS };
+ if (record.targets) options.targets = record.targets;
+ options.prompt = record.prompt ?? "";
+ const pairs: Array<[keyof DetailerOptions, number | undefined]> = [
+ ["denoise", record.denoise],
+ ["confidence", record.confidence],
+ ["cropFactor", record.crop_factor],
+ ["guideSize", record.guide_size],
+ ["maxSize", record.max_size],
+ ["dilation", record.dilation],
+ ["feather", record.feather],
+ ["dropSize", record.drop_size],
+ ];
+ for (const [key, value] of pairs) {
+ if (value !== undefined) (options as Record<string, unknown>)[key] = value;
+ }
+ options.guideSizeFor = isGuideSizeFor(record.guide_size_for) ? record.guide_size_for : DETAILER_DEFAULTS.guideSizeFor;
+ options.steps = record.steps ?? null;
+ options.negativePrompt = record.negative_prompt ?? "";
+ options.guidance = record.guidance ?? null;
+ return { enabled: true, options };
+}
+
+/** First-tab `ADetailer ...` infotext fields (lower-cased keys) -> a record. */
+export function detailerFromInfotext(fields: Record<string, string>): DetailerRecord | null {
+ const get = (key: string): string | undefined => fields[`adetailer ${key}`];
+ const model = get("model");
+ if (model === undefined || model === "None") return null;
+ const n = (key: string): number | undefined => {
+ const raw = get(key);
+ if (raw === undefined || raw === "") return undefined;
+ const value = Number(raw);
+ return Number.isFinite(value) ? value : undefined;
+ };
+ const out: DetailerRecord = { model };
+ // ADetailer's YOLO models carry no text; a face model means faces, a hand model hands.
+ const classes = get("model classes");
+ out.targets = classes || (/hand/i.test(model) ? "hand" : /person/i.test(model) ? "person" : "face");
+ const prompt = get("prompt");
+ if (prompt) out.prompt = prompt;
+ const negative = get("negative prompt");
+ if (negative) out.negative_prompt = negative;
+ const confidence = n("confidence");
+ if (confidence !== undefined) out.confidence = confidence;
+ const dilation = n("dilate erode");
+ if (dilation !== undefined) out.dilation = dilation;
+ const feather = n("mask blur");
+ if (feather !== undefined) out.feather = feather;
+ const denoise = n("denoising strength");
+ if (denoise !== undefined) out.denoise = denoise;
+ if (/^true$/i.test(get("use separate steps") ?? "")) {
+ const steps = n("steps");
+ if (steps !== undefined) out.steps = steps;
+ }
+ if (/^true$/i.test(get("use separate cfg scale") ?? "")) {
+ const cfg = n("cfg scale");
+ if (cfg !== undefined) out.guidance = cfg;
+ }
+ return out;
+}
+
+const INFOTEXT_RESTORED = new Set(
+ [
+ "model",
+ "model classes",
+ "prompt",
+ "negative prompt",
+ "confidence",
+ "dilate erode",
+ "mask blur",
+ "denoising strength",
+ "use separate steps",
+ "steps",
+ "use separate cfg scale",
+ "cfg scale",
+ "version",
+ ].map((key) => `adetailer ${key}`),
+);
+
+/** `ADetailer ...` keys an import drops (later tabs, unsupported options). */
+export function detailerInfotextNotApplied(fields: Record<string, string>): string[] {
+ const restored = detailerFromInfotext(fields) !== null;
+ return Object.keys(fields).filter((key) => {
+ if (key === "adetailer version" || !fields[key]) return false;
+ if (key === "adetailer model") return !restored && fields[key] !== "None";
+ if (key === "adetailer inpaint only masked") return !(restored && /^true$/i.test(fields[key] ?? ""));
+ return !(restored && INFOTEXT_RESTORED.has(key));
+ });
+}
+
+export function detailerNotApplied(recorded: boolean, ready: boolean): string[] {
+ return recorded && !ready ? ["Detailer (face fix)"] : [];
+}
+
+export function detailerSummary(record: DetailerRecord | null | undefined): string | null {
+ if (!record) return null;
+ if (record.error) return `not applied (${record.error})`;
+ const count = record.detections?.length ?? 0;
+ const targets = record.targets ?? "face";
+ return `${targets}: ${count} fixed, denoise ${record.denoise ?? DETAILER_DEFAULTS.denoise}`;
+}
+
+export function detailerErrors(images: Array<{ detailer?: unknown }>): string[] {
+ const errors = new Set<string>();
+ for (const image of images) {
+ const error = detailerRecordOf(image.detailer)?.error;
+ if (error && error !== "cancelled") errors.add(error);
+ }
+ return [...errors];
+}
+
+const KEY = "unsloth:images:detailer-options";
+
+export function readDetailerOptions(): DetailerOptions {
+ try {
+ const value = JSON.parse(localStorage.getItem(KEY) ?? "{}");
+ if (!value || typeof value !== "object" || Array.isArray(value)) return { ...DETAILER_DEFAULTS };
+ const out = { ...DETAILER_DEFAULTS };
+ for (const key of Object.keys(DETAILER_DEFAULTS) as Array<keyof DetailerOptions>) {
+ const v = (value as Record<string, unknown>)[key];
+ if (key === "steps" || key === "guidance")
+ out[key] = typeof v === "number" && Number.isFinite(v) ? v : null;
+ else if (key === "guideSizeFor") out.guideSizeFor = isGuideSizeFor(v) ? v : DETAILER_DEFAULTS.guideSizeFor;
+ else if (typeof v === typeof DETAILER_DEFAULTS[key] && (typeof v !== "number" || Number.isFinite(v)))
+ (out as Record<string, unknown>)[key] = v;
+ }
+ return out;
+ } catch {
+ return { ...DETAILER_DEFAULTS };
+ }
+}
+
+export function saveDetailerOptions(options: DetailerOptions): void {
+ try {
+ localStorage.setItem(KEY, JSON.stringify(options));
+ } catch {
+ // storage can be unavailable in private browsing
+ }
+}
diff --git a/studio/frontend/src/features/images/lib/image-metadata.ts b/studio/frontend/src/features/images/lib/image-metadata.ts
index 4b789d21621..50e63e54287 100644
--- a/studio/frontend/src/features/images/lib/image-metadata.ts
+++ b/studio/frontend/src/features/images/lib/image-metadata.ts
@@ -1,5 +1,11 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import {
+ detailerFromInfotext,
+ detailerInfotextNotApplied,
+ detailerRecordOf,
+ type DetailerRecord,
+} from "./detailer.ts";
import { parseHiresInfotext } from "./hires-choice.ts";
import { SAMPLER_LABELS, SCHEDULE_LABELS } from "./sampler-choice.ts";
import type { SdControlsCapability } from "./sd-controls.ts";
@@ -755,6 +761,9 @@ export function parseA1111Parameters(text: string): ImportedRecipe | null {
if (escaped.length > 0) out.prompt_escaped_brackets = escaped;
const notApplied: string[] = networks.other.slice();
const hires: Record<string, string> = {};
+ // ADetailer's own keys (adetailer/args.py), read as one Detailer pass.
+ const adetailer: Record<string, string> = {};
+ const adetailerLabels: Record<string, string> = {};
let legacySchedule: string | undefined;
for (const [label, value] of hasSettings ? fields : []) {
const key = label.toLowerCase();
@@ -819,6 +828,10 @@ export function parseA1111Parameters(text: string): ImportedRecipe | null {
else if (value && value.toLowerCase() !== "original")
notApplied.push(`${label}: ${value}`);
} else if (key.startsWith("hires")) hires[label] = value;
+ else if (key.startsWith("adetailer ")) {
+ adetailer[key] = value;
+ adetailerLabels[key] = label;
+ }
else if (key === "sampler") {
out.sampler = labelId(SAMPLER_LABELS, value);
// Before AUTOMATIC1111 1.9 the schedule rode on the sampler name: `DPM++ 2M Karras`.
@@ -849,6 +862,10 @@ export function parseA1111Parameters(text: string): ImportedRecipe | null {
if (hadHires) Object.assign(out, parseHiresInfotext(hires, typeof out.strength === "number" ? out.strength : undefined, { width: out.width, height: out.height }));
const hiresKeys = Object.keys(hires);
if (hiresKeys.length > 0) out.hires = hires;
+ const detailer = detailerFromInfotext(adetailer);
+ if (detailer) out.detailer = detailer;
+ for (const key of detailerInfotextNotApplied(adetailer))
+ notApplied.push(`${adetailerLabels[key]}: ${adetailer[key]}`);
// txt2img hires fix also writes a denoising strength; without it the strength means img2img,
// or inpaint when A1111 wrote its mask-only fields.
if (out.strength !== undefined) {
@@ -941,6 +958,7 @@ export interface RestorableRecipe {
mask_blur?: number;
inpaint_only_masked?: boolean;
inpaint_padding?: number;
+ detailer?: DetailerRecord | null;
upscale_model?: string | null;
hires_scale?: number;
hires_denoise?: number;
@@ -982,6 +1000,7 @@ export const APPLIED_RECIPE_KEYS = [
"mask_blur",
"inpaint_only_masked",
"inpaint_padding",
+ "detailer",
"upscale_model",
"hires_scale",
"hires_denoise",
@@ -1297,6 +1316,7 @@ export function recipeSettings(
? r.inpaint_only_masked
: undefined,
inpaint_padding: bounded(r, "inpaint_padding"),
+ detailer: detailerRecordOf(r.detailer),
...hiresRecipeFields(r),
};
}
diff --git a/studio/frontend/src/features/images/lib/mask-from-text.ts b/studio/frontend/src/features/images/lib/mask-from-text.ts
index 95a2ce4aaee..223f2b1ee7f 100644
--- a/studio/frontend/src/features/images/lib/mask-from-text.ts
+++ b/studio/frontend/src/features/images/lib/mask-from-text.ts
@@ -14,6 +14,10 @@ export const MASK_SEGMENT_ROUTES = {
segment: "/api/inference/images/segment",
} as const;
+/** Window event carrying every segmenter status read (detail: SegmenterStatus), so a second reader (the Detailer
+ * control) sees a download finish in the Mask from text panel without polling. */
+export const SEGMENTER_STATUS_EVENT = "unsloth:segmenter-status";
+
export const MASK_FROM_TEXT_LABEL = "Mask from text";
export const MASK_FROM_TEXT_PLACEHOLDER = "sofa, red lamp, pillow:2";
export const MASK_FROM_TEXT_HINT =
diff --git a/studio/frontend/src/features/images/mask-segment-api.ts b/studio/frontend/src/features/images/mask-segment-api.ts
index e546211ddac..ee541f7f8fb 100644
--- a/studio/frontend/src/features/images/mask-segment-api.ts
+++ b/studio/frontend/src/features/images/mask-segment-api.ts
@@ -10,6 +10,7 @@ import { readFastApiError } from "@/lib/format-fastapi-error";
import {
MASK_SEGMENT_ROUTES,
parseSegmenterStatus,
+ SEGMENTER_STATUS_EVENT,
parseSegmentResult,
type SegmenterStatus,
type SegmentRequest,
@@ -24,9 +25,13 @@ async function parseJson<T>(response: Response): Promise<T> {
}
export async function getSegmenterStatus(signal?: AbortSignal): Promise<SegmenterStatus> {
- return parseSegmenterStatus(
+ const status = parseSegmenterStatus(
await parseJson<unknown>(await authFetch(MASK_SEGMENT_ROUTES.status, { signal })),
);
+ if (typeof window !== "undefined") {
+ window.dispatchEvent(new CustomEvent(SEGMENTER_STATUS_EVENT, { detail: status }));
+ }
+ return status;
}
/** Start the background download of the active segmenter variant. Progress is read back from the status route. */
diff --git a/studio/frontend/tests/image-paste-parameters.test.ts b/studio/frontend/tests/image-paste-parameters.test.ts
index cc03cf16207..634dbb767e7 100644
--- a/studio/frontend/tests/image-paste-parameters.test.ts
+++ b/studio/frontend/tests/image-paste-parameters.test.ts
@@ -107,8 +107,9 @@ test("Clip skip, CFG rescale and variation seed are named as not applied; hires
"CFG rescale 0.7",
"variation seed 99 (strength 0.3)",
"VAE: sdxl_vae.safetensors",
- "ADetailer model: face_yolov8n.pt",
]);
+ // ADetailer's keys restore Studio's Detailer instead of being named as not applied.
+ assert.equal((r.detailer as { model?: string } | undefined)?.model, "face_yolov8n.pt");
assert.deepEqual(unknownRecipeKeys(r), []);
for (const label of [
"RescaleCFG: 0.5",
diff --git a/studio/frontend/tests/image-restore-negative-prompt.test.ts b/studio/frontend/tests/image-restore-negative-prompt.test.ts
index f2bece39bdf..6a55fdbee05 100644
--- a/studio/frontend/tests/image-restore-negative-prompt.test.ts
+++ b/studio/frontend/tests/image-restore-negative-prompt.test.ts
@@ -7,6 +7,7 @@ import ts from "@typescript/typescript6";
import { readSrc } from "./helpers/kit.ts";
import { enhanceRestoreFor } from "../src/features/images/lib/prompt-enhance.ts";
import { restoredInpaintMaskOptions } from "../src/features/images/lib/inpaint-options.ts";
+import { detailerNotApplied, restoredDetailer } from "../src/features/images/lib/detailer.ts";
function restoreSettingsWith(negativeCapable: boolean) {
const source = readSrc("features/images/images-page.tsx");
@@ -51,6 +52,8 @@ function restoreSettingsWith(negativeCapable: boolean) {
// Its return value is read, so it cannot be a recording no-op.
enhanceRestoreFor,
restoredInpaintMaskOptions,
+ restoredDetailer,
+ detailerNotApplied,
toast: { success: () => {} },
};
// Every closure name the callback reads that is not a real global resolves to a recording no-op,
diff --git a/studio/frontend/tests/images-detailer.test.ts b/studio/frontend/tests/images-detailer.test.ts
new file mode 100644
index 00000000000..f58f8473133
--- /dev/null
+++ b/studio/frontend/tests/images-detailer.test.ts
@@ -0,0 +1,334 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+import {
+ DETAILER_DEFAULTS,
+ detailerAvailable,
+ detailerEngagesNegative,
+ detailerErrors,
+ detailerFromInfotext,
+ detailerModelCapable,
+ detailerNotApplied,
+ detailerRecordOf,
+ detailerRequestField,
+ detailerSummary,
+ readDetailerOptions,
+ restoredDetailer,
+} from "../src/features/images/lib/detailer.ts";
+import {
+ APPLIED_RECIPE_KEYS,
+ importNotApplied,
+ parseA1111Parameters,
+ recipeSettings,
+} from "../src/features/images/lib/image-metadata.ts";
+
+const inpaintModel = { loaded: true, workflows: ["txt2img", "img2img", "inpaint", "outpaint"] };
+const ready = { supported: true, downloaded: true };
+
+test("the Detailer shows only for a model with an inpaint path and a downloaded SAM 3", () => {
+ assert.equal(detailerAvailable(inpaintModel, false, ready), true);
+ // No inpaint pipeline (FLUX.2 dev, edit models, sd.cpp): hidden.
+ assert.equal(detailerAvailable({ loaded: true, workflows: ["txt2img", "reference"] }, false, ready), false);
+ assert.equal(detailerAvailable({ loaded: true, workflows: ["edit"] }, false, ready), false);
+ // Nothing loaded, or a Connection (external provider): hidden.
+ assert.equal(detailerAvailable({ loaded: false, workflows: ["inpaint"] }, false, ready), false);
+ assert.equal(detailerAvailable(inpaintModel, true, ready), false);
+ assert.equal(detailerAvailable(null, false, ready), false);
+ // SAM 3 unsupported (transformers 4) or not downloaded, or its status not read yet: hidden.
+ assert.equal(detailerAvailable(inpaintModel, false, { supported: false, downloaded: false }), false);
+ assert.equal(detailerAvailable(inpaintModel, false, { supported: true, downloaded: false }), false);
+ assert.equal(detailerAvailable(inpaintModel, false, null), false);
+ // The segmenter status is only fetched for a capable model.
+ assert.equal(detailerModelCapable(inpaintModel, false), true);
+ assert.equal(detailerModelCapable({ loaded: true, workflows: ["txt2img"] }, false), false);
+});
+
+test("off by default, and nothing is sent unless on and available", () => {
+ assert.deepEqual(detailerRequestField(false, true, DETAILER_DEFAULTS), {});
+ assert.deepEqual(detailerRequestField(true, false, DETAILER_DEFAULTS), {});
+ // Defaults send only the targets: the backend holds the same FaceDetailer defaults.
+ assert.deepEqual(detailerRequestField(true, true, DETAILER_DEFAULTS), { detailer: { targets: "face" } });
+ // Node has no localStorage: the stored options read as the defaults.
+ assert.deepEqual(readDetailerOptions(), DETAILER_DEFAULTS);
+});
+
+test("changed options are sent, bounded to the request schema", () => {
+ const field = detailerRequestField(true, true, {
+ ...DETAILER_DEFAULTS,
+ targets: " ",
+ prompt: " detailed face ",
+ denoise: 0.35,
+ cropFactor: 99,
+ guideSize: 768,
+ feather: 5,
+ steps: 12.4,
+ });
+ assert.deepEqual(field, {
+ detailer: { targets: "face", prompt: "detailed face", denoise: 0.35, crop_factor: 10, guide_size: 768, steps: 12 },
+ });
+});
+
+test("a recipe's non-default guide_size_for and drop_size survive restore and are sent again", () => {
+ const restored = restoredDetailer({ detailer: { ...record, guide_size_for: "crop_region", drop_size: 32 } });
+ assert.equal(restored.options.guideSizeFor, "crop_region");
+ assert.equal(restored.options.dropSize, 32);
+ const field = detailerRequestField(true, true, { ...DETAILER_DEFAULTS, ...restored.options });
+ assert.equal(field.detailer?.guide_size_for, "crop_region");
+ assert.equal(field.detailer?.drop_size, 32);
+ // A record without them restores FaceDetailer's defaults, never a stale stored value.
+ const plain = restoredDetailer({ detailer: { targets: "face" } });
+ assert.deepEqual([plain.options.guideSizeFor, plain.options.dropSize], ["bbox", 10]);
+});
+
+test("the frontend defaults are the backend's", () => {
+ const source = readFileSync(
+ new URL("../../backend/core/inference/diffusion_detailer.py", import.meta.url),
+ "utf8",
+ );
+ const constant = (name: string) => Number(new RegExp(`^${name} = ([\\d.]+)`, "m").exec(source)?.[1]);
+ assert.equal(constant("DEFAULT_DENOISE"), DETAILER_DEFAULTS.denoise);
+ assert.equal(constant("DEFAULT_CONFIDENCE"), DETAILER_DEFAULTS.confidence);
+ assert.equal(constant("DEFAULT_CROP_FACTOR"), DETAILER_DEFAULTS.cropFactor);
+ assert.equal(constant("DEFAULT_GUIDE_SIZE"), DETAILER_DEFAULTS.guideSize);
+ assert.equal(constant("DEFAULT_MAX_SIZE"), DETAILER_DEFAULTS.maxSize);
+ assert.equal(constant("DEFAULT_DILATION"), DETAILER_DEFAULTS.dilation);
+ assert.equal(constant("DEFAULT_FEATHER"), DETAILER_DEFAULTS.feather);
+ assert.equal(constant("DEFAULT_DROP_SIZE"), DETAILER_DEFAULTS.dropSize);
+ assert.match(source, new RegExp(`^DEFAULT_GUIDE_SIZE_FOR = "${DETAILER_DEFAULTS.guideSizeFor}"`, "m"));
+});
+
+const record = {
+ targets: "face, hand:2",
+ prompt: "sharp eyes",
+ confidence: 0.5,
+ denoise: 0.4,
+ crop_factor: 2.5,
+ guide_size: 512,
+ guide_size_for: "bbox",
+ max_size: 1024,
+ dilation: 10,
+ feather: 5,
+ drop_size: 10,
+ steps: 12,
+ model: "sam3.1",
+ detections: [{ target: "face", score: 0.91, box: [1, 2, 3, 4], seed: 7 }],
+ found: 1,
+};
+
+test("a gallery record restores the pass and turns the switch on; one without turns it off", () => {
+ const on = restoredDetailer({ detailer: record });
+ assert.equal(on.enabled, true);
+ assert.deepEqual(on.options, {
+ targets: "face, hand:2",
+ prompt: "sharp eyes",
+ denoise: 0.4,
+ confidence: 0.5,
+ cropFactor: 2.5,
+ guideSize: 512,
+ maxSize: 1024,
+ dilation: 10,
+ feather: 5,
+ dropSize: 10,
+ guideSizeFor: "bbox",
+ steps: 12,
+ negativePrompt: "",
+ guidance: null,
+ });
+ assert.deepEqual(restoredDetailer({}), { enabled: false, options: {} });
+ assert.deepEqual(restoredDetailer({ detailer: "nonsense" }), { enabled: false, options: {} });
+ assert.deepEqual(detailerRecordOf({ denoise: "0.4", targets: 3, steps: 9 }), { steps: 9 });
+});
+
+test("the Unsloth recipe carries the record through import", () => {
+ const settings = recipeSettings(
+ { prompt: "p", detailer: record },
+ { width: 1024, height: 1024, steps: 9, guidance: 0 },
+ );
+ assert.equal(settings.detailer?.denoise, 0.4);
+ assert.ok((APPLIED_RECIPE_KEYS as readonly string[]).includes("detailer"));
+ assert.equal(recipeSettings({ prompt: "p" }, { width: 1024, height: 1024, steps: 9, guidance: 0 }).detailer, null);
+});
+
+// What core/inference/image_gallery._params_text writes with a pass (diffusion_detailer.infotext_fields).
+const STUDIO_INFOTEXT = [
+ "a portrait of two friends",
+ "Steps: 9, CFG scale: 0.0, Seed: 42, Size: 1024x1024, Model: Tongyi-MAI/Z-Image-Turbo, ADetailer model: sam3.1, " +
+ 'ADetailer model classes: face, ADetailer prompt: "detailed face, smile", ADetailer confidence: 0.5, ' +
+ "ADetailer dilate erode: 10, ADetailer mask blur: 5, ADetailer denoising strength: 0.4, " +
+ "ADetailer inpaint only masked: True, ADetailer use separate steps: True, ADetailer steps: 12, Version: Unsloth",
+].join("\n");
+
+test("Studio's own ADetailer infotext pastes back as the same pass", () => {
+ const parsed = parseA1111Parameters(STUDIO_INFOTEXT);
+ assert.ok(parsed);
+ assert.deepEqual(parsed.detailer, {
+ model: "sam3.1",
+ targets: "face",
+ prompt: "detailed face, smile",
+ confidence: 0.5,
+ dilation: 10,
+ feather: 5,
+ denoise: 0.4,
+ steps: 12,
+ });
+ // Not reported as unapplied, and the main image stays text-to-image (ADetailer's strength is not img2img's).
+ assert.equal(parsed.parameters_not_applied, undefined);
+ assert.equal(parsed.workflow, undefined);
+ assert.equal(parsed.strength, undefined);
+ assert.deepEqual(importNotApplied(parsed, "z-image", false), []);
+ const restored = restoredDetailer(recipeSettings(parsed, { width: 1024, height: 1024, steps: 9, guidance: 0 }));
+ assert.equal(restored.enabled, true);
+ assert.equal(restored.options.denoise, 0.4);
+ assert.equal(restored.options.steps, 12);
+});
+
+test("an AUTOMATIC1111 ADetailer infotext maps its YOLO model to a target", () => {
+ const parsed = parseA1111Parameters(
+ "girl\nSteps: 28, Sampler: DPM++ 2M, CFG scale: 7, Seed: 1, Size: 512x768, ADetailer model: face_yolov8n.pt, " +
+ "ADetailer confidence: 0.3, ADetailer dilate erode: 4, ADetailer mask blur: 4, ADetailer denoising strength: 0.4, " +
+ "ADetailer inpaint only masked: True, ADetailer inpaint padding: 32, ADetailer model 2nd: hand_yolov8n.pt, " +
+ "ADetailer version: 24.9.0",
+ );
+ assert.ok(parsed);
+ assert.deepEqual(parsed.detailer, {
+ model: "face_yolov8n.pt",
+ targets: "face",
+ confidence: 0.3,
+ dilation: 4,
+ feather: 4,
+ denoise: 0.4,
+ });
+ // Only the first tab restores: a later tab and the options Studio has no control for are named, not dropped.
+ assert.deepEqual(parsed.parameters_not_applied, [
+ "ADetailer inpaint padding: 32",
+ "ADetailer model 2nd: hand_yolov8n.pt",
+ ]);
+ assert.deepEqual(
+ parseA1111Parameters("a\nSteps: 20, Seed: 1, ADetailer model: None, ADetailer prompt: eyes")?.parameters_not_applied,
+ ["ADetailer prompt: eyes"],
+ );
+ assert.equal(detailerFromInfotext({ "adetailer model": "hand_yolov8s.pt" })?.targets, "hand");
+ assert.equal(detailerFromInfotext({ "adetailer model": "None" }), null);
+ assert.equal(detailerFromInfotext({}), null);
+});
+
+test("a restored or imported pass this load cannot run is named as not applied", () => {
+ assert.deepEqual(detailerNotApplied(true, false), ["Detailer (face fix)"]);
+ assert.deepEqual(detailerNotApplied(true, true), []);
+ assert.deepEqual(detailerNotApplied(false, false), []);
+ const page = readFileSync(new URL("../src/features/images/images-page.tsx", import.meta.url), "utf8");
+ // The gallery restore names it beside the SD controls, as an import does.
+ assert.match(page, /\.\.\.\(via === "restore" \? detailerNotApplied\(detail\.enabled, detailerReady\) : \[\]\)/);
+ assert.match(page, /notApplied\.unshift\(\.\.\.detailerNotApplied\(Boolean\(settings\.detailer\), detailerReady\)\)/);
+});
+
+test("errors are surfaced, a cancel is not", () => {
+ assert.deepEqual(
+ detailerErrors([{ detailer: { error: "cancelled" } }, { detailer: { error: "CUDA out of memory" } }, {}]),
+ ["CUDA out of memory"],
+ );
+ assert.equal(detailerSummary(record), "face, hand:2: 1 fixed, denoise 0.4");
+ assert.equal(detailerSummary({ error: "the model changed" }), "not applied (the model changed)");
+ assert.equal(detailerSummary(null), null);
+});
+
+test("the page wires the control behind the gate and keeps it out of the startup bundle", () => {
+ const page = readFileSync(new URL("../src/features/images/images-page.tsx", import.meta.url), "utf8");
+ assert.match(page, /<DetailerField\s+available=\{detailerReady\}/);
+ assert.match(page, /detailerRequestField\(detailerOn, detailerReady, detailerOptions\)/);
+ assert.match(page, /useState\(false\);\n\s*const \[detailerOptions/);
+ // The field is a separate module of the lazily loaded Images page; api.ts only takes its types.
+ const api = readFileSync(new URL("../src/features/images/api.ts", import.meta.url), "utf8");
+ assert.match(api, /^import type \{ DetailerRecord, DetailerRequest \} from "\.\/lib\/detailer";$/m);
+ const root = readFileSync(new URL("../src/app/routes/__root.tsx", import.meta.url), "utf8");
+ assert.match(root, /import\("@\/features\/images"\)/);
+});
+
+test("a recorded Detailer negative prompt and CFG restore and are sent again (ADetailer's separate ones)", () => {
+ const restored = restoredDetailer({ detailer: { ...record, negative_prompt: "blurry eyes", guidance: 4.5 } });
+ assert.equal(restored.options.negativePrompt, "blurry eyes");
+ assert.equal(restored.options.guidance, 4.5);
+ const field = detailerRequestField(true, true, { ...DETAILER_DEFAULTS, ...restored.options });
+ assert.equal(field.detailer?.negative_prompt, "blurry eyes");
+ assert.equal(field.detailer?.guidance, 4.5);
+ // From an AUTOMATIC1111 infotext: separate negative prompt and CFG scale.
+ const parsed = parseA1111Parameters(
+ "girl\nSteps: 28, CFG scale: 7, Seed: 1, Size: 512x768, ADetailer model: face_yolov8n.pt, " +
+ "ADetailer negative prompt: blurry, ADetailer denoising strength: 0.4, " +
+ "ADetailer use separate CFG scale: True, ADetailer CFG scale: 5.5",
+ );
+ assert.ok(parsed);
+ const fromPaste = restoredDetailer(recipeSettings(parsed, { width: 512, height: 768, steps: 28, guidance: 7 }));
+ const sent = detailerRequestField(true, true, { ...DETAILER_DEFAULTS, ...fromPaste.options }).detailer;
+ assert.equal(sent?.negative_prompt, "blurry");
+ assert.equal(sent?.guidance, 5.5);
+ // A record without them clears an override left from an earlier restore.
+ const plain = restoredDetailer({ detailer: record });
+ assert.equal(plain.options.negativePrompt, "");
+ assert.equal(plain.options.guidance, null);
+ assert.equal(detailerRequestField(true, true, DETAILER_DEFAULTS).detailer?.guidance, undefined);
+});
+
+test("the Detailer control learns about a finished SAM 3 download without a reload", () => {
+ const api = readFileSync(new URL("../src/features/images/mask-segment-api.ts", import.meta.url), "utf8");
+ // Every status read (the Mask from text panel polls while it downloads) is broadcast...
+ assert.match(api, /dispatchEvent\(new CustomEvent\(SEGMENTER_STATUS_EVENT, \{ detail: status \}\)\)/);
+ const field = readFileSync(new URL("../src/features/images/detailer-field.tsx", import.meta.url), "utf8");
+ // ...the Detailer's hook takes it, and polls on its own while a download runs.
+ assert.match(field, /addEventListener\(SEGMENTER_STATUS_EVENT, onStatus\)/);
+ assert.match(field, /status\?\.download\.downloading === true/);
+ assert.match(field, /setInterval\(read, 2000\)/);
+});
+
+test("an imported default ADetailer run does not keep geometry stored from an earlier run", () => {
+ // The page merges a restore over the stored options; an infotext carries no crop factor, guide size or max size.
+ const page = readFileSync(new URL("../src/features/images/images-page.tsx", import.meta.url), "utf8");
+ assert.match(page, /setDetailerOptions\(\{ \.\.\.readDetailerOptions\(\), \.\.\.detail\.options \}\)/);
+ const stored = { ...DETAILER_DEFAULTS, cropFactor: 6, guideSize: 1536, maxSize: 2048, guideSizeFor: "crop_region" as const };
+ const parsed = parseA1111Parameters(STUDIO_INFOTEXT);
+ assert.ok(parsed);
+ const restored = restoredDetailer(recipeSettings(parsed, { width: 1024, height: 1024, steps: 9, guidance: 0 }));
+ const sent = detailerRequestField(true, true, { ...stored, ...restored.options }).detailer;
+ assert.ok(sent);
+ assert.equal(sent.crop_factor, undefined);
+ assert.equal(sent.guide_size, undefined);
+ assert.equal(sent.max_size, undefined);
+ assert.equal(sent.guide_size_for, undefined);
+ assert.equal(sent.denoise, 0.4);
+ assert.equal(sent.steps, 12);
+ const custom = restoredDetailer({ detailer: { targets: "face", crop_factor: 4, guide_size: 768, max_size: 1536 } });
+ const resent = detailerRequestField(true, true, { ...DETAILER_DEFAULTS, ...custom.options }).detailer;
+ assert.equal(resent?.crop_factor, 4);
+ assert.equal(resent?.guide_size, 768);
+ assert.equal(resent?.max_size, 1536);
+});
+
+test("the Detailer's own CFG sends the main negative prompt it inherits", () => {
+ // Base CFG 0 leaves the negative prompt unused; a Detailer CFG of 5 with no separate negative reads it.
+ const cfg = { ...DETAILER_DEFAULTS, guidance: 5 };
+ assert.equal(detailerEngagesNegative(true, true, cfg, 1), true);
+ assert.equal(detailerEngagesNegative(true, true, cfg, null), true);
+ // Not when off, unavailable, using the image's CFG, at or under the threshold, or with its own negative prompt.
+ assert.equal(detailerEngagesNegative(false, true, cfg, 1), false);
+ assert.equal(detailerEngagesNegative(true, false, cfg, 1), false);
+ assert.equal(detailerEngagesNegative(true, true, DETAILER_DEFAULTS, 1), false);
+ assert.equal(detailerEngagesNegative(true, true, { ...cfg, guidance: 1 }, 1), false);
+ assert.equal(detailerEngagesNegative(true, true, { ...cfg, negativePrompt: "blurry" }, 1), false);
+ // A separate negative template that reads the image's own ([PROMPT], an empty [SEP] part) still needs it sent.
+ assert.equal(detailerEngagesNegative(true, true, { ...cfg, negativePrompt: "[PROMPT], bad teeth" }, 1), true);
+ assert.equal(detailerEngagesNegative(true, true, { ...cfg, negativePrompt: " [SEP] bad teeth" }, 1), true);
+ assert.equal(detailerEngagesNegative(true, true, { ...cfg, negativePrompt: "bad teeth [SEP] [SKIP]" }, 1), false);
+ // The page's gate reads it, after the Detailer state it depends on.
+ const page = readFileSync(new URL("../src/features/images/images-page.tsx", import.meta.url), "utf8");
+ const gate = page.indexOf("const negativeEngaged =");
+ assert.ok(gate > page.indexOf("const detailerReady ="));
+ assert.match(
+ page.slice(gate, gate + 600),
+ /detailerEngagesNegative\(detailerOn, detailerReady, detailerOptions, negativeMinGuidance\)/,
+ );
+ assert.match(page, /negative_prompt: negativeEngaged \? negativePrompt\.trim\(\) \|\| undefined : undefined/);
+});