Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio Images: SDXL prompt weighting, CLIP skip, CFG rescale and variation seeds (#14562)

Daniel Hand277aa6e94ec
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index be22f177fa2..1ca3bfa2ffa 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -26,6 +26,7 @@ import inspect
import json
import os
import re
+import secrets
import sys
import threading
import time
@@ -358,6 +359,7 @@ from .diffusion_sampler import (
build_scheduler,
sampler_options,
)
+from . import diffusion_sd_controls as sd_controls_mod
from .diffusion_single_file_converters import (
CONVERTERS as _ORIGINAL_LAYOUT_CONVERTERS,
load_original_layout_transformer,
@@ -12315,6 +12317,7 @@ class DiffusionBackend:
live_preview: Optional[bool] = None,
sampler: Optional[str] = None,
schedule_type: Optional[str] = None,
+ sd_controls: Optional["sd_controls_mod.SdControls"] = None,
shift: Optional[float] = None,
base_shift: Optional[float] = None,
max_shift: Optional[float] = None,
@@ -12379,6 +12382,17 @@ class DiffusionBackend:
batch_size = batch_size,
draw_seed = torch.Generator(device = state.device).seed,
)
+ # A batch keeps ONE base seed so it renders variations of a single image.
+ variation_seed_list: Optional[list[int]] = None
+ if sd_controls is not None and sd_controls.variation_strength:
+ if seeds is None and prompts is None:
+ jobs = [(p, seed) for p, _ in jobs]
+ base_variation = (
+ int(sd_controls.variation_seed)
+ if sd_controls.variation_seed is not None
+ else secrets.randbelow(2**32)
+ )
+ variation_seed_list = sd_controls_mod.variation_seeds(base_variation, len(jobs))
# Deferred speed auto: engage the compile profile on the 3rd image. Best-effort; a failure stays eager
# and never retries. NOT when a LoRA is requested: a compiled transformer rejects LoRA, breaking every
@@ -12639,6 +12653,7 @@ class DiffusionBackend:
pipe.scheduler = swapped
# Pipelines vary in accepted kwargs, so gate every optional one on the signature.
call_params = inspect.signature(pipe.__call__).parameters
+ sd_controls_mod.validate(sd_controls, fam, pipe, call_params, workflow)
mask_blur_applied = inpaint_padding_applied = None
soft_overlay = None
if workflow == "inpaint" and mask_pil is not None:
@@ -12996,6 +13011,22 @@ class DiffusionBackend:
if "callback_on_step_end" in call_params:
kwargs["callback_on_step_end"] = _on_step
+ kwargs.update(sd_controls_mod.guidance_rescale_kwargs(sd_controls, call_params))
+ weighting_used = [False]
+
+ def _render_chunk(
+ chunk_kwargs: dict[str, Any], chunk_variation: Optional[list[int]]
+ ):
+ if sd_controls_mod.apply_prompt_controls(
+ pipe, fam, sd_controls, chunk_kwargs, guidance = guidance
+ ):
+ weighting_used[0] = True
+ with sd_controls_mod.variation_latents(
+ pipe,
+ chunk_variation,
+ sd_controls.variation_strength if sd_controls is not None else None,
+ ):
+ return pipe(**chunk_kwargs).images
# EFFECTIVE steps: img2img at strength < 1 denoises a fraction of `steps`.
strength_applied = effective_request_strength(
@@ -13146,8 +13177,16 @@ class DiffusionBackend:
decode_phase(pipe, _enter_decode_phase),
_completed_step_poller(_pump),
):
+ chunk_variation = (
+ variation_seed_list[
+ len(per_image_seeds) : len(per_image_seeds) + len(chunk)
+ ]
+ if variation_seed_list
+ else None
+ )
out = render_thread.run(
- "diffusion", lambda: pipe(**chunk_kwargs).images
+ "diffusion",
+ lambda: _render_chunk(chunk_kwargs, chunk_variation),
)
if gen.phase in ("encode", "denoise"):
_flip_to_decode()
@@ -13314,6 +13353,11 @@ class DiffusionBackend:
if swapped is not None and sampler != DEFAULT_SAMPLER
else None,
"schedule_type": resolved_schedule,
+ **sd_controls_mod.record_fields(
+ sd_controls,
+ weighting_used = weighting_used[0],
+ variation_seed_list = variation_seed_list,
+ ),
# Recorded only when applied, so a plain inpaint recipe stays unchanged.
"mask_blur": mask_blur_applied,
"inpaint_only_masked": True if inpaint_padding_applied is not None else None,
@@ -13596,6 +13640,7 @@ class DiffusionBackend:
"attention_backend": None,
"transformer_cache": None,
"transformer_cache_stats": None,
+ "baked_loras": [],
"workflows": [],
"conditioning": None,
"supports_lora": False,
@@ -13652,6 +13697,7 @@ class DiffusionBackend:
"attention_backend": state.attention_backend,
"transformer_cache": state.transformer_cache,
"transformer_cache_stats": static_skip_stats(state.pipe),
+ "baked_loras": _baked_lora_names(state.pipe),
"multi_gpu": multigpu_placement_status(state.pipe),
"cfg_batch": cfg_batch_status(state.pipe),
"zimage_cfg_split": zimage_cfg_split_status(state.pipe),
@@ -13674,6 +13720,7 @@ class DiffusionBackend:
"supports_live_preview": _live_preview_available(state),
"prompt_hint": state.family.prompt_hint,
"samplers": sampler_options(getattr(state.pipe, "scheduler", None)),
+ "sd_controls": sd_controls_mod.status_fields(state.family, state.pipe),
# Hidden for a checkpoint that ships its own sampling grid (the load-time ComfyUI shift skips it too).
"shift_control": None
if pipe_sample_sigmas(state.pipe) is not None
diff --git a/studio/backend/core/inference/diffusion_prompt_weighting.py b/studio/backend/core/inference/diffusion_prompt_weighting.py
new file mode 100644
index 00000000000..8484eeb8a2c
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_prompt_weighting.py
@@ -0,0 +1,388 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""AUTOMATIC1111 prompt weighting for the CLIP-conditioned (SD 1.x / SDXL) diffusers pipelines.
+
+Follows stable-diffusion-webui (commit 82a973c) prompt_parser / sd_hijack_clip / sd_emphasis, rewritten not copied.
+Layer selection is diffusers' own, so at weight 1 a single-chunk prompt matches the pipeline bit for bit.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Optional, Sequence
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+_ATTENTION_RE = re.compile(
+ r"""
+\\\(|
+\\\)|
+\\\[|
+\\]|
+\\\\|
+\\|
+\(|
+\[|
+:\s*([+-]?[.\d]+)\s*\)|
+\)|
+]|
+[^\\()\[\]:]+|
+:
+""",
+ re.X,
+)
+_BREAK_RE = re.compile(r"\s*\bBREAK\b\s*", re.S)
+
+ROUND_MULTIPLIER = 1.1
+SQUARE_MULTIPLIER = 1 / 1.1
+BREAK_WEIGHT = -1.0
+CHUNK_TOKENS = 75
+COMMA_BACKTRACK = 20
+# webui and ComfyUI chunk without a limit; 32 chunks (2400 tokens) bounds what one request can encode.
+MAX_CHUNKS = 32
+# Text read per prompt (32 chars/token over the cap) bounds parsing however large the body is.
+MAX_PROMPT_CHARS = MAX_CHUNKS * CHUNK_TOKENS * 32
+
+
+def parse_prompt_attention(text: str) -> list[list[Any]]:
+ """``[[fragment, weight], ...]`` with A1111 semantics; a BREAK is ``["BREAK", -1.0]``."""
+ res: list[list[Any]] = []
+ round_open: list[int] = []
+ square_open: list[int] = []
+
+ def multiply(start: int, factor: float) -> None:
+ for pos in range(start, len(res)):
+ res[pos][1] *= factor
+
+ for match in _ATTENTION_RE.finditer(text):
+ token = match.group(0)
+ weight = match.group(1)
+ if token.startswith("\\"):
+ res.append([token[1:], 1.0])
+ elif token == "(":
+ round_open.append(len(res))
+ elif token == "[":
+ square_open.append(len(res))
+ elif weight is not None and round_open:
+ try:
+ value = float(weight)
+ except ValueError:
+ # A1111 raises on a malformed number such as "(a:1.2.3)"; keep the text as typed instead.
+ start = round_open.pop()
+ res.insert(start, ["(", 1.0])
+ square_open[:] = [pos + 1 if pos >= start else pos for pos in square_open]
+ res.append([token, 1.0])
+ else:
+ multiply(round_open.pop(), value)
+ elif token == ")" and round_open:
+ multiply(round_open.pop(), ROUND_MULTIPLIER)
+ elif token == "]" and square_open:
+ multiply(square_open.pop(), SQUARE_MULTIPLIER)
+ else:
+ for i, part in enumerate(_BREAK_RE.split(token)):
+ if i > 0:
+ res.append(["BREAK", BREAK_WEIGHT])
+ res.append([part, 1.0])
+ for pos in round_open:
+ multiply(pos, ROUND_MULTIPLIER)
+ for pos in square_open:
+ multiply(pos, SQUARE_MULTIPLIER)
+ if not res:
+ res = [["", 1.0]]
+ i = 0
+ while i + 1 < len(res):
+ if res[i][1] == res[i + 1][1]:
+ res[i][0] += res[i + 1][0]
+ res.pop(i + 1)
+ else:
+ i += 1
+ return res
+
+
+def has_prompt_syntax(text: Optional[str]) -> bool:
+ """True when parsing changes ``text``: any bracket, escape or BREAK. Plain prompts keep the pipeline's own path."""
+ if not text:
+ return False
+ text = text[:MAX_PROMPT_CHARS]
+ parsed = parse_prompt_attention(text)
+ return not (len(parsed) == 1 and parsed[0][0] == text and parsed[0][1] == 1.0)
+
+
+def exceeds_one_chunk(pipe: Any, text: Optional[str]) -> bool:
+ """True when ``text`` holds more than 75 CLIP tokens in any of the pipeline's tokenizers, i.e. when
+ ``encode_prompt`` would truncate it (77 = ``<bos>`` + 75 + ``<eos>``). Counted after the pipeline's own
+ ``maybe_convert_prompt``, as ``encode_prompt`` tokenizes."""
+ if not text:
+ return False
+ text = text[:MAX_PROMPT_CHARS]
+ convert = getattr(pipe, "maybe_convert_prompt", None)
+ for tokenizer, _ in _encoders(pipe):
+ converted = convert(text, tokenizer) if convert is not None else text
+ warned = getattr(tokenizer, "deprecation_warnings", None)
+ if isinstance(warned, dict):
+ warned["sequence-length-is-longer-than-the-specified-maximum"] = True
+ ids = tokenizer(converted, truncation = False, add_special_tokens = False, verbose = False)[
+ "input_ids"
+ ]
+ if len(ids) > CHUNK_TOKENS:
+ return True
+ return False
+
+
+def _special_ids(tokenizer: Any) -> tuple[int, int, int, Optional[int]]:
+ bos = tokenizer.bos_token_id
+ eos = tokenizer.eos_token_id
+ pad = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else eos
+ comma = tokenizer.get_vocab().get(",</w>")
+ return int(bos), int(eos), int(pad), comma
+
+
+def tokenize_chunks(
+ text: str,
+ tokenizer: Any,
+ *,
+ weighted: bool = True,
+ convert: Any = None,
+) -> list[tuple[list[int], list[float]]]:
+ """A1111 ``tokenize_line``: ``[(77 token ids, 77 weights), ...]``; ``weighted=False`` still chunks.
+ Stops at ``MAX_CHUNKS``: nothing past the cap is tokenized or kept."""
+ truncated = len(text) > MAX_PROMPT_CHARS
+ text = text[:MAX_PROMPT_CHARS]
+ parsed = parse_prompt_attention(text) if weighted else [[text, 1.0]]
+ bos, eos, pad, comma = _special_ids(tokenizer)
+ warned = getattr(tokenizer, "deprecation_warnings", None)
+ if isinstance(warned, dict):
+ warned["sequence-length-is-longer-than-the-specified-maximum"] = True
+ chunks: list[tuple[list[int], list[float]]] = []
+ tokens: list[int] = []
+ weights: list[float] = []
+ last_comma = -1
+
+ class _Full(Exception):
+ pass
+
+ def close_chunk() -> None:
+ nonlocal tokens, weights, last_comma
+ if len(chunks) >= MAX_CHUNKS:
+ raise _Full
+ fill = CHUNK_TOKENS - len(tokens)
+ ids = [bos] + tokens + [eos] * fill + [eos]
+ mults = [1.0] + weights + [1.0] * fill + [1.0]
+ # webui process_tokens: everything after the first <eos> is the pad id (OpenCLIP's differs from <eos>).
+ first_eos = ids.index(eos, 1)
+ ids = ids[: first_eos + 1] + [pad] * (len(ids) - first_eos - 1)
+ chunks.append((ids, mults))
+ tokens, weights, last_comma = [], [], -1
+
+ try:
+ for fragment, weight in parsed:
+ if fragment == "BREAK" and weight == BREAK_WEIGHT:
+ close_chunk()
+ continue
+ if convert is not None and fragment:
+ fragment = convert(fragment, tokenizer)
+ ids = tokenizer(fragment, truncation = False, add_special_tokens = False, verbose = False)[
+ "input_ids"
+ ]
+ for token in ids:
+ if token == comma:
+ last_comma = len(tokens)
+ elif (
+ COMMA_BACKTRACK
+ and len(tokens) == CHUNK_TOKENS
+ and last_comma != -1
+ and len(tokens) - last_comma <= COMMA_BACKTRACK
+ ):
+ cut = last_comma + 1
+ carry_tokens, carry_weights = tokens[cut:], weights[cut:]
+ tokens, weights = tokens[:cut], weights[:cut]
+ close_chunk()
+ tokens, weights = carry_tokens, carry_weights
+ if len(tokens) == CHUNK_TOKENS:
+ close_chunk()
+ if len(chunks) >= MAX_CHUNKS:
+ raise _Full
+ tokens.append(int(token))
+ weights.append(float(weight))
+ if tokens or not chunks:
+ close_chunk()
+ except _Full:
+ truncated = True
+ if truncated:
+ logger.warning(
+ "Prompt spans more than %d chunks of %d tokens; encoding the first %d and dropping the rest.",
+ MAX_CHUNKS,
+ CHUNK_TOKENS,
+ MAX_CHUNKS,
+ )
+ return chunks
+
+
+def _empty_chunk(tokenizer: Any) -> tuple[list[int], list[float]]:
+ bos, eos, pad, _ = _special_ids(tokenizer)
+ return [bos, eos] + [pad] * CHUNK_TOKENS, [1.0] * (CHUNK_TOKENS + 2)
+
+
+def apply_emphasis(z: Any, weights: Any) -> Any:
+ """webui ``EmphasisOriginal.after_transformers``: scale per token, then restore the chunk's mean."""
+ original_mean = z.mean()
+ z = z * weights.reshape(weights.shape + (1,)).expand(z.shape)
+ new_mean = z.mean()
+ return z * (original_mean / new_mean)
+
+
+def _encoders(pipe: Any) -> list[tuple[Any, Any]]:
+ pairs = []
+ for tok_attr, enc_attr in (("tokenizer", "text_encoder"), ("tokenizer_2", "text_encoder_2")):
+ tokenizer = getattr(pipe, tok_attr, None)
+ encoder = getattr(pipe, enc_attr, None)
+ if tokenizer is not None and encoder is not None:
+ pairs.append((tokenizer, encoder))
+ return pairs
+
+
+def is_sdxl_like(pipe: Any) -> bool:
+ """Two CLIP encoders whose penultimate layers concatenate (SDXL); else the SD 1.x single-encoder layout."""
+ return getattr(pipe, "text_encoder_2", None) is not None
+
+
+def _encode_chunk(
+ pipe: Any, encoder: Any, ids: list[int], clip_skip: Optional[int], device: Any, sdxl: bool
+) -> tuple[Any, Any]:
+ """One 77-token chunk through ``encoder``, picking the layer exactly as the pipeline's ``encode_prompt`` does."""
+ import torch
+
+ input_ids = torch.tensor([ids], dtype = torch.long).to(device)
+ if sdxl:
+ out = encoder(input_ids, output_hidden_states = True)
+ pooled = out[0] if out[0].ndim == 2 else None
+ if clip_skip is None:
+ z = out.hidden_states[-2]
+ else:
+ z = out.hidden_states[-(clip_skip + 2)]
+ return z, pooled
+ mask = None
+ if getattr(getattr(encoder, "config", None), "use_attention_mask", False):
+ mask = torch.ones_like(input_ids)
+ if clip_skip is None:
+ z = encoder(input_ids, attention_mask = mask)[0]
+ else:
+ out = encoder(input_ids, attention_mask = mask, output_hidden_states = True)
+ # transformers 5 flattened CLIPTextModel (no ``.text_model``); 4.x nests it.
+ final_norm = getattr(encoder, "text_model", encoder).final_layer_norm
+ z = final_norm(out[-1][-(clip_skip + 1)])
+ return z, None
+
+
+def _encode_texts(
+ pipe: Any,
+ texts: Sequence[str],
+ *,
+ weighted: bool,
+ clip_skip: Optional[int],
+ device: Any,
+ min_chunks: int = 1,
+) -> tuple[Any, Any, int]:
+ """``(embeds [B, 77k, D], pooled [B, P] or None, k)`` for ``texts``, every row padded to the longest with
+ empty-prompt chunks (webui ``process_texts`` / ``empty_chunk``)."""
+ import torch
+
+ sdxl = is_sdxl_like(pipe)
+ convert = getattr(pipe, "maybe_convert_prompt", None)
+ pairs = _encoders(pipe)
+ per_encoder = [
+ [tokenize_chunks(t, tok, weighted = weighted, convert = convert) for t in texts]
+ for tok, _ in pairs
+ ]
+ count = max([min_chunks] + [len(chunks) for rows in per_encoder for chunks in rows])
+ rows_out = []
+ pooled_rows = []
+ for row in range(len(texts)):
+ enc_parts = []
+ pooled = None
+ for (tokenizer, encoder), rows in zip(pairs, per_encoder):
+ chunks = list(rows[row]) + [_empty_chunk(tokenizer)] * (count - len(rows[row]))
+ zs = []
+ for index, (ids, mults) in enumerate(chunks):
+ z, chunk_pooled = _encode_chunk(pipe, encoder, ids, clip_skip, device, sdxl)
+ if index == 0 and pooled is None and chunk_pooled is not None:
+ pooled = chunk_pooled
+ weights = torch.tensor(mults, dtype = torch.float32, device = z.device)
+ if bool((weights != 1.0).any()):
+ z = apply_emphasis(z.float(), weights.unsqueeze(0)).to(z.dtype)
+ zs.append(z)
+ enc_parts.append(torch.cat(zs, dim = 1))
+ rows_out.append(torch.cat(enc_parts, dim = -1) if sdxl else enc_parts[0])
+ pooled_rows.append(pooled)
+ embeds = torch.cat(rows_out, dim = 0)
+ pooled_out = torch.cat(pooled_rows, dim = 0) if all(p is not None for p in pooled_rows) else None
+ return embeds, pooled_out, count
+
+
+def _target_dtype(pipe: Any) -> Any:
+ encoder_2 = getattr(pipe, "text_encoder_2", None)
+ if encoder_2 is not None:
+ return encoder_2.dtype
+ encoder = getattr(pipe, "text_encoder", None)
+ if encoder is not None:
+ return encoder.dtype
+ return pipe.unet.dtype
+
+
+def encode_weighted_prompts(
+ pipe: Any,
+ prompts: Sequence[str],
+ negative_prompts: Optional[Sequence[str]],
+ *,
+ guidance_engaged: bool,
+ weighted: bool = True,
+ clip_skip: Optional[int] = None,
+) -> dict[str, Any]:
+ """Pipeline kwargs (``prompt_embeds``, ``pooled_prompt_embeds`` and their negatives) for ``prompts``. Both share one chunk count (CFG batches them)."""
+ import torch
+
+ device = getattr(pipe, "_execution_device", None) or pipe.device
+ dtype = _target_dtype(pipe)
+ embeds, pooled, count = _encode_texts(
+ pipe, list(prompts), weighted = weighted, clip_skip = clip_skip, device = device
+ )
+ kwargs: dict[str, Any] = {}
+ neg_embeds = neg_pooled = None
+ if guidance_engaged:
+ zero_out = negative_prompts is None and bool(
+ getattr(getattr(pipe, "config", None), "force_zeros_for_empty_prompt", False)
+ )
+ if not zero_out:
+ texts = list(negative_prompts) if negative_prompts is not None else [""] * len(prompts)
+ neg_embeds, neg_pooled, neg_count = _encode_texts(
+ pipe,
+ texts,
+ weighted = weighted,
+ clip_skip = clip_skip,
+ device = device,
+ min_chunks = count,
+ )
+ if neg_count > count:
+ embeds, pooled, count = _encode_texts(
+ pipe,
+ list(prompts),
+ weighted = weighted,
+ clip_skip = clip_skip,
+ device = device,
+ min_chunks = neg_count,
+ )
+ embeds = embeds.to(dtype = dtype, device = device)
+ kwargs["prompt_embeds"] = embeds
+ if pooled is not None:
+ kwargs["pooled_prompt_embeds"] = pooled.to(device = device)
+ if guidance_engaged:
+ if neg_embeds is None:
+ neg_embeds = torch.zeros_like(embeds)
+ neg_pooled = torch.zeros_like(pooled) if pooled is not None else None
+ kwargs["negative_prompt_embeds"] = neg_embeds.to(dtype = dtype, device = device)
+ if neg_pooled is not None:
+ kwargs["negative_pooled_prompt_embeds"] = neg_pooled.to(device = device)
+ return kwargs
diff --git a/studio/backend/core/inference/diffusion_sd_controls.py b/studio/backend/core/inference/diffusion_sd_controls.py
new file mode 100644
index 00000000000..262438d8ebe
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_sd_controls.py
@@ -0,0 +1,356 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""SDXL generation controls (diffusers engine): A1111 prompt weighting, CLIP skip, CFG rescale, variation seeds.
+
+CLIP skip uses A1111 numbering: A1111 N = diffusers N - 1 on SD 1.x and N - 2 on SDXL (diffusers already reads
+the penultimate layer there; matches webui-forge minimal_clip_skip). On SDXL it moves CLIP-L and OpenCLIP-G
+together, as ComfyUI's CLIPSetLastLayer (sdxl_clip.SDXLClipModel.set_clip_options) and Forge's
+StableDiffusionXL.set_clip_skip do. Variation seeds follow webui modules/rng.py;
+the base noise is still drawn by prepare_latents, so the generator advances exactly as before.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import inspect
+from dataclasses import dataclass
+from typing import Any, Iterator, Optional, Sequence
+
+from .diffusion_prompt_weighting import (
+ encode_weighted_prompts,
+ exceeds_one_chunk,
+ has_prompt_syntax,
+)
+
+# A1111 CLIP skip at or below this = the family's default layer (SD 1.x would be 1).
+_CLIP_SKIP_BASE: dict[str, int] = {"sdxl": 2}
+CLIP_SKIP_MAX = 12
+GUIDANCE_RESCALE_MAX = 1.0
+# arXiv 2305.08891 sec 3.4; a recommendation only, never a load default.
+GUIDANCE_RESCALE_VPRED_HINT = 0.7
+VARY_STRENGTH = 0.15
+
+
+@dataclass(frozen = True)
+class SdControls:
+ """Request values. ``None`` everywhere = today's behaviour."""
+
+ prompt_weighting: Optional[bool] = None
+ clip_skip: Optional[int] = None
+ cfg_rescale: Optional[float] = None
+ variation_seed: Optional[int] = None
+ variation_strength: Optional[float] = None
+
+ def requested(self) -> bool:
+ return (
+ self.prompt_weighting is True
+ or (self.clip_skip is not None and self.clip_skip > 1)
+ or bool(self.cfg_rescale)
+ or bool(self.variation_strength)
+ )
+
+
+def supports_sd_controls(family: Any, pipe: Any) -> bool:
+ """The loaded pipeline is a CLIP-conditioned U-Net family these controls are defined for."""
+ name = getattr(family, "name", None)
+ if name not in _CLIP_SKIP_BASE or pipe is None:
+ return False
+ return (
+ getattr(pipe, "text_encoder", None) is not None
+ and getattr(pipe, "tokenizer", None) is not None
+ )
+
+
+def diffusers_clip_skip(family_name: str, a1111: Optional[int]) -> Optional[int]:
+ """A1111 CLIP skip (1 = off) -> diffusers ``clip_skip`` (None = the pipeline's layer)."""
+ if a1111 is None:
+ return None
+ skip = int(a1111) - _CLIP_SKIP_BASE.get(family_name, 1)
+ return skip if skip > 0 else None
+
+
+def _min_text_layers(pipe: Any) -> Optional[int]:
+ layers = []
+ for attr in ("text_encoder", "text_encoder_2"):
+ config = getattr(getattr(pipe, attr, None), "config", None)
+ value = getattr(config, "num_hidden_layers", None)
+ if isinstance(value, int):
+ layers.append(value)
+ return min(layers) if layers else None
+
+
+def clip_skip_max(family_name: str, pipe: Any) -> int:
+ """Largest A1111 value whose layer exists in every text encoder (the embedding output excluded)."""
+ layers = _min_text_layers(pipe)
+ if layers is None:
+ return CLIP_SKIP_MAX
+ return max(1, min(CLIP_SKIP_MAX, layers))
+
+
+def _prediction_type(pipe: Any) -> Optional[str]:
+ config = getattr(getattr(pipe, "scheduler", None), "config", None) or {}
+ try:
+ return config.get("prediction_type")
+ except AttributeError:
+ return None
+
+
+def status_fields(family: Any, pipe: Any) -> Optional[dict[str, Any]]:
+ """``sd_controls`` for the status poll: None when the loaded model has none of these controls."""
+ if not supports_sd_controls(family, pipe):
+ return None
+ prediction = _prediction_type(pipe)
+ return {
+ "prompt_weighting": True,
+ "clip_skip": {
+ "min": 1,
+ "max": clip_skip_max(family.name, pipe),
+ "default": 1,
+ "same_as_default_through": _CLIP_SKIP_BASE.get(family.name, 1),
+ },
+ "cfg_rescale": {
+ "min": 0.0,
+ "max": GUIDANCE_RESCALE_MAX,
+ "default": 0.0,
+ "recommended": GUIDANCE_RESCALE_VPRED_HINT if prediction == "v_prediction" else None,
+ },
+ "variation_seed": True,
+ "vary_strength": VARY_STRENGTH,
+ "prediction_type": prediction,
+ }
+
+
+def validate(
+ controls: Optional[SdControls], family: Any, pipe: Any, call_params: Any, workflow: str
+) -> None:
+ """Refuse (ValueError -> 400) a control the loaded model or workflow cannot apply, before any GPU work."""
+ if controls is None or not controls.requested():
+ return
+ label = getattr(family, "label", None) or getattr(family, "name", "this model")
+ if not supports_sd_controls(family, pipe):
+ names = [
+ name
+ for name, on in (
+ ("prompt weighting", controls.prompt_weighting is True),
+ ("CLIP skip", controls.clip_skip is not None and controls.clip_skip > 1),
+ ("CFG rescale", bool(controls.cfg_rescale)),
+ ("variation seed", bool(controls.variation_strength)),
+ )
+ if on
+ ]
+ raise ValueError(
+ f"{', '.join(names)} applies to SDXL models only; the loaded model is {label}. "
+ "Leave these unset for this model."
+ )
+ if controls.clip_skip is not None and controls.clip_skip > clip_skip_max(family.name, pipe):
+ raise ValueError(
+ f"CLIP skip {controls.clip_skip} exceeds this model's text encoder depth "
+ f"(at most {clip_skip_max(family.name, pipe)})."
+ )
+ if controls.cfg_rescale and "guidance_rescale" not in call_params:
+ raise ValueError(f"CFG rescale is not available for the {workflow} workflow of {label}.")
+ if (controls.prompt_weighting is True or _clip_skip_engaged(controls, family)) and (
+ "prompt_embeds" not in call_params
+ ):
+ raise ValueError(
+ f"Prompt weighting and CLIP skip are not available for the {workflow} workflow of {label}."
+ )
+ if controls.variation_strength and (
+ workflow not in ("txt2img", "controlnet") or not _latents_hookable(pipe)
+ ):
+ raise ValueError(
+ "Variation seed applies to text-to-image (and ControlNet) generation only; the "
+ f"{workflow} workflow derives its noise from the source image."
+ )
+
+
+def _clip_skip_engaged(controls: SdControls, family: Any) -> bool:
+ return diffusers_clip_skip(getattr(family, "name", ""), controls.clip_skip) is not None
+
+
+def weighting_engaged(
+ controls: Optional[SdControls], family: Any, pipe: Any, texts: Sequence[Optional[str]]
+) -> bool:
+ """Whether this call encodes its prompts here instead of in the pipeline.
+
+ Engaged by weight syntax, a prompt past 75 tokens or a CLIP skip past the model's layer; otherwise the
+ pipeline encodes, bit-identically."""
+ if not supports_sd_controls(family, pipe):
+ return False
+ controls = controls or SdControls()
+ if _clip_skip_engaged(controls, family):
+ return True
+ if controls.prompt_weighting is not False and any(has_prompt_syntax(t) for t in texts):
+ return True
+ return any(exceeds_one_chunk(pipe, t) for t in texts)
+
+
+def apply_prompt_controls(
+ pipe: Any,
+ family: Any,
+ controls: Optional[SdControls],
+ call_kwargs: dict[str, Any],
+ *,
+ guidance: float,
+) -> bool:
+ """Swap ``prompt`` / ``negative_prompt`` for weighted embeddings in ``call_kwargs`` when engaged.
+ True when weight syntax was applied. Runs inside the render thread and grad mode."""
+ prompt = call_kwargs.get("prompt")
+ negative = call_kwargs.get("negative_prompt")
+ prompts = [prompt] if isinstance(prompt, str) else list(prompt or [])
+ negatives = None
+ if negative is not None:
+ negatives = [negative] * len(prompts) if isinstance(negative, str) else list(negative)
+ if not prompts or not weighting_engaged(controls, family, pipe, prompts + (negatives or [])):
+ return False
+ controls = controls or SdControls()
+ unet_config = getattr(getattr(pipe, "unet", None), "config", None)
+ # The pipeline's do_classifier_free_guidance: guidance > 1 and no guidance-embedding U-Net (LCM).
+ guidance_engaged = (
+ float(guidance) > 1.0 and getattr(unet_config, "time_cond_proj_dim", None) is None
+ )
+ embeds = encode_weighted_prompts(
+ pipe,
+ prompts,
+ negatives,
+ guidance_engaged = guidance_engaged,
+ weighted = controls.prompt_weighting is not False,
+ clip_skip = diffusers_clip_skip(family.name, controls.clip_skip),
+ )
+ call_kwargs.pop("prompt", None)
+ call_kwargs.pop("negative_prompt", None)
+ call_kwargs.update(embeds)
+ return controls.prompt_weighting is not False and any(
+ has_prompt_syntax(t) for t in prompts + (negatives or [])
+ )
+
+
+def guidance_rescale_kwargs(controls: Optional[SdControls], call_params: Any) -> dict[str, Any]:
+ if controls is None or not controls.cfg_rescale or "guidance_rescale" not in call_params:
+ return {}
+ return {"guidance_rescale": float(controls.cfg_rescale)}
+
+
+def slerp(val: float, low: Any, high: Any) -> Any:
+ """webui ``modules/rng.py`` ``slerp`` on one image's ``[C, H, W]`` noise (norms over dim 1, as webui)."""
+ import torch
+
+ low_norm = low / torch.norm(low, dim = 1, keepdim = True)
+ high_norm = high / torch.norm(high, dim = 1, keepdim = True)
+ dot = (low_norm * high_norm).sum(1)
+ if dot.mean() > 0.9995:
+ return low * val + high * (1 - val)
+ omega = torch.acos(dot)
+ so = torch.sin(omega)
+ return (torch.sin((1.0 - val) * omega) / so).unsqueeze(1) * low + (
+ torch.sin(val * omega) / so
+ ).unsqueeze(1) * high
+
+
+def variation_noise(base: Any, variation: Any, strength: float) -> Any:
+ """Per-image slerp of ``[B, C, H, W]`` latents in float32, cast back to the latents' dtype."""
+ import torch
+
+ if not strength:
+ return base
+ rows = [
+ slerp(float(strength), b.float(), v.float())
+ for b, v in zip(base.unbind(0), variation.unbind(0))
+ ]
+ return torch.stack(rows).to(base.dtype)
+
+
+def _latents_hookable(pipe: Any) -> bool:
+ fn = getattr(pipe, "prepare_latents", None)
+ if not callable(fn):
+ return False
+ try:
+ params = inspect.signature(fn).parameters
+ except (TypeError, ValueError):
+ return False
+ return "num_channels_latents" in params and "generator" in params and "latents" in params
+
+
+def variation_seeds(variation_seed: int, count: int) -> list[int]:
+ """One variation seed per image, ``variation_seed + index`` (webui ``all_subseeds``)."""
+ return [int(variation_seed) + i for i in range(count)]
+
+
+def variation_latents(
+ pipe: Any, seeds: Optional[Sequence[int]], strength: Optional[float]
+) -> Iterator[None]:
+ """For one pipeline call, slerp the initial latents toward ``seeds``' noise by ``strength``. No-op when off."""
+ if not strength or not seeds:
+ yield
+ return
+ from diffusers.utils.torch_utils import randn_tensor
+ import torch
+
+ original = pipe.prepare_latents
+ signature = inspect.signature(original)
+ had_own = "prepare_latents" in getattr(pipe, "__dict__", {})
+
+ def prepare_latents(*args: Any, **kwargs: Any) -> Any:
+ latents = original(*args, **kwargs)
+ if kwargs.get("latents") is not None:
+ return latents
+ bound = signature.bind(*args, **kwargs)
+ if bound.arguments.get("latents") is not None:
+ return latents
+ generator = bound.arguments.get("generator")
+ gens = generator if isinstance(generator, list) else [generator]
+ if len(gens) != latents.shape[0] and len(gens) == 1:
+ gens = gens * latents.shape[0]
+ devices = [g.device if g is not None else latents.device for g in gens]
+ var_gens = [
+ torch.Generator(device = d).manual_seed(int(s))
+ for d, s in zip(devices, list(seeds)[: len(devices)])
+ ]
+ if len(var_gens) != latents.shape[0]:
+ raise ValueError("variation seeds do not match the batch")
+ var = randn_tensor(
+ tuple(latents.shape),
+ generator = var_gens if len(var_gens) > 1 else var_gens[0],
+ device = latents.device,
+ dtype = latents.dtype,
+ )
+ # Same scaling prepare_latents applied to the base noise, so strength 1 is exactly the variation seed's noise.
+ var = var * pipe.scheduler.init_noise_sigma
+ return variation_noise(latents, var, float(strength))
+
+ pipe.prepare_latents = prepare_latents
+ try:
+ yield
+ finally:
+ if had_own:
+ pipe.prepare_latents = original
+ else:
+ try:
+ del pipe.prepare_latents
+ except AttributeError:
+ pass
+
+
+def record_fields(
+ controls: Optional[SdControls],
+ *,
+ weighting_used: bool,
+ variation_seed_list: Optional[Sequence[int]],
+) -> dict[str, Any]:
+ """Result / recipe fields for what actually ran; None = off (prompt_weighting: False = turned off)."""
+ controls = controls or SdControls()
+ varied = bool(controls.variation_strength) and bool(variation_seed_list)
+ return {
+ "prompt_weighting": True
+ if weighting_used
+ else (False if controls.prompt_weighting is False else None),
+ "clip_skip": int(controls.clip_skip)
+ if controls.clip_skip and controls.clip_skip > 1
+ else None,
+ "cfg_rescale": float(controls.cfg_rescale) if controls.cfg_rescale else None,
+ "variation_seed": int(variation_seed_list[0]) if varied else None,
+ "variation_seeds": [int(s) for s in variation_seed_list] if varied else None,
+ "variation_strength": float(controls.variation_strength) if varied else None,
+ }
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index 71a655ca18a..aac1f9a35bd 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -73,6 +73,19 @@ def _params_text(meta: dict[str, Any]) -> str:
f"Seed: {meta.get('seed')}, Size: {meta.get('width')}x{meta.get('height')}, "
f"Model: {meta.get('model', '')}"
)
+ # webui infotext names (processing.create_infotext).
+ if meta.get("variation_strength"):
+ fields += (
+ f", Variation seed: {meta.get('variation_seed')}, "
+ f"Variation seed strength: {meta['variation_strength']}"
+ )
+ if meta.get("clip_skip"):
+ fields += f", Clip skip: {meta['clip_skip']}"
+ if meta.get("cfg_rescale"):
+ fields += f", CFG rescale: {meta['cfg_rescale']}"
+ # webui opts.emphasis infotext: a prompt run with weighting off replays literally there too.
+ if meta.get("prompt_weighting") is False:
+ fields += ", Emphasis: None"
if meta.get("workflow") in _IMG2IMG_WORKFLOWS and meta.get("strength") is not None:
fields += f", Denoising strength: {meta['strength']}"
# A1111 inpaint keys (modules/processing.py), only when this inpaint applied them.
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index e4d04210ffa..552e130e738 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -2884,6 +2884,8 @@ _UNREAD_LOADING_CARD = object()
class SdCppDiffusionBackend:
"""Native sd.cpp backend with the diffusers ``DiffusionBackend`` method surface."""
+ # sd.cpp parses A1111 attention syntax ((word:1.2), [word]) in every prompt; it cannot be turned off.
+ parses_prompt_attention = True
# Class-level defaults so an instance built with ``__new__`` answers before any load ran.
_loading_cards = None
_committed_loading_card: Optional[str] = None
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 28d86e132a8..7a4290dcc3f 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5070,6 +5070,44 @@ class DiffusionGenerateRequest(BaseModel):
description = "Noise schedule for the sampler. Omitted or 'automatic' uses the sampler's "
"usual schedule (Karras for DPM++ 2M, Exponential for DPM++ 2M SDE, else the model's).",
)
+ prompt_weighting: Optional[bool] = Field(
+ None,
+ description = "AUTOMATIC1111 prompt syntax on SDXL: (word:1.2), (word), [word], \\( escapes and "
+ "BREAK. Omitted: on. A prompt without that syntax that fits in 75 CLIP tokens renders exactly "
+ "as before; a longer one is encoded in 75-token chunks instead of truncated. false reads "
+ "brackets literally (long prompts still chunk). "
+ "true is refused for a model without the control.",
+ )
+ clip_skip: Optional[int] = Field(
+ None,
+ ge = 1,
+ le = 12,
+ description = "CLIP skip in AUTOMATIC1111 numbering: 1 = the model's own layer. On SDXL 1 and 2 "
+ "both read the penultimate layer the model already uses (as in A1111 and Forge), 3 reads one "
+ "layer earlier. Values above 1 are refused for models without the control.",
+ )
+ cfg_rescale: Optional[float] = Field(
+ None,
+ ge = 0.0,
+ le = 1.0,
+ description = "CFG rescale (diffusers guidance_rescale, arXiv 2305.08891): 0 = off. Mostly for "
+ "v-prediction / zero-terminal-SNR checkpoints, where 0.7 is the paper's value. Above 0 is "
+ "refused for models without the control.",
+ )
+ variation_seed: Optional[int] = Field(
+ None,
+ ge = 0,
+ le = 2**32 - 1,
+ description = "Variation seed (AUTOMATIC1111 subseed): the seed's noise is slerped toward this "
+ "seed's noise by variation_strength. Omitted with a strength: random. Text-to-image only.",
+ )
+ variation_strength: Optional[float] = Field(
+ None,
+ ge = 0.0,
+ le = 1.0,
+ description = "How far toward the variation seed's noise: 0 = off, 1 = the variation seed alone. "
+ "With a batch, every image keeps the seed and image i uses variation_seed + i.",
+ )
shift: Optional[float] = Field(
None,
gt = 0.0,
@@ -5213,6 +5251,10 @@ class GalleryImage(BaseModel):
)
model: Optional[str] = Field(None, description = "Model repo id that produced it")
# The load-time BUILD. The repo id alone does not identify a pipeline (quant choice, torchao scheme, baked adapters), so without these a recipe cannot be rebuilt.
+ engine: Optional[str] = Field(
+ None,
+ description = "Engine that ran it (diffusers, sd_cpp or mlx); the same seed draws different noise on each",
+ )
model_kind: Optional[str] = Field(
None, description = "How the model was loaded: gguf, single_file or pipeline"
)
@@ -5304,6 +5346,21 @@ class GalleryImage(BaseModel):
schedule_type: Optional[str] = Field(
None, description = "Schedule type the sampler ran with (resolved); null = the config's"
)
+ prompt_weighting: Optional[bool] = Field(
+ None,
+ description = "True when AUTOMATIC1111 weight syntax was applied; false when the request turned "
+ "it off (literal prompt); null = nothing to weight",
+ )
+ clip_skip: Optional[int] = Field(
+ None, description = "CLIP skip (AUTOMATIC1111 numbering) above 1; null = the model's layer"
+ )
+ cfg_rescale: Optional[float] = Field(None, description = "CFG rescale; null = off")
+ variation_seed: Optional[int] = Field(
+ None, description = "Variation seed THIS image used; null = no variation"
+ )
+ variation_strength: Optional[float] = Field(
+ None, description = "Variation strength; null = no variation"
+ )
shift: Optional[float] = Field(
None, description = "Requested static flow shift; null = the model's"
)
@@ -5688,6 +5745,11 @@ class DiffusionStatusResponse(BaseModel):
transformer_cache: Optional[str] = Field(
None, description = "Step cache engaged: fbcache | static | null"
)
+ baked_loras: list[str] = Field(
+ default_factory = list,
+ description = "Adapter ids baked into the transformer at load time, as GalleryImage.baked_loras "
+ "records them; empty when the load baked none",
+ )
transformer_cache_stats: Optional[dict] = Field(
None,
description = "Static step skip only: mode, schedule and the last generation's "
@@ -5796,6 +5858,13 @@ class DiffusionStatusResponse(BaseModel):
"first is 'default' (the shipped scheduler). Empty when there is no choice: flow-matching "
"models, the native and MLX engines, or nothing loaded.",
)
+ sd_controls: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "SDXL prompt weighting, CLIP skip, CFG rescale and variation seed: "
+ "{prompt_weighting, clip_skip {min, max, default, same_as_default_through}, cfg_rescale "
+ "{min, max, default, recommended}, variation_seed, vary_strength, prediction_type}. Null when "
+ "the loaded model has none of them (every non-SDXL family, the native and MLX engines).",
+ )
shift_control: Optional[Dict[str, Any]] = Field(
None,
description = "The Shift control the loaded flow-matching model takes, or null to hide it: "
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1863e7fea00..1ed7c02e4f7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -51906,6 +51906,41 @@ def _sampler_kwargs(generate: Any, sampler: Optional[str], schedule_type: Option
return {}
+def _sd_controls_kwargs(generate: Any, request: Any) -> dict:
+ """``sd_controls`` (prompt weighting, CLIP skip, CFG rescale, variation seed) for an engine whose generate() takes
+ them (diffusers only). A value that changes the render, sent to another engine, is refused (ValueError -> 400)
+ rather than ignored, so a recipe never records a control that did not run."""
+ from core.inference.diffusion_sd_controls import SdControls
+
+ controls = SdControls(
+ prompt_weighting = getattr(request, "prompt_weighting", None),
+ clip_skip = getattr(request, "clip_skip", None),
+ cfg_rescale = getattr(request, "cfg_rescale", None),
+ variation_seed = getattr(request, "variation_seed", None),
+ variation_strength = getattr(request, "variation_strength", None),
+ )
+ try:
+ import inspect
+ if "sd_controls" in inspect.signature(generate).parameters:
+ return {"sd_controls": controls}
+ except (TypeError, ValueError):
+ pass
+ if controls.requested():
+ raise ValueError(
+ "Prompt weighting, CLIP skip, CFG rescale and variation seeds are supported on the "
+ "diffusers engine only."
+ )
+ if controls.prompt_weighting is False and getattr(
+ getattr(generate, "__self__", None), "parses_prompt_attention", False
+ ):
+ # sd.cpp reads (word:1.5) itself, so the prompt would still render weighted.
+ raise ValueError(
+ "Turning prompt weighting off is supported on the diffusers engine only: the native "
+ "sd.cpp engine always reads (word:1.2) attention syntax."
+ )
+ return {}
+
+
def _shift_kwargs(generate: Any, request: Any) -> dict:
"""``shift`` / ``base_shift`` / ``max_shift`` for an engine whose generate() takes them (diffusers only).
@@ -52390,6 +52425,7 @@ async def _generate_diffusion_image_run(
),
**_live_preview_kwarg(backend.generate, request.live_preview),
**_sampler_kwargs(backend.generate, request.sampler, request.schedule_type),
+ **_sd_controls_kwargs(backend.generate, request),
**_shift_kwargs(backend.generate, request),
**_inpaint_mask_kwargs(backend.generate, request),
)
@@ -52509,6 +52545,16 @@ async def _generate_diffusion_image_run(
"localized_edit": result.get("localized_edit"),
"sampler": result.get("sampler"),
"schedule_type": result.get("schedule_type"),
+ "prompt_weighting": result.get("prompt_weighting"),
+ "clip_skip": result.get("clip_skip"),
+ "cfg_rescale": result.get("cfg_rescale"),
+ "variation_seed": (
+ result["variation_seeds"][index]
+ if result.get("variation_seeds")
+ and index < len(result["variation_seeds"])
+ else None
+ ),
+ "variation_strength": result.get("variation_strength"),
**{
key: result[key]
for key in (
diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py
index c360f43ca22..ef180b98157 100644
--- a/studio/backend/tests/test_diffusion_backend.py
+++ b/studio/backend/tests/test_diffusion_backend.py
@@ -7232,6 +7232,28 @@ def test_status_names_the_nvfp4_kernel_backend_that_actually_ran():
assert dmod._transformer_quant_backend(hostile) is None
+def test_status_reports_the_load_time_baked_loras(fake_runtime, tmp_path):
+ # Vary compares a gallery record's baked_loras with the load's, so status must carry the same ids, and an
+ # unbaked or unloaded state must say none rather than omit the key.
+ from models.inference import DiffusionStatusResponse
+
+ assert DiffusionBackend().status()["baked_loras"] == []
+ backend = DiffusionBackend()
+ (tmp_path / "z-image-turbo-Q8_0.gguf").write_bytes(b"x")
+ _load_into(backend, tmp_path, gguf_filename = "z-image-turbo-Q8_0.gguf", base_repo = None)
+ assert backend.status()["baked_loras"] == []
+ pipe = backend._state.pipe
+ pipe._unsloth_loras = (
+ ("sloth", "/adapters/sloth.safetensors", 0.8),
+ ("ink", "/adapters/ink.safetensors", 0.5),
+ )
+ pipe._unsloth_loras_baked = True
+ status = backend.status()
+ assert status["baked_loras"] == ["sloth", "ink"]
+ assert DiffusionStatusResponse(**status).model_dump()["baked_loras"] == ["sloth", "ink"]
+ assert DiffusionStatusResponse(loaded = True).model_dump()["baked_loras"] == []
+
+
def test_the_unloaded_status_declares_the_quant_backend_key():
status = DiffusionBackend().status()
assert "transformer_quant_backend" in status
diff --git a/studio/backend/tests/test_diffusion_sd_controls.py b/studio/backend/tests/test_diffusion_sd_controls.py
new file mode 100644
index 00000000000..64f7d9d8fec
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_sd_controls.py
@@ -0,0 +1,742 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""SDXL prompt weighting, CLIP skip, CFG rescale and variation seeds (diffusion_sd_controls / prompt_weighting)."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+
+from core.inference.diffusion_prompt_weighting import (
+ CHUNK_TOKENS,
+ MAX_CHUNKS,
+ MAX_PROMPT_CHARS,
+ apply_emphasis,
+ has_prompt_syntax,
+ parse_prompt_attention,
+ tokenize_chunks,
+)
+from core.inference.diffusion_sd_controls import (
+ SdControls,
+ apply_prompt_controls,
+ diffusers_clip_skip,
+ guidance_rescale_kwargs,
+ record_fields,
+ slerp,
+ status_fields,
+ validate,
+ variation_latents,
+ variation_noise,
+ variation_seeds,
+ weighting_engaged,
+)
+
+SDXL = SimpleNamespace(name = "sdxl", label = "SDXL")
+FLUX = SimpleNamespace(name = "flux.1-dev", label = "FLUX.1 dev")
+
+
+# --- parser: webui modules/prompt_parser.py parse_prompt_attention doctests, plus the escapes and BREAK ---
+
+
+def _approx(parsed):
+ return [[t, round(w, 6)] for t, w in parsed]
+
+
+ "text, expected",
+ [
+ ("normal text", [["normal text", 1.0]]),
+ ("an (important) word", [["an ", 1.0], ["important", 1.1], [" word", 1.0]]),
+ ("(unbalanced", [["unbalanced", 1.1]]),
+ ("\\(literal\\]", [["(literal]", 1.0]]),
+ ("(unnecessary)(parens)", [["unnecessaryparens", 1.1]]),
+ (
+ "a (((house:1.3)) [on] a (hill:0.5), sun, (((sky))).",
+ [
+ ["a ", 1.0],
+ ["house", 1.573],
+ [" ", 1.1],
+ ["on", 1.0],
+ [" a ", 1.1],
+ ["hill", 0.55],
+ [", sun, ", 1.1],
+ ["sky", 1.4641],
+ [".", 1.1],
+ ],
+ ),
+ ("[faded]", [["faded", round(1 / 1.1, 6)]]),
+ ("(word:1.2)", [["word", 1.2]]),
+ ("( word : 0.5 )", [[" word ", 0.5]]),
+ ("a cat: big", [["a cat: big", 1.0]]),
+ ("stray) and ] close", [["stray) and ] close", 1.0]]),
+ ("horror \\(theme\\)", [["horror (theme)", 1.0]]),
+ ("back\\\\slash", [["back\\slash", 1.0]]),
+ ("a BREAK b", [["a", 1.0], ["BREAK", -1.0], ["b", 1.0]]),
+ ("", [["", 1.0]]),
+ ("(cat:1.2.3)", [["(cat:1.2.3)", 1.0]]),
+ ("a ((cat:1.2.3) dog)", [["a ", 1.0], ["(cat:1.2.3) dog", 1.1]]),
+ ("([cat:1.2.3)]", [["(", 1.0], ["cat:1.2.3)", round(1 / 1.1, 6)]]),
+ ],
+)
+def test_parse_matches_webui(text, expected):
+ assert _approx(parse_prompt_attention(text)) == _approx(expected)
+
+
+ "text, syntax",
+ [
+ ("a red fox in the snow", False),
+ ("a cat: big, 4k", False),
+ ("photo, 85mm, f/1.8", False),
+ ("a (fox)", True),
+ ("a [fox]", True),
+ ("(fox:1.0)", True),
+ ("horror \\(theme\\)", True),
+ ("a BREAK b", True),
+ ("BREAKFAST", False),
+ (None, False),
+ ("", False),
+ ],
+)
+def test_has_prompt_syntax(text, syntax):
+ assert has_prompt_syntax(text) is syntax
+
+
+# --- tokenization / chunking: webui sd_hijack_clip.tokenize_line ---
+
+
+class _WordTokenizer:
+ """One id per space-separated word; ``,`` is its own token (webui's comma_token)."""
+
+ bos_token_id = 1
+ eos_token_id = 2
+
+ def __init__(self, pad = None):
+ self.pad_token_id = pad
+ self.vocab = {",</w>": 3}
+
+ def get_vocab(self):
+ return self.vocab
+
+ def __call__(self, text, **_kw):
+ ids = []
+ for word in text.replace(",", " , ").split():
+ ids.append(3 if word == "," else self.vocab.setdefault(word, len(self.vocab) + 10))
+ return {"input_ids": ids}
+
+
+def test_short_prompt_is_one_padded_chunk():
+ chunks = tokenize_chunks("a (b:1.5) c", _WordTokenizer(pad = 2))
+ assert len(chunks) == 1
+ ids, mults = chunks[0]
+ assert len(ids) == len(mults) == CHUNK_TOKENS + 2
+ assert ids[0] == 1 and ids[4] == 2 and set(ids[4:]) == {2}
+ assert mults[:5] == [1.0, 1.0, 1.5, 1.0, 1.0]
+
+
+def test_pad_after_first_eos_uses_the_tokenizer_pad_id():
+ # SDXL's OpenCLIP tokenizer pads with "!" (id 0), not <eos>: webui's id_pad replacement.
+ ids, _ = tokenize_chunks("a b", _WordTokenizer(pad = 0))[0]
+ assert ids[:4] == [1, 11, 12, 2]
+ assert ids[4:] == [0] * (len(ids) - 4)
+
+
+def test_long_prompt_chunks_instead_of_truncating():
+ words = " ".join(f"w{i}" for i in range(160))
+ chunks = tokenize_chunks(words, _WordTokenizer(pad = 2))
+ assert len(chunks) == 3
+ assert all(len(ids) == CHUNK_TOKENS + 2 for ids, _ in chunks)
+ body = [t for ids, _ in chunks for t in ids if t not in (1, 2)]
+ assert len(body) == 160
+
+
+def test_chunk_count_is_capped():
+ tok = _WordTokenizer(pad = 2)
+ words = " ".join(f"w{i}" for i in range(CHUNK_TOKENS * (MAX_CHUNKS + 5)))
+ assert len(tokenize_chunks(words, tok)) == MAX_CHUNKS
+ assert len(tokenize_chunks(" BREAK ".join(["a"] * (MAX_CHUNKS * 3)), tok)) == MAX_CHUNKS
+ at_cap = " ".join(f"w{i}" for i in range(CHUNK_TOKENS * MAX_CHUNKS))
+ assert len(tokenize_chunks(at_cap, tok)) == MAX_CHUNKS
+
+
+class _CountingTokenizer(_WordTokenizer):
+ def __init__(self, pad = None):
+ super().__init__(pad)
+ self.calls = 0
+ self.chars = 0
+
+ def __call__(self, text, **kw):
+ self.calls += 1
+ self.chars += len(text)
+ return super().__call__(text, **kw)
+
+
+def test_chunking_stops_reading_at_the_cap():
+ # A huge API prompt is neither tokenized nor chunked past the cap: work stays bounded, not proportional.
+ tok = _CountingTokenizer(pad = 2)
+ chunks = tokenize_chunks("a BREAK " * 200_000, tok)
+ assert len(chunks) == MAX_CHUNKS
+ assert tok.calls <= MAX_CHUNKS + 1
+ literal = _CountingTokenizer(pad = 2)
+ assert len(tokenize_chunks("w " * 5_000_000, literal, weighted = False)) == MAX_CHUNKS
+ assert literal.chars <= MAX_PROMPT_CHARS
+ # Words past the 32nd chunk never reach a chunk; the kept chunks are the same as for the capped text.
+ words = " ".join(f"w{i}" for i in range(CHUNK_TOKENS * (MAX_CHUNKS + 5)))
+ at_cap = " ".join(f"w{i}" for i in range(CHUNK_TOKENS * MAX_CHUNKS))
+ assert tokenize_chunks(words, _WordTokenizer(pad = 2)) == tokenize_chunks(
+ at_cap, _WordTokenizer(pad = 2)
+ )
+
+
+def test_break_starts_a_new_chunk():
+ chunks = tokenize_chunks("a b BREAK c", _WordTokenizer(pad = 2))
+ assert len(chunks) == 2
+ assert chunks[1][0][1] != 2 and chunks[1][0][2] == 2
+
+
+def test_comma_backtrack_moves_the_tail_to_the_next_chunk():
+ # 70 words, a comma, then 10 more: the chunk fills mid-phrase, so webui moves the words after the comma.
+ text = " ".join(f"w{i}" for i in range(70)) + " , " + " ".join(f"x{i}" for i in range(10))
+ chunks = tokenize_chunks(text, _WordTokenizer(pad = 2))
+ first_ids = chunks[0][0]
+ assert 3 in first_ids
+ body = [t for t in first_ids[1:] if t != 2]
+ assert body[-1] == 3 # the first chunk ends at the comma
+ assert len([t for t in chunks[1][0] if t not in (1, 2)]) == 10
+
+
+def test_literal_mode_reads_brackets_as_text():
+ tok = _WordTokenizer(pad = 2)
+ ids, mults = tokenize_chunks("(a:1.5)", tok, weighted = False)[0]
+ assert set(mults) == {1.0}
+ assert "(a:1.5)" in tok.vocab
+
+
+def test_emphasis_restores_the_mean():
+ torch = pytest.importorskip("torch")
+ z = torch.arange(1, 13, dtype = torch.float32).reshape(1, 3, 4)
+ w = torch.tensor([[1.0, 1.5, 0.5]])
+ out = apply_emphasis(z, w)
+ assert torch.allclose(out.mean(), z.mean())
+ ones = apply_emphasis(z, torch.ones(1, 3))
+ assert torch.equal(ones, z)
+
+
+ "family, a1111, expected",
+ [
+ ("sdxl", None, None),
+ ("sdxl", 1, None),
+ (
+ "sdxl",
+ 2,
+ None,
+ ), # SDXL already reads the penultimate layer (webui-forge minimal_clip_skip=2)
+ ("sdxl", 3, 1),
+ ("sdxl", 4, 2),
+ ("sd15", 1, None), # SD 1.x layout: A1111 N = diffusers N - 1
+ ("sd15", 2, 1),
+ ],
+)
+def test_clip_skip_mapping(family, a1111, expected):
+ assert diffusers_clip_skip(family, a1111) == expected
+
+
+def _fake_pipe(layers = 12):
+ enc = SimpleNamespace(config = SimpleNamespace(num_hidden_layers = layers))
+ return SimpleNamespace(
+ text_encoder = enc,
+ tokenizer = _WordTokenizer(pad = 2),
+ text_encoder_2 = enc,
+ tokenizer_2 = _WordTokenizer(pad = 0),
+ )
+
+
+def test_long_plain_prompts_engage_the_chunked_encoder():
+ from core.inference.diffusion_prompt_weighting import exceeds_one_chunk
+
+ pipe = _fake_pipe()
+ fits = " ".join(f"w{i}" for i in range(CHUNK_TOKENS))
+ longer = fits + " extra"
+ assert not exceeds_one_chunk(pipe, fits)
+ assert exceeds_one_chunk(pipe, longer)
+ assert not weighting_engaged(None, SDXL, pipe, [fits])
+ assert weighting_engaged(None, SDXL, pipe, [longer])
+ # Literal mode still chunks a long prompt (webui's "None" emphasis), but not a short bracketed one.
+ assert weighting_engaged(SdControls(prompt_weighting = False), SDXL, pipe, [longer])
+ assert not weighting_engaged(SdControls(prompt_weighting = False), SDXL, pipe, ["(a:2)"])
+ # A long negative prompt alone engages it too, since both branches share one length.
+ assert weighting_engaged(None, SDXL, pipe, ["a fox", longer])
+ assert not weighting_engaged(None, FLUX, pipe, [longer])
+
+
+_TXT2IMG_PARAMS = {"prompt_embeds": 1, "guidance_rescale": 1, "negative_prompt": 1}
+
+
+def test_defaults_request_nothing():
+ assert not SdControls().requested()
+ assert not SdControls(prompt_weighting = False, clip_skip = 1, cfg_rescale = 0.0).requested()
+ assert not SdControls(variation_seed = 5, variation_strength = 0.0).requested()
+
+
+def test_flux_refuses_requested_controls_clearly():
+ pipe = SimpleNamespace(transformer = object())
+ assert status_fields(FLUX, pipe) is None
+ validate(SdControls(), FLUX, pipe, {}, "txt2img")
+ validate(SdControls(clip_skip = 1, prompt_weighting = False), FLUX, pipe, {}, "txt2img")
+ with pytest.raises(ValueError, match = "CLIP skip, CFG rescale applies to SDXL models only"):
+ validate(SdControls(clip_skip = 2, cfg_rescale = 0.5), FLUX, pipe, {}, "txt2img")
+ with pytest.raises(ValueError, match = "variation seed applies to SDXL"):
+ validate(SdControls(variation_strength = 0.2), FLUX, pipe, {}, "txt2img")
+ assert not weighting_engaged(None, FLUX, pipe, ["a (fox:1.5)"])
+
+
+def test_sdxl_status_and_validation():
+ pipe = _fake_pipe(layers = 12)
+ pipe.scheduler = SimpleNamespace(config = {"prediction_type": "v_prediction"})
+ fields = status_fields(SDXL, pipe)
+ assert fields["clip_skip"] == {"min": 1, "max": 12, "default": 1, "same_as_default_through": 2}
+ assert fields["cfg_rescale"]["default"] == 0.0
+ assert fields["cfg_rescale"]["recommended"] == 0.7
+ pipe.scheduler = SimpleNamespace(config = {"prediction_type": "epsilon"})
+ assert status_fields(SDXL, pipe)["cfg_rescale"]["recommended"] is None
+ validate(SdControls(clip_skip = 12, cfg_rescale = 0.7), SDXL, pipe, _TXT2IMG_PARAMS, "txt2img")
+ with pytest.raises(ValueError, match = "exceeds"):
+ validate(SdControls(clip_skip = 13), SDXL, pipe, _TXT2IMG_PARAMS, "txt2img")
+ with pytest.raises(ValueError, match = "CFG rescale is not available"):
+ validate(SdControls(cfg_rescale = 0.7), SDXL, pipe, {"prompt_embeds": 1}, "controlnet")
+ with pytest.raises(ValueError, match = "text-to-image"):
+ validate(SdControls(variation_strength = 0.2), SDXL, pipe, _TXT2IMG_PARAMS, "img2img")
+ # Inpaint's prepare_latents takes the same leading arguments but returns noised image latents.
+ pipe.prepare_latents = _LatentPipe().prepare_latents
+ validate(SdControls(variation_strength = 0.2), SDXL, pipe, _TXT2IMG_PARAMS, "txt2img")
+ with pytest.raises(ValueError, match = "text-to-image"):
+ validate(SdControls(variation_strength = 0.2), SDXL, pipe, _TXT2IMG_PARAMS, "inpaint")
+
+
+def test_record_keeps_a_literal_prompt_choice():
+ def pw(controls, used):
+ return record_fields(controls, weighting_used = used, variation_seed_list = None)[
+ "prompt_weighting"
+ ]
+
+ assert pw(SdControls(prompt_weighting = False), False) is False
+ assert pw(SdControls(), True) is True
+ assert pw(SdControls(), False) is None
+ assert pw(None, False) is None
+
+
+def test_guidance_rescale_passthrough():
+ assert guidance_rescale_kwargs(None, _TXT2IMG_PARAMS) == {}
+ assert guidance_rescale_kwargs(SdControls(cfg_rescale = 0.0), _TXT2IMG_PARAMS) == {}
+ assert guidance_rescale_kwargs(SdControls(cfg_rescale = 0.7), _TXT2IMG_PARAMS) == {
+ "guidance_rescale": 0.7
+ }
+ assert guidance_rescale_kwargs(SdControls(cfg_rescale = 0.7), {}) == {}
+
+
+def test_plain_prompt_leaves_the_call_untouched():
+ pipe = _fake_pipe()
+ kwargs = {"prompt": "a red fox", "negative_prompt": "blurry", "num_inference_steps": 4}
+ before = dict(kwargs)
+ assert apply_prompt_controls(pipe, SDXL, None, kwargs, guidance = 7.0) is False
+ assert apply_prompt_controls(pipe, SDXL, SdControls(clip_skip = 2), kwargs, guidance = 7.0) is False
+ assert (
+ apply_prompt_controls(
+ pipe, SDXL, SdControls(prompt_weighting = False), {"prompt": "(a:2)"}, guidance = 7.0
+ )
+ is False
+ )
+ assert kwargs == before
+
+
+# --- variation seeds: webui modules/rng.py slerp / ImageRNG ---
+
+
+def _webui_slerp(val, low, high):
+ import torch
+
+ low_norm = low / torch.norm(low, dim = 1, keepdim = True)
+ high_norm = high / torch.norm(high, dim = 1, keepdim = True)
+ dot = (low_norm * high_norm).sum(1)
+ if dot.mean() > 0.9995:
+ return low * val + high * (1 - val)
+ omega = torch.acos(dot)
+ so = torch.sin(omega)
+ return (torch.sin((1.0 - val) * omega) / so).unsqueeze(1) * low + (
+ torch.sin(val * omega) / so
+ ).unsqueeze(1) * high
+
+
+def test_slerp_matches_webui_and_its_endpoints():
+ torch = pytest.importorskip("torch")
+ g = torch.Generator().manual_seed(0)
+ low = torch.randn(4, 8, 8, generator = g)
+ high = torch.randn(4, 8, 8, generator = g)
+ for val in (0.0, 0.1, 0.5, 1.0):
+ assert torch.equal(slerp(val, low, high), _webui_slerp(val, low, high))
+ assert torch.allclose(slerp(0.0, low, high), low)
+ assert torch.equal(slerp(1.0, low, high), high)
+ # Spherical: the interpolant keeps a unit-variance-like norm, a lerp would shrink it.
+ mid = slerp(0.5, low, high)
+ assert abs(mid.std().item() - 1.0) < 0.2
+ assert torch.equal(slerp(0.25, low, low), low * 0.25 + low * 0.75)
+
+
+def test_variation_noise_per_image_and_dtype():
+ torch = pytest.importorskip("torch")
+ g = torch.Generator().manual_seed(1)
+ base = torch.randn(2, 4, 8, 8, generator = g).half()
+ var = torch.randn(2, 4, 8, 8, generator = g).half()
+ assert variation_noise(base, var, 0.0) is base
+ out = variation_noise(base, var, 1.0)
+ assert out.dtype == torch.float16 and torch.equal(out, var)
+ mixed = variation_noise(base, var, 0.3)
+ expected = torch.stack(
+ [_webui_slerp(0.3, b.float(), v.float()) for b, v in zip(base, var)]
+ ).half()
+ assert torch.equal(mixed, expected)
+
+
+def test_variation_seeds_follow_webui_subseeds():
+ assert variation_seeds(100, 3) == [100, 101, 102]
+
+
+class _LatentPipe:
+ """prepare_latents shaped like StableDiffusionXLPipeline's."""
+
+ def __init__(self):
+ self.scheduler = SimpleNamespace(init_noise_sigma = 2.0)
+
+ def prepare_latents(
+ self,
+ batch_size,
+ num_channels_latents,
+ height,
+ width,
+ dtype,
+ device,
+ generator,
+ latents = None,
+ ):
+ from diffusers.utils.torch_utils import randn_tensor
+
+ shape = (batch_size, num_channels_latents, height // 8, width // 8)
+ if latents is None:
+ latents = randn_tensor(shape, generator = generator, device = device, dtype = dtype)
+ return latents * self.scheduler.init_noise_sigma
+
+
+def test_variation_latents_reproducible_and_restored():
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("diffusers")
+ pipe = _LatentPipe()
+
+ def draw(
+ seed,
+ var = None,
+ strength = None,
+ ):
+ with variation_latents(pipe, var, strength):
+ gen = torch.Generator().manual_seed(seed)
+ out = pipe.prepare_latents(1, 4, 64, 64, torch.float32, torch.device("cpu"), gen)
+ return out, gen.initial_seed(), torch.randn(1, generator = gen)
+
+ plain, _, after_plain = draw(11)
+ off, _, _ = draw(11, [12], 0.0)
+ assert torch.equal(plain, off)
+ full, _, after_full = draw(11, [12], 1.0)
+ assert torch.equal(full, draw(12)[0]) # strength 1 is exactly the variation seed's noise
+ # The request's generator advanced exactly as without a variation (ancestral samplers unchanged).
+ assert torch.equal(after_plain, after_full)
+ a, _, _ = draw(11, [12], 0.3)
+ b, _, _ = draw(11, [12], 0.3)
+ assert torch.equal(a, b) and not torch.equal(a, plain)
+ assert "prepare_latents" not in pipe.__dict__ # the class method is back
+
+
+# --- real tiny SDXL encode path vs the pipeline's own encode_prompt ---
+
+
[email protected](scope = "module")
+def tiny_sdxl():
+ torch = pytest.importorskip("torch")
+ diffusers = pytest.importorskip("diffusers")
+ from huggingface_hub import snapshot_download
+
+ try:
+ path = snapshot_download("hf-internal-testing/tiny-stable-diffusion-xl-pipe")
+ except Exception as exc: # noqa: BLE001
+ pytest.skip(f"tiny SDXL pipeline not reachable: {exc}")
+ pipe = diffusers.StableDiffusionXLPipeline.from_pretrained(path, torch_dtype = torch.float32)
+ pipe.set_progress_bar_config(disable = True)
+ return pipe
+
+
[email protected]("negative", [None, "blurry, low quality"])
+def test_weight_one_reproduces_encode_prompt_bit_for_bit(tiny_sdxl, negative):
+ import torch
+
+ from core.inference.diffusion_prompt_weighting import encode_weighted_prompts
+
+ with torch.inference_mode():
+ ref = tiny_sdxl.encode_prompt(
+ "a red fox in the snow",
+ device = "cpu",
+ do_classifier_free_guidance = True,
+ negative_prompt = negative,
+ )
+ mine = encode_weighted_prompts(
+ tiny_sdxl,
+ ["a red (fox:1.0) in the snow"],
+ [negative] if negative else None,
+ guidance_engaged = True,
+ )
+ assert torch.equal(ref[0], mine["prompt_embeds"])
+ assert torch.equal(ref[1], mine["negative_prompt_embeds"])
+ assert torch.equal(ref[2], mine["pooled_prompt_embeds"])
+ assert torch.equal(ref[3], mine["negative_pooled_prompt_embeds"])
+
+
+def test_clip_skip_reads_the_diffusers_layer(tiny_sdxl):
+ import torch
+
+ from core.inference.diffusion_prompt_weighting import encode_weighted_prompts
+ with torch.inference_mode():
+ for skip in (1, 2):
+ ref = tiny_sdxl.encode_prompt(
+ "a red fox", device = "cpu", do_classifier_free_guidance = False, clip_skip = skip
+ )
+ mine = encode_weighted_prompts(
+ tiny_sdxl, ["a red fox"], None, guidance_engaged = False, clip_skip = skip
+ )
+ assert torch.equal(ref[0], mine["prompt_embeds"])
+ assert torch.equal(ref[2], mine["pooled_prompt_embeds"])
+
+
+def test_weights_and_long_prompts_change_the_embeddings(tiny_sdxl):
+ import torch
+
+ from core.inference.diffusion_prompt_weighting import encode_weighted_prompts
+
+ with torch.inference_mode():
+ one = encode_weighted_prompts(tiny_sdxl, ["a red (fox:1.0)"], None, guidance_engaged = True)
+ more = encode_weighted_prompts(tiny_sdxl, ["a red (fox:1.5)"], None, guidance_engaged = True)
+ long = encode_weighted_prompts(
+ tiny_sdxl, ["a (fox:1.2), " + "snowy forest, " * 40], ["blurry"], guidance_engaged = True
+ )
+ assert not torch.equal(one["prompt_embeds"], more["prompt_embeds"])
+ assert torch.equal(one["pooled_prompt_embeds"], more["pooled_prompt_embeds"])
+ seq = long["prompt_embeds"].shape[1]
+ assert seq % 77 == 0 and seq > 77
+ assert long["negative_prompt_embeds"].shape == long["prompt_embeds"].shape
+ # Zero negatives for an empty negative prompt (force_zeros_for_empty_prompt), as the pipeline.
+ assert not one["negative_prompt_embeds"].any()
+
+
+def test_request_model_ranges():
+ from pydantic import ValidationError
+
+ from models.inference import DiffusionGenerateRequest
+
+ ok = DiffusionGenerateRequest(
+ prompt = "x", clip_skip = 2, cfg_rescale = 0.7, variation_seed = 5, variation_strength = 0.15
+ )
+ assert ok.clip_skip == 2 and ok.prompt_weighting is None
+ plain = DiffusionGenerateRequest(prompt = "x")
+ assert (
+ plain.clip_skip,
+ plain.cfg_rescale,
+ plain.variation_seed,
+ plain.variation_strength,
+ ) == (
+ None,
+ None,
+ None,
+ None,
+ )
+ for bad in (
+ {"clip_skip": 0},
+ {"clip_skip": 13},
+ {"cfg_rescale": 1.5},
+ {"cfg_rescale": -0.1},
+ {"variation_strength": 1.1},
+ {"variation_seed": -1},
+ {"variation_seed": 2**32},
+ ):
+ with pytest.raises(ValidationError):
+ DiffusionGenerateRequest(prompt = "x", **bad)
+
+
+def test_route_refuses_controls_the_engine_cannot_run():
+ from routes.inference import _sd_controls_kwargs
+
+ def native_generate(*, prompt):
+ return None
+
+ def diffusers_generate(*, prompt, sd_controls = None):
+ return None
+
+ plain = SimpleNamespace(
+ prompt_weighting = None,
+ clip_skip = None,
+ cfg_rescale = None,
+ variation_seed = None,
+ variation_strength = None,
+ )
+ assert _sd_controls_kwargs(native_generate, plain) == {}
+ with pytest.raises(ValueError, match = "diffusers engine only"):
+ _sd_controls_kwargs(native_generate, SimpleNamespace(**{**vars(plain), "clip_skip": 3}))
+ out = _sd_controls_kwargs(
+ diffusers_generate, SimpleNamespace(**{**vars(plain), "cfg_rescale": 0.7})
+ )
+ assert out["sd_controls"] == SdControls(cfg_rescale = 0.7)
+
+
+def test_route_refuses_weighting_off_on_an_engine_that_parses_attention():
+ # sd.cpp reads (word:1.5) itself, so prompt_weighting false cannot be honoured there.
+ from core.inference.sd_cpp_backend import SdCppDiffusionBackend
+ from routes.inference import _sd_controls_kwargs
+
+ class Literal:
+ def generate(self, *, prompt):
+ return None
+
+ off = SimpleNamespace(
+ prompt_weighting = False,
+ clip_skip = None,
+ cfg_rescale = None,
+ variation_seed = None,
+ variation_strength = None,
+ )
+ native = SdCppDiffusionBackend.__new__(SdCppDiffusionBackend)
+ with pytest.raises(ValueError, match = "prompt weighting off"):
+ _sd_controls_kwargs(native.generate, off)
+ # An engine that passes the prompt as plain text already renders it literally.
+ assert _sd_controls_kwargs(Literal().generate, off) == {}
+ assert (
+ _sd_controls_kwargs(
+ native.generate, SimpleNamespace(**{**vars(off), "prompt_weighting": None})
+ )
+ == {}
+ )
+
+
+def test_infotext_names_follow_webui():
+ from core.inference.image_gallery import _params_text
+
+ meta = {
+ "prompt": "a (fox:1.2)",
+ "steps": 25,
+ "guidance": 7.0,
+ "seed": 11,
+ "width": 1024,
+ "height": 1024,
+ "model": "m",
+ "variation_seed": 12,
+ "variation_strength": 0.15,
+ "clip_skip": 2,
+ "cfg_rescale": 0.7,
+ }
+ text = _params_text(meta)
+ assert "Variation seed: 12, Variation seed strength: 0.15" in text
+ assert "Clip skip: 2" in text and "CFG rescale: 0.7" in text
+ plain = _params_text(
+ {
+ k: v
+ for k, v in meta.items()
+ if k not in ("variation_seed", "variation_strength", "clip_skip", "cfg_rescale")
+ }
+ )
+ assert "Variation" not in plain and "Clip skip" not in plain and "CFG rescale" not in plain
+ assert "Emphasis" not in text and "Emphasis" not in plain
+
+
+def test_infotext_marks_literal_prompt_with_emphasis_none():
+ # webui reads "Emphasis: None" (opts.emphasis) to parse (fox:1.5) literally on replay.
+ from core.inference.image_gallery import _params_text
+
+ meta = {
+ "prompt": "a (fox:1.5)",
+ "steps": 20,
+ "guidance": 7.0,
+ "seed": 1,
+ "width": 64,
+ "height": 64,
+ }
+ assert ", Emphasis: None" in _params_text({**meta, "prompt_weighting": False})
+ assert "Emphasis" not in _params_text({**meta, "prompt_weighting": True})
+ assert "Emphasis" not in _params_text({**meta, "prompt_weighting": None})
+
+
+def test_long_plain_prompt_is_chunked_not_truncated(tiny_sdxl):
+ import torch
+
+ kwargs = {"prompt": "a red fox, " + "snowy forest, " * 40, "negative_prompt": "blurry"}
+ with torch.inference_mode():
+ weighted = apply_prompt_controls(tiny_sdxl, SDXL, None, kwargs, guidance = 7.0)
+ assert weighted is False # no weight syntax: chunked only
+ assert "prompt" not in kwargs and kwargs["prompt_embeds"].shape[1] % 77 == 0
+ assert kwargs["prompt_embeds"].shape[1] > 77
+ assert kwargs["negative_prompt_embeds"].shape == kwargs["prompt_embeds"].shape
+ short = {"prompt": "a red fox, snowy forest", "negative_prompt": "blurry"}
+ assert apply_prompt_controls(tiny_sdxl, SDXL, None, short, guidance = 7.0) is False
+ assert short == {"prompt": "a red fox, snowy forest", "negative_prompt": "blurry"}
+
+
+# --- SD 1.x layout (one CLIP encoder, final_layer_norm after a skipped layer) ---
+
+
[email protected](scope = "module")
+def tiny_sd15(tiny_sdxl):
+ """diffusers' SD 1.x ``StableDiffusionPipeline`` around the tiny SDXL's CLIP-L encoder: its own
+ ``encode_prompt`` (last_hidden_state, or a skipped layer plus final_layer_norm) is what is compared."""
+ diffusers = pytest.importorskip("diffusers")
+ return diffusers.StableDiffusionPipeline(
+ vae = tiny_sdxl.vae,
+ text_encoder = tiny_sdxl.text_encoder,
+ tokenizer = tiny_sdxl.tokenizer,
+ unet = tiny_sdxl.unet,
+ scheduler = tiny_sdxl.scheduler,
+ safety_checker = None,
+ feature_extractor = None,
+ requires_safety_checker = False,
+ )
+
+
+def test_sd15_encoder_matches_encode_prompt(tiny_sd15):
+ import torch
+
+ from core.inference.diffusion_prompt_weighting import encode_weighted_prompts
+ with torch.inference_mode():
+ ref = tiny_sd15.encode_prompt("a red fox", "cpu", 1, True, negative_prompt = "blurry")
+ mine = encode_weighted_prompts(
+ tiny_sd15, ["a red (fox:1.0)"], ["blurry"], guidance_engaged = True
+ )
+ assert torch.equal(ref[0], mine["prompt_embeds"])
+ assert torch.equal(ref[1], mine["negative_prompt_embeds"])
+ assert "pooled_prompt_embeds" not in mine
+ # A1111 "Clip skip 2" on SD 1.x = diffusers clip_skip 1: hidden_states[-2] then final_layer_norm.
+ skip = diffusers_clip_skip("sd15", 2)
+ assert skip == 1
+ try:
+ ref_skip = tiny_sd15.encode_prompt("a red fox", "cpu", 1, False, clip_skip = skip)[0]
+ except AttributeError:
+ # transformers 5 dropped ``text_model`` that diffusers' SD 1.x clip_skip reads: rebuild it by hand.
+ ids = tiny_sd15.tokenizer(
+ "a red fox",
+ padding = "max_length",
+ max_length = 77,
+ truncation = True,
+ return_tensors = "pt",
+ ).input_ids
+ enc = tiny_sd15.text_encoder
+ hidden = enc(ids, output_hidden_states = True)[-1][-(skip + 1)]
+ ref_skip = getattr(enc, "text_model", enc).final_layer_norm(hidden)
+ mine_skip = encode_weighted_prompts(
+ tiny_sd15, ["a red fox"], None, guidance_engaged = False, clip_skip = skip
+ )
+ assert torch.equal(ref_skip, mine_skip["prompt_embeds"])
+ assert not torch.equal(ref_skip, ref[0])
diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.py
index 10f3895870f..4aa771c0258 100644
--- a/studio/backend/tests/test_image_gallery.py
+++ b/studio/backend/tests/test_image_gallery.py
@@ -706,6 +706,10 @@ def test_connection_images_record_no_local_engine(monkeypatch):
remote = gallery.save(_img(), {**base, "model_kind": "connection"})
assert local["engine"] == "diffusers"
assert remote["engine"] is None
+ # Vary compares it with the loaded engine, so the served record must keep it (FastAPI drops undeclared fields).
+ from models.inference import GalleryImage
+
+ assert GalleryImage(**local).model_dump()["engine"] == "diffusers"
def test_parameters_chunk_carries_the_encoded_caption():
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 1707af81986..94ffc85f406 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -12,6 +12,7 @@ import {
MemoryEstimateRefusalError,
} from "./lib/memory-refusal";
import type { PromptEnhancement } from "./lib/prompt-enhance";
+import type { SdControlsCapability } from "./lib/sd-controls";
import type { ShiftControl } from "./lib/shift-choice";
// One Advanced control's resolved value and provenance, for the Advanced-panel badges. `value` is the engaged
@@ -69,6 +70,7 @@ export interface DiffusionStatus {
// Attention backend engaged via the diffusers dispatcher (e.g. "_native_cudnn"), null = default SDPA.
attention_backend?: string | null;
transformer_cache?: string | null;
+ baked_loras?: string[] | null;
vae_tiling?: boolean;
// Image workflows the loaded family supports (drives tab gating). Absent when nothing is loaded or
// on the native engine.
@@ -103,6 +105,7 @@ export interface DiffusionStatus {
prompt_hint?: string | null;
// "default" first; empty or absent = no choice (flow models, native / MLX engines, older backend).
samplers?: DiffusionSamplerOption[];
+ sd_controls?: SdControlsCapability | null;
// The Shift override the loaded flow-matching model takes; null/absent = no Shift control.
shift_control?: ShiftControl | null;
// Textual-inversion embedding format the loaded model takes ("sdxl"); null/absent = no embeddings.
@@ -269,6 +272,12 @@ export interface DiffusionGenerateRequest {
controlnet?: ControlNetSpecInput;
sampler?: string;
schedule_type?: string;
+ // Sent only off their defaults (lib/sd-controls); refused (400) for a model without them.
+ prompt_weighting?: boolean;
+ clip_skip?: number;
+ cfg_rescale?: number;
+ variation_seed?: number;
+ variation_strength?: number;
// `shift` for "sd3", `base_shift` / `max_shift` for "flux"; omitted = the model's own schedule.
shift?: number;
base_shift?: number;
@@ -336,6 +345,8 @@ export interface GalleryImage {
model: string | null;
// The load-time build. The repo id alone does not identify a pipeline (quant choice, torchao
// scheme, baked adapters), so these are what a recipe needs once the model is unloaded.
+ // Engine that ran it ("diffusers" | "sd_cpp" | "mlx"); the same seed draws different noise on each.
+ engine?: string | null;
model_kind?: string | null;
gguf_filename?: string | null;
transformer_quant?: string | null;
@@ -364,6 +375,11 @@ export interface GalleryImage {
// null = the model's shipped scheduler; schedule_type is the resolved schedule.
sampler?: string | null;
schedule_type?: string | null;
+ prompt_weighting?: boolean | null;
+ clip_skip?: number | null;
+ cfg_rescale?: number | null;
+ variation_seed?: number | null;
+ variation_strength?: number | null;
shift?: number | null;
base_shift?: number | null;
max_shift?: number | null;
@@ -576,17 +592,6 @@ export async function listDiffusionLoras(family?: string): Promise<DiffusionLora
return data.loras ?? [];
}
-/** List diffusion ControlNet models, optionally filtered to a model family. */
-export async function listDiffusionControlNets(
- family?: string,
-): Promise<DiffusionControlNetInfo[]> {
- const qs = family ? `?family=${encodeURIComponent(family)}` : "";
- const data = await parseJson<{ controlnets: DiffusionControlNetInfo[] }>(
- await authFetch(`/api/models/diffusion-controlnets${qs}`),
- );
- return data.controlnets ?? [];
-}
-
export interface GalleryPage {
images: GalleryImage[];
has_more: boolean;
@@ -602,31 +607,6 @@ export async function getGallery(offset = 0, limit = 50, archived = false): Prom
}
/** Pin/unpin or archive/restore one image; omitted flags are left alone. Returns the new record. */
-/** Move one image to just after `afterId` (null = front). The server also decides the pin. */
-export async function moveGalleryImage(id: string, afterId: string | null): Promise<GalleryImage> {
- return parseJson(
- await authFetch(`/api/inference/images/gallery/${id}/move`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ after_id: afterId }),
- }),
- );
-}
-
-/** Copy one image into a chat project's folder. */
-export async function addGalleryImageToProject(
- id: string,
- projectId: string,
-): Promise<{ path: string; already: boolean }> {
- return parseJson(
- await authFetch(`/api/inference/images/gallery/${id}/project`, {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ project_id: projectId }),
- }),
- );
-}
-
export async function setGalleryImageFlags(
id: string,
flags: { pinned?: boolean; archived?: boolean },
diff --git a/studio/frontend/src/features/images/images-page-api.ts b/studio/frontend/src/features/images/images-page-api.ts
new file mode 100644
index 00000000000..d9f243c1eae
--- /dev/null
+++ b/studio/frontend/src/features/images/images-page-api.ts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Kept out of api.ts, which the eager chat chunk carries: only the Images page calls these.
+import { authFetch } from "@/features/auth";
+import { type DiffusionControlNetInfo, type GalleryImage, parseJson } from "./api";
+
+export async function listDiffusionControlNets(
+ family?: string,
+): Promise<DiffusionControlNetInfo[]> {
+ const qs = family ? `?family=${encodeURIComponent(family)}` : "";
+ const data = await parseJson<{ controlnets: DiffusionControlNetInfo[] }>(
+ await authFetch(`/api/models/diffusion-controlnets${qs}`),
+ );
+ return data.controlnets ?? [];
+}
+
+/** Move one image to just after `afterId` (null = front). The server also decides the pin. */
+export async function moveGalleryImage(id: string, afterId: string | null): Promise<GalleryImage> {
+ return parseJson(
+ await authFetch(`/api/inference/images/gallery/${id}/move`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ after_id: afterId }),
+ }),
+ );
+}
+
+export async function addGalleryImageToProject(
+ id: string,
+ projectId: string,
+): Promise<{ path: string; already: boolean }> {
+ return parseJson(
+ await authFetch(`/api/inference/images/gallery/${id}/project`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ project_id: projectId }),
+ }),
+ );
+}
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 79f84f8ab12..2a4a211e84b 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -287,10 +287,7 @@ import {
getDiffusionStatus,
getGallery,
getGenerateProgress,
- listDiffusionControlNets,
listDiffusionLoras,
- addGalleryImageToProject,
- moveGalleryImage,
setGalleryImageFlags,
getDiffusionDownloadPlan,
loadDiffusionModel,
@@ -319,6 +316,8 @@ import {
type ImportedRecipe,
importedRecipe,
importNotApplied,
+ literalPromptNotApplied,
+ variationApplies,
textualInversionLabel,
isGenerationParametersPaste,
loraTagNames,
@@ -366,6 +365,7 @@ import {
withEmbeddingTrigger,
} from "./lib/shift-choice";
import { listDiffusionEmbeddings } from "./embeddings-api";
+import { addGalleryImageToProject, listDiffusionControlNets, moveGalleryImage } from "./images-page-api";
import { EmbeddingsField, ShiftField } from "./shift-embeddings-controls";
import {
enhancePromptRequestField,
@@ -402,6 +402,19 @@ import {
sendsEngine,
withEngineChoice,
} from "./lib/engine-choice";
+import {
+ canVaryImage,
+ DEFAULT_SD_CONTROLS,
+ FALLBACK_VARY_STRENGTH,
+ parseVariationSeed,
+ randomVariationSeed,
+ sdControlsFromRecipe,
+ sdControlsNotRestored,
+ sdControlsRecipeRows,
+ sdControlsRequestFields,
+ type SdControlsChoice,
+ varyBlockedReason,
+} from "./lib/sd-controls";
import { useNavigate, useSearch } from "@tanstack/react-router";
import { useStagedDownload, type StagedDownloadEntry } from "@/features/hub/download-manager";
import { DiffusionTrainPanel } from "./train/diffusion-train-panel";
@@ -1409,6 +1422,9 @@ function RecipePopover({
value={samplerRecipeLabel(image.sampler, image.schedule_type) ?? ""}
/>
) : null}
+ {sdControlsRecipeRows(image).map(([label, value]) => (
+ <RecipeRow key={label} label={label} value={value} />
+ ))}
{!viaConnection && <RecipeRow label="Seed" value={String(image.seed)} mono />}
</div>
<div className="shrink-0 border-t border-border/60 px-3 py-2.5">
@@ -1795,6 +1811,9 @@ export function ImagesPage({
const [livePreviewOff, setLivePreviewOff] = usePersistedToggle("unsloth_images_live_preview_off");
const livePreview = !livePreviewOff;
const oversizedOnce = useRef(false);
+ const [sdChoice, setSdChoice] = useState<SdControlsChoice>(DEFAULT_SD_CONTROLS);
+ // Set by Vary for the next Generate only: the seed to keep and the variation seed to slerp toward.
+ const varyOnce = useRef<{ seed: number; variationSeed: number; strength: number } | null>(null);
// Set only around a confirmed Load anyway retry.
const oversizedLoadOnce = useRef(false);
const lastLoadAttempt = useRef<{ overrideSent: boolean; retry: () => void } | null>(null);
@@ -2194,6 +2213,9 @@ export function ImagesPage({
// Refresh the ControlNet options when the loaded family changes, and clear a stale selection.
const controlnetSupported = Boolean(status?.loaded && status?.supports_controlnet);
const controlnetCapable = !external && controlnetSupported;
+ // What Create sends as a ControlNet run (strength 0 runs plain text-to-image): its SDXL pipeline has no CFG rescale.
+ const controlnetEngaged =
+ controlnetCapable && Boolean(controlnetId && controlImage) && workflow === "create" && controlStrength > 0;
const negativeCapable = !external && status?.supports_negative_prompt !== false;
// A model that reads no guidance here runs at its default, which the request and recipe then carry.
const guidanceFixed = !external && status?.loaded === true && status.supports_guidance === false;
@@ -2210,6 +2232,7 @@ export function ImagesPage({
const livePreviewUnavailable =
!external && status?.loaded === true && status.supports_live_preview === false;
const samplerOptions = (!external && status?.loaded ? status.samplers : null) ?? NO_SAMPLERS;
+ const sdCaps = (!external && status?.loaded ? status.sd_controls : null) ?? null;
// The checkpoint, not residentDefaults: community SDXL files all share the SDXL base repo.
const samplerModelKey = status?.loaded ? `${status.repo_id ?? ""}|${status.gguf_filename ?? ""}` : "";
const samplerChoice = normalizeSamplerChoice(
@@ -2781,7 +2804,8 @@ export function ImagesPage({
[dropFromStrip],
);
- const restoreSettings = useCallback((image: GalleryImage, exactCaption = false) => {
+ // `via`: "import" names what was not applied in its own toast; "vary" restores silently for the Vary run.
+ const restoreSettings = useCallback((image: GalleryImage, exactCaption = false, via: "restore" | "import" | "vary" = "restore") => {
const viaConnection = image.model_kind === "connection";
// Negative prompt only applies when guidance>0; do not restore a hidden value.
const restoredNegative = image.guidance > 0 ? (image.negative_prompt ?? "") : "";
@@ -2797,6 +2821,7 @@ export function ImagesPage({
};
if (samplerOptions.length > 0) changeSamplerChoice(restoredChoice);
else setRestoredSampler(restoredChoice);
+ setSdChoice(sdControlsFromRecipe(image));
// Restore from the BASE batch seed, not this image's derived seed, or a replay with batch_size
// advances it again.
if (!viaConnection) setSeed(String(image.batch_seed ?? image.seed));
@@ -2865,12 +2890,28 @@ export function ImagesPage({
// Say so, rather than letting a conditioned image restore as a plain Create that generates something unrelated.
const conditioned =
restoreInputsNote(image) ?? CONDITIONED_WORKFLOW_INPUTS[image.workflow ?? ""];
+ if (via === "vary") return;
if (conditioned) {
toast.success(`Settings restored. Add ${conditioned} again to reproduce this image.`, rescaled);
} else {
toast.success("Settings restored to inputs", rescaled);
}
- }, [changeSamplerChoice, samplerOptions.length, setEnhanceOverride, setPromptFor, setShiftValue, setWorkflow, sizeLimits]);
+ // sd.cpp has no off switch for (word:1.2) syntax, so a literal-prompt run would replay weighted.
+ const literal =
+ via === "restore"
+ ? literalPromptNotApplied(image, Boolean(status?.loaded && isNativeEngineStatus(status)), sdCaps)
+ : null;
+ // Generate omits (or clamps) the SD controls this load lacks, so name them rather than claim a full restore.
+ const notApplied = [
+ ...(literal ? [literal] : []),
+ ...(via === "restore" && status?.loaded ? sdControlsNotRestored(image, sdCaps) : []),
+ ];
+ if (notApplied.length > 0) {
+ toast.warning("Some settings in this image were not applied", {
+ description: `Not applied: ${notApplied.join(", ")}. This model or build does not support them.`,
+ });
+ }
+ }, [changeSamplerChoice, samplerOptions.length, sdCaps, setEnhanceOverride, setPromptFor, setShiftValue, setWorkflow, sizeLimits, status]);
// Like dropping an image on ComfyUI: Unsloth's recipe first, else AUTOMATIC1111 / ComfyUI metadata.
const importFileRef = useRef<HTMLInputElement>(null);
@@ -2943,17 +2984,24 @@ export function ImagesPage({
: recipe;
const settings = recipeSettings(merged, { width, height, steps, guidance });
const localized = settings.localized_edit;
- restoreSettings({
- ...settings,
- id: "",
- url: "",
- seed: settings.seed ?? 0,
- batch_index: 0,
- created_at: 0,
- localized_edit:
- localized === "annotate" || localized === "paint" || localized === "mask" ? localized : null,
- });
+ restoreSettings(
+ {
+ ...settings,
+ id: "",
+ url: "",
+ seed: settings.seed ?? 0,
+ batch_index: 0,
+ created_at: 0,
+ localized_edit:
+ localized === "annotate" || localized === "paint" || localized === "mask" ? localized : null,
+ },
+ false,
+ "import",
+ );
if (settings.seed === null && settings.model_kind !== "connection") setSeed("");
+ // A variation seed this model or workflow cannot apply is named below, not left armed (the backend refuses it).
+ if (!variationApplies(merged, sdCaps))
+ setSdChoice((c) => ({ ...c, variationSeed: "", variationStrength: 0 }));
const loaded = status?.display_repo_id ?? status?.repo_id ?? null;
const quant = typeof recipe.gguf_filename === "string" ? recipe.gguf_filename : null;
if (settings.model && settings.model !== loaded) {
@@ -2981,6 +3029,7 @@ export function ImagesPage({
recipe,
status?.loaded ? status.family : null,
Boolean(status?.loaded && isNativeEngineStatus(status)),
+ sdCaps,
),
...tagged.unmatched.map((name) => `LoRA ${name} (not installed)`),
...lorasOverLimit(merged).map((id) => `LoRA ${id} (over the ${MAX_LORAS} adapter limit)`),
@@ -3049,7 +3098,7 @@ export function ImagesPage({
}
return true;
},
- [restoreSettings, width, height, steps, guidance, status, samplerOptions, availableLoras, loraCapable, loraCatalogFamily, embeddingsEnabled, shiftControl],
+ [restoreSettings, width, height, steps, guidance, status, samplerOptions, availableLoras, loraCapable, loraCatalogFamily, sdCaps, embeddingsEnabled, shiftControl],
);
const importSettingsFromFile = useCallback(
async (file: File | Uint8Array) => {
@@ -5068,6 +5117,8 @@ export function ImagesPage({
// Consume before any early return so it never leaks into a later run.
const allowOversizedSent = allowOversizedField(allowOversized, oversizedOnce.current) === true;
oversizedOnce.current = false;
+ const vary = sdCaps?.variation_seed ? varyOnce.current : null;
+ varyOnce.current = null;
if (!prompt.trim()) {
toast.error("Prompt is empty");
return;
@@ -5184,7 +5235,9 @@ export function ImagesPage({
}
// Resolve a base seed up front, so each sequential image gets a distinct, reproducible seed (base + i).
let baseSeed: number;
- if (seed.trim() && !external) {
+ if (vary) {
+ baseSeed = vary.seed;
+ } else if (seed.trim() && !external) {
const n = Number(seed);
if (!Number.isInteger(n) || n < 0 || n > Number.MAX_SAFE_INTEGER) {
toast.error("Seed must be a non-negative integer");
@@ -5194,6 +5247,17 @@ export function ImagesPage({
} else {
baseSeed = Math.floor(Math.random() * 2 ** 32);
}
+ const runChoice: SdControlsChoice = vary
+ ? { ...sdChoice, variationSeed: String(vary.variationSeed), variationStrength: vary.strength }
+ : sdChoice;
+ const varying = Boolean(sdCaps?.variation_seed) && workflow === "create" && runChoice.variationStrength > 0;
+ if (varying && runChoice.variationSeed.trim() && parseVariationSeed(runChoice.variationSeed) === null) {
+ toast.error("Variation seed must be an integer from 0 to 4294967295");
+ return;
+ }
+ const baseVariationSeed = varying
+ ? (parseVariationSeed(runChoice.variationSeed) ?? randomVariationSeed())
+ : null;
// Snap to the model's grid and bounds so a half-typed value cannot 400.
const sent = unifiedEditRun ? editSize : fitSize(width, height, sizeLimits);
@@ -5283,7 +5347,7 @@ export function ImagesPage({
);
// Offset runs by the batch size: the native engine seeds image j at seed+j, so a +1 offset
// would regenerate batch-mates.
- const runBaseSeed = baseSeed + i * batchSize;
+ const runBaseSeed = varying ? baseSeed : baseSeed + i * batchSize;
let res: DiffusionGenerateResponse;
try {
res = await generateDiffusionImage({
@@ -5306,6 +5370,13 @@ export function ImagesPage({
allow_oversized: allowOversizedSent ? true : undefined,
live_preview: livePreview,
...samplerRequestFields(samplerOptions, samplerChoice),
+ ...sdControlsRequestFields(
+ sdCaps,
+ runChoice,
+ baseVariationSeed !== null ? (baseVariationSeed + i * batchSize) % 2 ** 32 : null,
+ varying,
+ !controlnetEngaged,
+ ),
...shiftRequestFields(shiftControl, shiftValue),
...enhancePromptRequestField(upsamplerCapability, enhanceOn),
...condFields,
@@ -5381,7 +5452,10 @@ export function ImagesPage({
duration: 20_000,
action: {
label: GENERATE_ANYWAY_LABEL,
- onClick: () => setOversizedRetryQueued(true),
+ onClick: () => {
+ varyOnce.current = vary;
+ setOversizedRetryQueued(true);
+ },
},
});
} else if (report)
@@ -5404,7 +5478,7 @@ export function ImagesPage({
setGenStep(null);
setStopping(false);
}
- }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeEngaged, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, shiftControl, shiftValue, width, height, steps, effectiveGuidance, seed, batchSize, count, workflow, initImage, maskImage, strength, maskBlur, inpaintOnlyMasked, inpaintPadding, status?.supports_inpaint_only_masked, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, multiImageEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
+ }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeEngaged, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, sdCaps, sdChoice, shiftControl, shiftValue, width, height, steps, effectiveGuidance, seed, batchSize, count, workflow, initImage, maskImage, strength, maskBlur, inpaintOnlyMasked, inpaintPadding, status?.supports_inpaint_only_masked, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetEngaged, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, multiImageEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
// Stop the in-flight generation. Latch FIRST, so a multi-run request stops even if the POST
// races the run that is already finishing.
@@ -5519,6 +5593,28 @@ export function ImagesPage({
});
}, [oversizedRetryQueued, busy, handleGenerateWithRecall]);
+ const canVary =
+ !external && Boolean(status?.loaded && sdCaps?.variation_seed) && workflow === "create" && selected !== null &&
+ canVaryImage(selected, status?.display_repo_id ?? status?.repo_id ?? null);
+ const varyBlocked = canVary && selected ? varyBlockedReason(selected, loraCapable, sdCaps, status ?? {}) : null;
+ // Vary runs the SELECTED image's recipe, not the form: restore it, then generate once the form has rendered.
+ const [varyQueued, setVaryQueued] = useState(false);
+ const handleVary = useCallback(() => {
+ if (busy !== null || !selected || !sdCaps?.variation_seed || varyBlocked) return;
+ varyOnce.current = {
+ seed: selected.seed,
+ variationSeed: randomVariationSeed(),
+ strength: sdCaps.vary_strength ?? FALLBACK_VARY_STRENGTH,
+ };
+ restoreSettings(selected, true, "vary");
+ setVaryQueued(true);
+ }, [busy, restoreSettings, sdCaps, selected, varyBlocked]);
+ useEffect(() => {
+ if (!varyQueued || busy !== null) return;
+ setVaryQueued(false);
+ void handleGenerate();
+ }, [varyQueued, busy, handleGenerate]);
+
useEffect(() => {
const pending = pendingRecalledGeneration.current;
if (!pending) return;
@@ -5787,6 +5883,80 @@ export function ImagesPage({
aria-label="Live preview"
/>
</div>
+ {sdCaps && (
+ <>
+ {sdCaps.prompt_weighting && (
+ <div className="flex items-center justify-between">
+ <span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">
+ Prompt weighting
+ <InfoHint>
+ AUTOMATIC1111 syntax: (word:1.2) weights a word, (word) is 1.1x, [word] is 1/1.1x,
+ \( and \) type a literal bracket, BREAK starts a new 75-token chunk. Prompts past 75
+ tokens are encoded in chunks instead of cut off. A prompt without this syntax that fits in
+ 75 tokens renders exactly as before. Off reads brackets as plain text.
+ </InfoHint>
+ </span>
+ <Switch
+ checked={sdChoice.promptWeighting}
+ onCheckedChange={(on) => setSdChoice((c) => ({ ...c, promptWeighting: on }))}
+ aria-label="Prompt weighting"
+ />
+ </div>
+ )}
+ {sdCaps.clip_skip && (
+ <SliderField
+ label="CLIP skip"
+ hint={`AUTOMATIC1111 numbering: 1 uses the model's own text-encoder layer. SDXL already reads the penultimate layer, so 1 and 2 render the same (Pony and Illustrious cards often say 2). ${sdCaps.clip_skip.max > 2 ? "3 and up read earlier layers." : ""}`}
+ value={sdChoice.clipSkip}
+ min={1}
+ max={sdCaps.clip_skip.max}
+ step={1}
+ onChange={(v) => setSdChoice((c) => ({ ...c, clipSkip: v }))}
+ />
+ )}
+ {sdCaps.cfg_rescale && (
+ <SliderField
+ label="CFG rescale"
+ hint={
+ controlnetEngaged
+ ? "Not available with a ControlNet: the SDXL ControlNet pipeline has no CFG rescale. Set the ControlNet strength to 0 or clear it to use this."
+ : sdCaps.cfg_rescale.recommended
+ ? `Rescales the guided prediction to fix overexposure. This is a v-prediction model: ${sdCaps.cfg_rescale.recommended} is the value from the zero-terminal-SNR paper. 0 = off.`
+ : "Rescales the guided prediction to fix overexposure, mostly for v-prediction checkpoints (0.7 there). 0 = off."
+ }
+ value={sdChoice.cfgRescale}
+ min={0}
+ max={sdCaps.cfg_rescale.max}
+ step={0.05}
+ onChange={(v) => setSdChoice((c) => ({ ...c, cfgRescale: v }))}
+ disabled={controlnetEngaged}
+ />
+ )}
+ {sdCaps.variation_seed && workflow === "create" && (
+ <>
+ <SliderField
+ label="Variation strength"
+ hint="Blends the seed's starting noise toward the variation seed's (AUTOMATIC1111 Variation seed): 0 = off, small values keep the composition, 1 = the variation seed alone. Text to image only. A batch keeps one seed and steps the variation seed."
+ value={sdChoice.variationStrength}
+ min={0}
+ max={1}
+ step={0.01}
+ onChange={(v) => setSdChoice((c) => ({ ...c, variationStrength: v }))}
+ />
+ {sdChoice.variationStrength > 0 && (
+ <Field label="Variation seed" hint="Leave empty for a random variation seed each run.">
+ <Input
+ placeholder="Random if empty"
+ value={sdChoice.variationSeed}
+ onChange={(e) => setSdChoice((c) => ({ ...c, variationSeed: e.target.value }))}
+ aria-label="Variation seed"
+ />
+ </Field>
+ )}
+ </>
+ )}
+ </>
+ )}
<LoadedBuildSummary status={status} />
</>
);
@@ -6833,6 +7003,31 @@ export function ImagesPage({
>
Generate
</Button>
+ {canVary && (
+ <Tooltip>
+ <TooltipTrigger asChild={true}>
+ {/* A disabled button gets no pointer events; the span keeps the reason reachable. */}
+ <span className="relative z-10 inline-flex">
+ <Button
+ className="h-11 px-5"
+ variant="secondary"
+ disabled={busy !== null || varyBlocked !== null}
+ onClick={handleVary}
+ >
+ Vary
+ </Button>
+ </span>
+ </TooltipTrigger>
+ <TooltipContent>
+ {varyBlocked ?? (
+ <>
+ Restore the selected image&apos;s settings and re-run its seed with a new variation seed at
+ strength {sdCaps?.vary_strength ?? FALLBACK_VARY_STRENGTH}: same composition, small changes
+ </>
+ )}
+ </TooltipContent>
+ </Tooltip>
+ )}
{!external && status?.loaded && (canReapply || residentReloadTarget(status) !== null) && (
<Tooltip>
<TooltipTrigger asChild={true}>
diff --git a/studio/frontend/src/features/images/lib/image-metadata.ts b/studio/frontend/src/features/images/lib/image-metadata.ts
index 4163ae41c58..6f6b163a55e 100644
--- a/studio/frontend/src/features/images/lib/image-metadata.ts
+++ b/studio/frontend/src/features/images/lib/image-metadata.ts
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { SAMPLER_LABELS, SCHEDULE_LABELS } from "./sampler-choice.ts";
+import type { SdControlsCapability } from "./sd-controls.ts";
/** Generation settings in exported images, read back from dropped ones. PNG: `unsloth` + `parameters` text chunks;
* JPEG / WebP: EXIF UserComment (`parameters`, read by AUTOMATIC1111 / Forge / Civitai) + MakerNote (recipe). */
@@ -795,8 +796,10 @@ export function parseA1111Parameters(text: string): ImportedRecipe | null {
}
else if (key === "shift" && n !== undefined) out.shift = n;
// A1111 writes Emphasis only when it is not the default; "Original" is what Studio's prompt encoding does.
+ // "None" skips the parser entirely (brackets are prompt text): Studio's prompt weighting off.
else if (key === "emphasis") {
- if (value && value.toLowerCase() !== "original")
+ if (value.toLowerCase() === "none") out.prompt_weighting = false;
+ else if (value && value.toLowerCase() !== "original")
notApplied.push(`${label}: ${value}`);
} else if (key.startsWith("hires")) hires[label] = value;
else if (key === "sampler") {
@@ -903,6 +906,12 @@ export interface RestorableRecipe {
model_kind?: string;
sampler?: string;
schedule_type?: string;
+ // SDXL controls (lib/sd-controls); absent = off, except prompt_weighting (absent = on).
+ prompt_weighting?: boolean;
+ clip_skip?: number;
+ cfg_rescale?: number;
+ variation_seed?: number;
+ variation_strength?: number;
shift?: number;
base_shift?: number;
max_shift?: number;
@@ -933,6 +942,10 @@ export const APPLIED_RECIPE_KEYS = [
"localized_edit",
"sampler",
"schedule_type",
+ "clip_skip",
+ "cfg_rescale",
+ "variation_seed",
+ "variation_strength",
"shift",
"base_shift",
"max_shift",
@@ -964,6 +977,7 @@ export const INFO_RECIPE_KEYS = [
"engine",
"generator",
"schema_version",
+ "prompt_weighting",
// The prompt names the embeddings it used, so restoring the prompt is what applies them.
"embeddings",
] as const;
@@ -973,10 +987,6 @@ export const RECIPE_SCHEMA_VERSION = 1;
/** Read from AUTOMATIC1111 text but not sent by this build. */
export const PENDING_RECIPE_KEYS = [
- "clip_skip",
- "cfg_rescale",
- "variation_seed",
- "variation_strength",
"hires",
"lora_tags",
"prompt_emphasis",
@@ -1023,18 +1033,24 @@ const bounded = (r: ImportedRecipe, key: keyof typeof RECIPE_BOUNDS): number | u
};
/** `family`: the loaded model's family, when one is loaded. `parsesEmphasis`: the loaded engine reads
- * A1111 attention syntax itself (sd.cpp's parse_prompt_attention); diffusers and MLX do not. */
+ * A1111 attention syntax itself (sd.cpp's parse_prompt_attention); diffusers and MLX do not.
+ * `sdControls`: status `sd_controls`; only the controls it lacks are named. */
export function importNotApplied(
r: ImportedRecipe,
family?: string | null,
parsesEmphasis = false,
+ sdControls?: SdControlsCapability | null,
): string[] {
const out: string[] = [];
+ const literal = literalPromptNotApplied(r, parsesEmphasis, sdControls);
+ if (literal) out.push(literal);
+ // Weighting off reads brackets and escapes as typed, which an engine without the parser does too.
+ if (sdControls?.prompt_weighting || r.prompt_weighting === false) parsesEmphasis = true;
if (!parsesEmphasis && Array.isArray(r.prompt_emphasis) && r.prompt_emphasis.length > 0) {
const groups = r.prompt_emphasis.filter((g): g is string => typeof g === "string");
const shown = groups.slice(0, 3).join(", ") + (groups.length > 3 ? ", ..." : "");
out.push(`prompt emphasis (${shown}) is passed as plain text`);
- }
+ } else if (!parsesEmphasis && r.prompt_weighting) out.push("prompt weighting");
if (
!parsesEmphasis &&
Array.isArray(r.prompt_escaped_brackets) &&
@@ -1054,14 +1070,23 @@ export function importNotApplied(
// SDXL already encodes from the penultimate layer, which is what A1111 / Forge render for Clip skip
// 1 and 2 (Forge minimal_clip_skip=2; A1111 skips CLIP-G never, CLIP-L only with sdxl_clip_l_skip).
// Elsewhere Studio applies no clip skip, so anything past A1111's default 1 is not applied.
- if (clipSkip !== undefined && clipSkip > (family === "sdxl" ? 2 : 1))
- out.push(`Clip skip ${clipSkip}`);
+ const clipMax = sdControls?.clip_skip?.max;
+ if (clipSkip !== undefined && clipSkip > (family === "sdxl" ? 2 : 1)) {
+ if (clipMax === undefined) out.push(`Clip skip ${clipSkip}`);
+ else if (clipSkip > clipMax) out.push(`Clip skip ${clipSkip} (clamped to ${clipMax})`);
+ }
const rescale = num(r.cfg_rescale);
- if (rescale !== undefined && rescale !== 0)
- out.push(`CFG rescale ${rescale}`);
+ if (rescale !== undefined && rescale !== 0) {
+ // recipeSettings clamps to the request's [0, 1], the request to the model's max.
+ const hi = sdControls?.cfg_rescale ? Math.min(1, sdControls.cfg_rescale.max) : undefined;
+ const sent = hi === undefined ? undefined : clamp(rescale, 0, hi);
+ if (sent === undefined) out.push(`CFG rescale ${rescale}`);
+ else if (sent !== rescale) out.push(`CFG rescale ${rescale} (clamped to ${sent})`);
+ }
const variation = num(r.variation_seed);
const variationStrength = num(r.variation_strength);
- if (variation !== undefined && variationStrength !== 0) {
+ if (variationApplies(r, sdControls)) {
+ } else if (variation !== undefined && variationStrength !== 0) {
out.push(
`variation seed ${variation}${variationStrength !== undefined ? ` (strength ${variationStrength})` : ""}`,
);
@@ -1086,6 +1111,33 @@ export function importNotApplied(
return out;
}
+/** A run that turned prompt weighting off (its prompt has attention syntax) cannot be replayed literally on an
+ * engine that always parses that syntax itself (sd.cpp) and has no off switch. */
+export function literalPromptNotApplied(
+ r: ImportedRecipe | { prompt_weighting?: boolean | null },
+ parsesEmphasis: boolean,
+ sdControls?: SdControlsCapability | null,
+): string | null {
+ return r.prompt_weighting === false && parsesEmphasis && !sdControls?.prompt_weighting
+ ? "prompt weighting off (the native sd.cpp engine always reads (word:1.2) syntax)"
+ : null;
+}
+
+/** The variation seed restores only on a model that offers it, for text-to-image (and ControlNet):
+ * the backend refuses it for a workflow that derives its noise from a source image. */
+export function variationApplies(
+ r: ImportedRecipe,
+ sdControls?: SdControlsCapability | null,
+): boolean {
+ const strength = num(r.variation_strength);
+ if (!sdControls?.variation_seed || strength === undefined || strength <= 0) return false;
+ // Out of the request's bounds (32-bit seed, strength up to 1) it would restore changed: a random seed, a clamped strength.
+ const seed = num(r.variation_seed);
+ if (strength > 1 || (seed !== undefined && !(Number.isInteger(seed) && seed >= 0 && seed < 2 ** 32))) return false;
+ const workflow = typeof r.workflow === "string" ? r.workflow : undefined;
+ return workflow === undefined || workflow === "txt2img" || workflow === "create" || workflow === "controlnet";
+}
+
/** How importNotApplied names a TI hashes embedding; the page re-labels the ones it matched. */
export function textualInversionLabel(name: string): string {
return `textual inversion ${name}`;
@@ -1180,6 +1232,14 @@ export function recipeSettings(
model_kind: r.model_kind === "connection" ? "connection" : undefined,
sampler: str(r.sampler),
schedule_type: str(r.schedule_type),
+ prompt_weighting: r.prompt_weighting === false ? false : undefined,
+ clip_skip: clamp(int(r.clip_skip), 1, 12),
+ cfg_rescale: clamp(num(r.cfg_rescale), 0, 1),
+ variation_seed: (() => {
+ const n = int(r.variation_seed);
+ return n !== undefined && n >= 0 && n < 2 ** 32 ? n : undefined;
+ })(),
+ variation_strength: clamp(num(r.variation_strength), 0, 1),
// The request's bounds: shift in (0, 100], base / max shift in [0, 100].
shift: (() => {
const n = num(r.shift);
diff --git a/studio/frontend/src/features/images/lib/sd-controls.ts b/studio/frontend/src/features/images/lib/sd-controls.ts
new file mode 100644
index 00000000000..f7cb4312937
--- /dev/null
+++ b/studio/frontend/src/features/images/lib/sd-controls.ts
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+export interface SdControlsCapability {
+ prompt_weighting?: boolean;
+ clip_skip?: {
+ min: number;
+ max: number;
+ default: number;
+ same_as_default_through?: number;
+ } | null;
+ cfg_rescale?: {
+ min: number;
+ max: number;
+ default: number;
+ recommended?: number | null;
+ } | null;
+ variation_seed?: boolean;
+ vary_strength?: number;
+ prediction_type?: string | null;
+}
+
+export interface SdControlsChoice {
+ promptWeighting: boolean;
+ clipSkip: number;
+ cfgRescale: number;
+ // Text field: empty = a random variation seed per Generate.
+ variationSeed: string;
+ variationStrength: number;
+}
+
+export const DEFAULT_SD_CONTROLS: SdControlsChoice = {
+ promptWeighting: true,
+ clipSkip: 1,
+ cfgRescale: 0,
+ variationSeed: "",
+ variationStrength: 0,
+};
+
+export const FALLBACK_VARY_STRENGTH = 0.15;
+
+export interface SdControlsRequest {
+ prompt_weighting?: boolean;
+ clip_skip?: number;
+ cfg_rescale?: number;
+ variation_seed?: number;
+ variation_strength?: number;
+}
+
+export function randomVariationSeed(): number {
+ return Math.floor(Math.random() * 2 ** 32);
+}
+
+export function parseVariationSeed(text: string): number | null {
+ const trimmed = text.trim();
+ if (!trimmed) return null;
+ const n = Number(trimmed);
+ return Number.isInteger(n) && n >= 0 && n < 2 ** 32 ? n : null;
+}
+
+/** Request fields for the choice, only those off their default; nothing when the model lacks them.
+ * `variation`: false on image-conditioned workflows, whose noise comes from the source (the backend 400s).
+ * `rescale`: false with a ControlNet engaged, whose SDXL pipeline has no guidance_rescale (the backend 400s). */
+export function sdControlsRequestFields(
+ capability: SdControlsCapability | null | undefined,
+ choice: SdControlsChoice,
+ variationSeed?: number | null,
+ variation = true,
+ rescale = true,
+): SdControlsRequest {
+ if (!capability) return {};
+ const out: SdControlsRequest = {};
+ if (capability.prompt_weighting && !choice.promptWeighting)
+ out.prompt_weighting = false;
+ const clip = capability.clip_skip;
+ if (clip && choice.clipSkip > 1)
+ out.clip_skip = Math.min(Math.round(choice.clipSkip), clip.max);
+ const rescaleCap = capability.cfg_rescale;
+ if (rescale && rescaleCap && choice.cfgRescale > 0) {
+ out.cfg_rescale = Math.min(choice.cfgRescale, rescaleCap.max);
+ }
+ if (variation && capability.variation_seed && choice.variationStrength > 0) {
+ out.variation_strength = Math.min(choice.variationStrength, 1);
+ const seed = variationSeed ?? parseVariationSeed(choice.variationSeed);
+ if (seed !== null && seed !== undefined) out.variation_seed = seed;
+ }
+ return out;
+}
+
+export function varyChoice(
+ capability: SdControlsCapability | null | undefined,
+ choice: SdControlsChoice,
+ variationSeed: number = randomVariationSeed(),
+): SdControlsChoice {
+ return {
+ ...choice,
+ variationSeed: String(variationSeed),
+ variationStrength: capability?.vary_strength ?? FALLBACK_VARY_STRENGTH,
+ };
+}
+
+/** Vary restores the selected image's recipe for its run, so it is offered only where that recipe replays on the
+ * loaded model: a text-to-image run (a conditioned one needs inputs the gallery does not keep) from that model. */
+export function canVaryImage(
+ image: { model?: string | null; workflow?: string | null },
+ loadedModel: string | null,
+): boolean {
+ const workflow = image.workflow ?? "txt2img";
+ if (workflow !== "txt2img" && workflow !== "create") return false;
+ return !image.model || image.model === loadedModel;
+}
+
+type VaryBuild = {
+ engine?: string | null;
+ model_kind?: string | null;
+ gguf_filename?: string | null;
+ transformer_quant?: string | null;
+ text_encoder_quant?: string | null;
+ attention_backend?: string | null;
+ transformer_cache?: string | null;
+ baked_loras?: string[] | null;
+};
+const VARY_BUILD_KEYS = [
+ "engine",
+ "model_kind",
+ "gguf_filename",
+ "transformer_quant",
+ "text_encoder_quant",
+ "attention_backend",
+ "transformer_cache",
+] as const;
+// Baked adapters are a set (the bake order is not recorded as meaningful); absent reads as none.
+const bakedKey = (b: VaryBuild) => [...new Set(b.baked_loras ?? [])].sort().join("\n");
+
+/** Why `image` did not provably come off the loaded build, else null; a record that cannot be verified
+ * (no engine or no schema_version) is blocked rather than guessed. */
+export function varyBuildMismatch(image: VaryBuild & { schema_version?: number | null }, loaded: VaryBuild): string | null {
+ if (!image.schema_version || !image.engine)
+ return "This image does not record which engine and build made it, so Vary cannot check it matches the current load.";
+ const differs: string[] = VARY_BUILD_KEYS.filter((k) => (image[k] ?? null) !== (loaded[k] ?? null));
+ if (bakedKey(image) !== bakedKey(loaded)) differs.push("baked_loras");
+ return differs.length
+ ? `This image came from a different build (${differs.map((k) => k.replace(/_/g, " ")).join(", ")}) than the current load, so the same seed would not give a nearby image. Load that build to vary it.`
+ : null;
+}
+
+/** Why Vary cannot replay `image` as recorded on the current load (it would silently drop a part), else null. */
+export function varyBlockedReason(
+ image: VaryBuild & {
+ schema_version?: number | null;
+ loras?: string[] | null;
+ prompt_weighting?: boolean | null;
+ clip_skip?: number | null;
+ cfg_rescale?: number | null;
+ variation_strength?: number | null;
+ },
+ loraCapable: boolean,
+ capability: SdControlsCapability | null | undefined,
+ loaded: VaryBuild,
+): string | null {
+ const build = varyBuildMismatch(image, loaded);
+ if (build) return build;
+ // Its noise was a blend of its seed and its variation seed, which no single seed reproduces.
+ if (image.variation_strength)
+ return "This image already used a variation seed, so Vary cannot start from its noise. Restore it and change the variation seed instead.";
+ const missing: string[] = [];
+ if (image.loras?.length && !loraCapable) missing.push("its LoRAs");
+ if (image.prompt_weighting === false && !capability?.prompt_weighting) missing.push("prompt weighting off");
+ if ((image.clip_skip ?? 1) > (capability?.clip_skip?.max ?? 1)) missing.push(`CLIP skip ${image.clip_skip}`);
+ if (image.cfg_rescale && !capability?.cfg_rescale) missing.push(`CFG rescale ${image.cfg_rescale}`);
+ return missing.length
+ ? `The current load cannot apply ${missing.join(", ")} from this image's recipe, so Vary would not match it.`
+ : null;
+}
+
+/** The SD controls a restored gallery recipe recorded that the current load drops or clamps (the request
+ * fields omit what `capability` lacks); prompt weighting off is literalPromptNotApplied's. */
+export function sdControlsNotRestored(
+ image: {
+ clip_skip?: number | null;
+ cfg_rescale?: number | null;
+ variation_seed?: number | null;
+ variation_strength?: number | null;
+ },
+ capability: SdControlsCapability | null | undefined,
+): string[] {
+ const out: string[] = [];
+ const clipSkip = image.clip_skip ?? 1;
+ const clipMax = capability?.clip_skip?.max;
+ if (clipSkip > 1) {
+ if (clipMax === undefined) out.push(`CLIP skip ${clipSkip}`);
+ else if (clipSkip > clipMax) out.push(`CLIP skip ${clipSkip} (clamped to ${clipMax})`);
+ }
+ const rescale = image.cfg_rescale ?? 0;
+ if (rescale) {
+ const hi = capability?.cfg_rescale ? Math.min(1, capability.cfg_rescale.max) : undefined;
+ if (hi === undefined) out.push(`CFG rescale ${rescale}`);
+ else if (rescale > hi) out.push(`CFG rescale ${rescale} (clamped to ${hi})`);
+ }
+ if (image.variation_strength && !capability?.variation_seed) {
+ out.push(
+ image.variation_seed != null
+ ? `variation seed ${image.variation_seed} (strength ${image.variation_strength})`
+ : `variation strength ${image.variation_strength}`,
+ );
+ }
+ return out;
+}
+
+/** The choice a gallery recipe restores. Absent fields (older records, other models) read as off. */
+export function sdControlsFromRecipe(image: {
+ prompt_weighting?: boolean | null;
+ clip_skip?: number | null;
+ cfg_rescale?: number | null;
+ variation_seed?: number | null;
+ variation_strength?: number | null;
+ batch_index?: number | null;
+ batch_size?: number | null;
+}): SdControlsChoice {
+ // A batch replays whole from its first image's variation seed (image i used base + i), like its seed.
+ const offset = (image.batch_size ?? 1) > 1 ? (image.batch_index ?? 0) : 0;
+ return {
+ // false = the run turned weighting off; null / absent = it was on (or never mattered).
+ promptWeighting: image.prompt_weighting !== false,
+ clipSkip: image.clip_skip ?? 1,
+ cfgRescale: image.cfg_rescale ?? 0,
+ variationSeed:
+ image.variation_seed != null && image.variation_strength
+ ? String(Math.max(0, image.variation_seed - offset))
+ : "",
+ variationStrength: image.variation_strength ?? 0,
+ };
+}
+
+export function sdControlsRecipeRows(image: {
+ clip_skip?: number | null;
+ cfg_rescale?: number | null;
+ variation_seed?: number | null;
+ variation_strength?: number | null;
+}): [string, string][] {
+ const rows: [string, string][] = [];
+ if (image.clip_skip) rows.push(["CLIP skip", String(image.clip_skip)]);
+ if (image.cfg_rescale) rows.push(["CFG rescale", String(image.cfg_rescale)]);
+ if (image.variation_strength && image.variation_seed != null) {
+ rows.push([
+ "Variation",
+ `${image.variation_seed} @ ${image.variation_strength}`,
+ ]);
+ }
+ return rows;
+}
diff --git a/studio/frontend/tests/image-memory-refusal.test.ts b/studio/frontend/tests/image-memory-refusal.test.ts
index fa3ac4a9a7e..8328042d94c 100644
--- a/studio/frontend/tests/image-memory-refusal.test.ts
+++ b/studio/frontend/tests/image-memory-refusal.test.ts
@@ -69,7 +69,8 @@ test("Generate anyway clicked during the refused run's cleanup waits for busy to
assert.equal(shouldRunQueuedOversizedRetry({ queued: true, busy: null }), true);
assert.equal(shouldRunQueuedOversizedRetry({ queued: false, busy: null }), false);
const page = readSrc("features/images/images-page.tsx");
- assert.match(page, /onClick: \(\) => setOversizedRetryQueued\(true\)/);
+ // The refused run already consumed Vary's one-shot: the retry gets it back, or it renders an unrelated image.
+ assert.match(page, /onClick: \(\) => \{\s*varyOnce\.current = vary;\s*setOversizedRetryQueued\(true\);/);
assert.match(
page,
/shouldRunQueuedOversizedRetry\(\{ queued: oversizedRetryQueued, busy \}\)[\s\S]*?\}, \[oversizedRetryQueued, busy, handleGenerateWithRecall\]\);/,
diff --git a/studio/frontend/tests/image-metadata.test.ts b/studio/frontend/tests/image-metadata.test.ts
index b935d2c4e25..0fa85a94a2d 100644
--- a/studio/frontend/tests/image-metadata.test.ts
+++ b/studio/frontend/tests/image-metadata.test.ts
@@ -659,6 +659,7 @@ test("a newer recipe loads what this build knows and names what it does not", ()
...JSON.parse(RECIPE),
schema_version: 2,
cfg_rescale: 0.7,
+ freeu_b1: 1.1,
tiling: false,
empty_list: [],
unset: null,
@@ -667,9 +668,14 @@ test("a newer recipe loads what this build knows and names what it does not", ()
assert.ok(recipe);
const settings = recipeSettings(recipe, defaults);
assert.equal(settings.seed, 6630007);
- assert.deepEqual(unknownRecipeKeys(recipe), ["tiling"]);
- // A setting this build parses but cannot send yet is named by label, not by key.
+ assert.deepEqual(unknownRecipeKeys(recipe), ["freeu_b1", "tiling"]);
+ assert.equal(settings.cfg_rescale, 0.7);
+ // Without the control on the loaded model it is named by label, not by key; with it, it restores.
assert.deepEqual(importNotApplied(recipe), ["CFG rescale 0.7"]);
+ assert.deepEqual(
+ importNotApplied(recipe, "sdxl", false, { cfg_rescale: { min: 0, max: 1, default: 0 } }),
+ [],
+ );
assert.equal(newerRecipeSchema(recipe), true);
// Every key today's recipe writes is known, so a current image reports nothing.
assert.deepEqual(unknownRecipeKeys(JSON.parse(RECIPE)), []);
diff --git a/studio/frontend/tests/image-paste-parameters.test.ts b/studio/frontend/tests/image-paste-parameters.test.ts
index df6715be1fc..416ad5d2483 100644
--- a/studio/frontend/tests/image-paste-parameters.test.ts
+++ b/studio/frontend/tests/image-paste-parameters.test.ts
@@ -288,7 +288,7 @@ test("the Images page wires paste into the same restore path as a drop", () => {
);
assert.match(
page,
- /\.\.\.importNotApplied\(\n\s+recipe,\n\s+status\?\.loaded \? status\.family : null,\n\s+Boolean\(status\?\.loaded && isNativeEngineStatus\(status\)\),\n\s+\),/,
+ /\.\.\.importNotApplied\(\n\s+recipe,\n\s+status\?\.loaded \? status\.family : null,\n\s+Boolean\(status\?\.loaded && isNativeEngineStatus\(status\)\),\n\s+sdCaps,\n\s+\),/,
);
assert.match(
page,
diff --git a/studio/frontend/tests/image-seed-inpaint-steps.test.ts b/studio/frontend/tests/image-seed-inpaint-steps.test.ts
index 968e6f36d5a..f55743c13a3 100644
--- a/studio/frontend/tests/image-seed-inpaint-steps.test.ts
+++ b/studio/frontend/tests/image-seed-inpaint-steps.test.ts
@@ -175,7 +175,8 @@ test("restoring an inpaint record resets mask options it omitted at their defaul
test("a run whose response was lost still becomes the seed Reuse last seed picks", () => {
const page = readSrc("features/images/images-page.tsx");
- assert.match(page, /const runBaseSeed = baseSeed \+ i \* batchSize;/);
+ // A variation run keeps one seed across runs (the variation seed advances instead).
+ assert.match(page, /const runBaseSeed = varying \? baseSeed : baseSeed \+ i \* batchSize;/);
assert.match(page, /seed: runBaseSeed,/);
assert.match(
page,
diff --git a/studio/frontend/tests/images-sd-controls.test.ts b/studio/frontend/tests/images-sd-controls.test.ts
new file mode 100644
index 00000000000..d2af6a1994d
--- /dev/null
+++ b/studio/frontend/tests/images-sd-controls.test.ts
@@ -0,0 +1,449 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ DEFAULT_SD_CONTROLS,
+ parseVariationSeed,
+ sdControlsFromRecipe,
+ sdControlsRecipeRows,
+ sdControlsRequestFields,
+ varyChoice,
+} from "../src/features/images/lib/sd-controls.ts";
+import { readSrc } from "./helpers/kit.ts";
+
+const SDXL = {
+ prompt_weighting: true,
+ clip_skip: { min: 1, max: 12, default: 1, same_as_default_through: 2 },
+ cfg_rescale: { min: 0, max: 1, default: 0, recommended: null },
+ variation_seed: true,
+ vary_strength: 0.15,
+ prediction_type: "epsilon",
+};
+
+test("defaults send nothing, so the request is unchanged", () => {
+ assert.deepEqual(sdControlsRequestFields(SDXL, DEFAULT_SD_CONTROLS), {});
+ assert.deepEqual(
+ sdControlsRequestFields(null, { ...DEFAULT_SD_CONTROLS, clipSkip: 3 }),
+ {},
+ );
+});
+
+test("a model without the controls never gets them", () => {
+ const choice = {
+ promptWeighting: false,
+ clipSkip: 3,
+ cfgRescale: 0.7,
+ variationSeed: "5",
+ variationStrength: 0.2,
+ };
+ assert.deepEqual(sdControlsRequestFields(undefined, choice), {});
+ assert.deepEqual(sdControlsRequestFields({}, choice), {});
+});
+
+test("non-default values map to request fields", () => {
+ assert.deepEqual(
+ sdControlsRequestFields(SDXL, {
+ promptWeighting: false,
+ clipSkip: 2,
+ cfgRescale: 0.7,
+ variationSeed: "42",
+ variationStrength: 0.25,
+ }),
+ {
+ prompt_weighting: false,
+ clip_skip: 2,
+ cfg_rescale: 0.7,
+ variation_strength: 0.25,
+ variation_seed: 42,
+ },
+ );
+ // An explicit per-run variation seed wins over the field; an empty field leaves it to the server.
+ assert.deepEqual(
+ sdControlsRequestFields(
+ SDXL,
+ { ...DEFAULT_SD_CONTROLS, variationStrength: 0.1 },
+ 7,
+ ),
+ { variation_strength: 0.1, variation_seed: 7 },
+ );
+ assert.deepEqual(
+ sdControlsRequestFields(SDXL, {
+ ...DEFAULT_SD_CONTROLS,
+ variationStrength: 0.1,
+ }),
+ {
+ variation_strength: 0.1,
+ },
+ );
+ assert.equal(
+ sdControlsRequestFields(
+ { ...SDXL, clip_skip: { min: 1, max: 5, default: 1 } },
+ { ...DEFAULT_SD_CONTROLS, clipSkip: 9 },
+ ).clip_skip,
+ 5,
+ );
+});
+
+test("image-conditioned workflows never send the variation fields", () => {
+ const choice = { ...DEFAULT_SD_CONTROLS, cfgRescale: 0.5, variationSeed: "7", variationStrength: 0.3 };
+ assert.deepEqual(sdControlsRequestFields(SDXL, choice, 7, false), { cfg_rescale: 0.5 });
+ assert.equal(sdControlsRequestFields(SDXL, choice, 7).variation_strength, 0.3);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(
+ src,
+ /const varying = Boolean\(sdCaps\?\.variation_seed\) && workflow === "create" && runChoice\.variationStrength > 0;/,
+ );
+ assert.match(src, /% 2 \*\* 32 : null,\n\s+varying,\n\s+!controlnetEngaged,\n\s+\),/);
+ assert.match(src, /\{sdCaps\.variation_seed && workflow === "create" && \(/);
+});
+
+test("variation seeds parse as 32-bit integers", () => {
+ assert.equal(parseVariationSeed(""), null);
+ assert.equal(parseVariationSeed(" 12 "), 12);
+ assert.equal(parseVariationSeed("-1"), null);
+ assert.equal(parseVariationSeed("1.5"), null);
+ assert.equal(parseVariationSeed(String(2 ** 32)), null);
+});
+
+test("Vary keeps the settings and uses the advertised small strength", () => {
+ const v = varyChoice(SDXL, { ...DEFAULT_SD_CONTROLS, clipSkip: 3 }, 99);
+ assert.equal(v.variationSeed, "99");
+ assert.equal(v.variationStrength, 0.15);
+ assert.equal(v.clipSkip, 3);
+ assert.equal(varyChoice({}, DEFAULT_SD_CONTROLS, 1).variationStrength, 0.15);
+});
+
+test("recipe restore and rows", () => {
+ assert.deepEqual(sdControlsFromRecipe({}), DEFAULT_SD_CONTROLS);
+ const restored = sdControlsFromRecipe({
+ clip_skip: 2,
+ cfg_rescale: 0.7,
+ variation_seed: 15,
+ variation_strength: 0.2,
+ batch_index: 3,
+ batch_size: 4,
+ });
+ assert.equal(restored.variationSeed, "12");
+ assert.equal(restored.clipSkip, 2);
+ assert.equal(restored.promptWeighting, true);
+ // A run with weighting off records false; restoring it must keep the prompt literal.
+ assert.equal(sdControlsFromRecipe({ prompt_weighting: false }).promptWeighting, false);
+ assert.equal(sdControlsFromRecipe({ prompt_weighting: null }).promptWeighting, true);
+ assert.equal(sdControlsFromRecipe({ prompt_weighting: true }).promptWeighting, true);
+ assert.deepEqual(sdControlsRecipeRows({}), []);
+ assert.deepEqual(
+ sdControlsRecipeRows({
+ clip_skip: 2,
+ variation_seed: 5,
+ variation_strength: 0.1,
+ }),
+ [
+ ["CLIP skip", "2"],
+ ["Variation", "5 @ 0.1"],
+ ],
+ );
+});
+
+test("the page gates the controls and Vary on the status capability", () => {
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(
+ src,
+ /const sdCaps = \(!external && status\?\.loaded \? status\.sd_controls : null\) \?\? null;/,
+ );
+ assert.match(src, /\{sdCaps && \(/);
+ assert.match(src, /\.\.\.sdControlsRequestFields\(/);
+ assert.match(
+ src,
+ /Boolean\(status\?\.loaded && sdCaps\?\.variation_seed\) && workflow === "create"/,
+ );
+});
+
+test("AUTOMATIC1111 parameters import the controls", async () => {
+ const { parseA1111Parameters, recipeSettings, unknownRecipeKeys } =
+ await import("../src/features/images/lib/image-metadata.ts");
+ const parsed = parseA1111Parameters(
+ "a (fox:1.2)\nSteps: 25, Sampler: Euler, CFG scale: 5, Seed: 11, Size: 832x1216, Model: m, Variation seed: 12, Variation seed strength: 0.15, Clip skip: 2, CFG rescale: 0.7",
+ );
+ assert.ok(parsed);
+ assert.deepEqual(unknownRecipeKeys(parsed), []);
+ const settings = recipeSettings(parsed, {
+ width: 1024,
+ height: 1024,
+ steps: 9,
+ guidance: 0,
+ });
+ assert.equal(settings.clip_skip, 2);
+ assert.equal(settings.cfg_rescale, 0.7);
+ assert.equal(settings.variation_seed, 12);
+ assert.equal(settings.variation_strength, 0.15);
+ assert.equal(settings.prompt_weighting, undefined);
+ const literal = recipeSettings(
+ { prompt: "(fox:1.5)", prompt_weighting: false },
+ { width: 1024, height: 1024, steps: 9, guidance: 0 },
+ );
+ assert.equal(sdControlsFromRecipe(literal).promptWeighting, false);
+ const clamped = recipeSettings(
+ {
+ prompt: "p",
+ clip_skip: 40,
+ cfg_rescale: 3,
+ variation_seed: 2 ** 40,
+ variation_strength: -1,
+ },
+ { width: 1024, height: 1024, steps: 9, guidance: 0 },
+ );
+ assert.equal(clamped.clip_skip, 12);
+ assert.equal(clamped.cfg_rescale, 1);
+ assert.equal(clamped.variation_seed, undefined);
+ assert.equal(clamped.variation_strength, 0);
+});
+
+test("pasted A1111 Clip skip, CFG Rescale and Variation seed restore into the SDXL controls", async () => {
+ const { parseA1111Parameters, importNotApplied, recipeSettings, variationApplies } = await import(
+ "../src/features/images/lib/image-metadata.ts"
+ );
+ const r = parseA1111Parameters(
+ "(a cat:1.2), \\(sign\\)\nSteps: 25, Sampler: Euler a, CFG scale: 5, Seed: 1, Size: 1024x1024, Clip skip: 3, CFG Rescale: 0.7, Variation seed: 99, Variation seed strength: 0.3",
+ );
+ assert.ok(r);
+ const settings = recipeSettings(r, { width: 1024, height: 1024, steps: 30, guidance: 5 });
+ assert.deepEqual(sdControlsFromRecipe(settings), {
+ promptWeighting: true,
+ clipSkip: 3,
+ cfgRescale: 0.7,
+ variationSeed: "99",
+ variationStrength: 0.3,
+ });
+ assert.equal(variationApplies(r, SDXL), true);
+ // SDXL's prompt weighting reads the emphasis and escapes, so nothing is reported.
+ assert.deepEqual(importNotApplied(r, "sdxl", false, SDXL), []);
+ assert.deepEqual(importNotApplied(r, "flux", true, null), [
+ "Clip skip 3",
+ "CFG rescale 0.7",
+ "variation seed 99 (strength 0.3)",
+ ]);
+ // Past the model's CLIP skip range the restore clamps, and says so.
+ assert.deepEqual(
+ importNotApplied({ ...r, clip_skip: 14 }, "sdxl", false, { ...SDXL, clip_skip: { ...SDXL.clip_skip, max: 12 } }),
+ ["Clip skip 14 (clamped to 12)"],
+ );
+ // The backend refuses a variation seed for img2img / inpaint, so it is named and not restored.
+ const img2img = { ...r, workflow: "img2img" };
+ assert.equal(variationApplies(img2img, SDXL), false);
+ assert.deepEqual(importNotApplied(img2img, "sdxl", false, SDXL), ["variation seed 99 (strength 0.3)"]);
+ // A structured Unsloth recipe records prompt_weighting: true, not the parser-only prompt_emphasis.
+ const weighted = { prompt: "a (fox:1.5)", prompt_weighting: true };
+ assert.deepEqual(importNotApplied(weighted, "flux", false, null), ["prompt weighting"]);
+ assert.deepEqual(importNotApplied(weighted, "sdxl", true, null), []);
+ assert.deepEqual(importNotApplied(weighted, "sdxl", false, SDXL), []);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(src, /if \(!variationApplies\(merged, sdCaps\)\)/);
+ assert.match(src, /isNativeEngineStatus\(status\)\),\n\s+sdCaps,/);
+ // importNotApplied already names the controls a model lacks; the page must not add them again.
+ assert.doesNotMatch(src, /notApplied\.unshift\(`CLIP skip|notApplied\.unshift\(`CFG rescale|notApplied\.unshift\("variation seed"\)/);
+});
+
+test("Vary runs the selected image's recipe, not the current form", async () => {
+ const { canVaryImage } = await import("../src/features/images/lib/sd-controls.ts");
+ assert.equal(canVaryImage({ model: "sdxl", workflow: "txt2img" }, "sdxl"), true);
+ assert.equal(canVaryImage({ model: null, workflow: null }, "sdxl"), true);
+ // Made with another model, or from a source image the gallery does not keep: its recipe cannot replay here.
+ assert.equal(canVaryImage({ model: "other", workflow: "txt2img" }, "sdxl"), false);
+ assert.equal(canVaryImage({ model: "sdxl", workflow: "img2img" }, "sdxl"), false);
+ assert.equal(canVaryImage({ model: "sdxl", workflow: "controlnet" }, "sdxl"), false);
+ const src = readSrc("features/images/images-page.tsx");
+ const vary = src.slice(src.indexOf("const handleVary = useCallback("));
+ assert.match(vary, /^[\s\S]*?restoreSettings\(selected, true, "vary"\);\n\s+setVaryQueued\(true\);/);
+ assert.doesNotMatch(vary.slice(0, vary.indexOf("}, [busy")), /void handleGenerate\(\)/);
+ assert.match(src, /if \(!varyQueued \|\| busy !== null\) return;\n\s+setVaryQueued\(false\);\n\s+void handleGenerate\(\);/);
+ assert.match(src, /canVaryImage\(selected, status\?\.display_repo_id \?\? status\?\.repo_id \?\? null\)/);
+ // The Vary restore is silent, and the import path reports what was not applied itself.
+ assert.match(src, /if \(via === "vary"\) return;/);
+ assert.match(src, /false,\n\s+"import",\n\s+\);/);
+});
+
+test("a literal-prompt recipe restored onto sd.cpp names prompt weighting off as not applied", async () => {
+ const { importNotApplied, literalPromptNotApplied } = await import(
+ "../src/features/images/lib/image-metadata.ts"
+ );
+ const literal = { prompt: "a (fox:1.5)", prompt_weighting: false };
+ const note = "prompt weighting off (the native sd.cpp engine always reads (word:1.2) syntax)";
+ assert.equal(literalPromptNotApplied(literal, true, null), note);
+ // Diffusers SDXL turns weighting off; diffusers without the control never parsed the syntax anyway.
+ assert.equal(literalPromptNotApplied(literal, false, SDXL), null);
+ assert.equal(literalPromptNotApplied(literal, false, null), null);
+ assert.equal(literalPromptNotApplied({ prompt_weighting: true }, true, null), null);
+ assert.deepEqual(importNotApplied(literal, "sdxl", true, null), [note]);
+ assert.deepEqual(importNotApplied(literal, "sdxl", false, SDXL), []);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(
+ src,
+ /via === "restore"\n\s+\? literalPromptNotApplied\(image, Boolean\(status\?\.loaded && isNativeEngineStatus\(status\)\), sdCaps\)/,
+ );
+});
+
+test("imported variation values out of the request bounds are named, not randomised or clamped", async () => {
+ const { importNotApplied, variationApplies } = await import(
+ "../src/features/images/lib/image-metadata.ts"
+ );
+ const ok = { prompt: "x", variation_seed: 99, variation_strength: 0.2 };
+ assert.equal(variationApplies(ok, SDXL), true);
+ assert.equal(variationApplies({ prompt: "x", variation_strength: 0.2 }, SDXL), true);
+ assert.equal(variationApplies({ prompt: "x", variation_seed: 2 ** 32 - 1, variation_strength: 1 }, SDXL), true);
+ for (const bad of [
+ { prompt: "x", variation_seed: 2 ** 40, variation_strength: 0.2 },
+ { prompt: "x", variation_seed: 2 ** 32, variation_strength: 0.2 },
+ { prompt: "x", variation_seed: -1, variation_strength: 0.2 },
+ { prompt: "x", variation_seed: 1.5, variation_strength: 0.2 },
+ { prompt: "x", variation_seed: 99, variation_strength: 1.5 },
+ ]) {
+ assert.equal(variationApplies(bad, SDXL), false, JSON.stringify(bad));
+ assert.deepEqual(importNotApplied(bad, "sdxl", false, SDXL), [
+ `variation seed ${bad.variation_seed} (strength ${bad.variation_strength})`,
+ ]);
+ }
+ assert.deepEqual(importNotApplied({ prompt: "x", variation_strength: 3 }, "sdxl", false, SDXL), ["variation strength 3"]);
+});
+
+test("Vary is disabled with a reason when the load cannot apply the selected recipe as recorded", async () => {
+ const { varyBlockedReason } = await import("../src/features/images/lib/sd-controls.ts");
+ const BUILD = { schema_version: 1, engine: "diffusers", model_kind: "pipeline" };
+ const recipe = { ...BUILD, loras: ["style:0.8"], cfg_rescale: 0.7, clip_skip: 2, prompt_weighting: false };
+ assert.equal(varyBlockedReason(recipe, true, SDXL, BUILD), null);
+ assert.equal(varyBlockedReason({ ...BUILD, loras: [] }, false, SDXL, BUILD), null);
+ // Speed=Auto compiled after the render: supports_lora is now false, so the adapters would be dropped.
+ assert.match(varyBlockedReason({ ...BUILD, loras: ["style:0.8"] }, false, SDXL, BUILD) ?? "", /its LoRAs/);
+ const noRescale = { ...SDXL, cfg_rescale: null, clip_skip: { ...SDXL.clip_skip, max: 2 } };
+ assert.match(varyBlockedReason({ ...BUILD, cfg_rescale: 0.7 }, true, noRescale, BUILD) ?? "", /CFG rescale 0\.7/);
+ assert.match(varyBlockedReason({ ...BUILD, clip_skip: 4 }, true, noRescale, BUILD) ?? "", /CLIP skip 4/);
+ assert.match(varyBlockedReason({ ...BUILD, prompt_weighting: false }, true, { ...SDXL, prompt_weighting: false }, BUILD) ?? "", /weighting off/);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(src, /const varyBlocked = canVary && selected \? varyBlockedReason\(selected, loraCapable, sdCaps, status \?\? \{\}\) : null;/);
+ assert.match(src, /disabled=\{busy !== null \|\| varyBlocked !== null\}/);
+ assert.match(src, /\{varyBlocked \?\? \(/);
+});
+
+test("Vary is disabled unless the selected image came off the exact loaded build", async () => {
+ const { varyBlockedReason } = await import("../src/features/images/lib/sd-controls.ts");
+ const loaded = {
+ engine: "diffusers",
+ model_kind: "gguf",
+ gguf_filename: "sdxl-Q4_K_M.gguf",
+ transformer_quant: null,
+ text_encoder_quant: "fp8",
+ };
+ const image = { ...loaded, schema_version: 1, model: "sdxl", workflow: "txt2img" };
+ // The exact same build keeps Vary on; null and absent quant fields read alike.
+ assert.equal(varyBlockedReason(image, true, SDXL, loaded), null);
+ assert.equal(varyBlockedReason({ ...image, transformer_quant: undefined }, true, SDXL, loaded), null);
+ for (const [field, value, label] of [
+ ["engine", "sd_cpp", /engine/],
+ ["model_kind", "pipeline", /model kind/],
+ ["gguf_filename", "sdxl-Q8_0.gguf", /gguf filename/],
+ ["transformer_quant", "int8", /transformer quant/],
+ ["text_encoder_quant", null, /text encoder quant/],
+ ] as const) {
+ assert.match(
+ varyBlockedReason({ ...image, [field]: value }, true, SDXL, loaded) ?? "",
+ new RegExp(`different build \\(.*${label.source}`),
+ field,
+ );
+ }
+ // Older records: no schema_version (build not stored) or no engine (stored before it was recorded) cannot be checked.
+ const { schema_version: _drop, ...noSchema } = image;
+ assert.match(varyBlockedReason(noSchema, true, SDXL, loaded) ?? "", /does not record which engine/);
+ assert.match(varyBlockedReason({ ...image, engine: null }, true, SDXL, loaded) ?? "", /does not record which engine/);
+ // The build check comes before the variation-seed one.
+ assert.match(varyBlockedReason({ ...image, engine: "mlx", variation_strength: 0.2 }, true, SDXL, loaded) ?? "", /different build/);
+});
+
+test("Vary compares the load-time baked LoRAs, attention backend and step cache as part of the build", async () => {
+ const { varyBlockedReason } = await import("../src/features/images/lib/sd-controls.ts");
+ const loaded = { engine: "diffusers", model_kind: "pipeline", baked_loras: ["ink", "sloth"], attention_backend: "_native_cudnn", transformer_cache: "fbcache" };
+ const image = { ...loaded, schema_version: 1, baked_loras: ["sloth", "ink"] };
+ // The same adapter set in another order is the same build.
+ assert.equal(varyBlockedReason(image, true, SDXL, loaded), null);
+ // A baked record against a load without the bake, a different set, or a bake the record lacks.
+ for (const other of [{ ...loaded, baked_loras: [] }, { ...loaded, baked_loras: undefined }, { ...loaded, baked_loras: ["sloth"] }])
+ assert.match(varyBlockedReason(image, true, SDXL, other) ?? "", /different build \(baked loras\)/);
+ assert.match(varyBlockedReason({ ...image, baked_loras: [] }, true, SDXL, loaded) ?? "", /baked loras/);
+ // An empty and an absent set read alike on both sides.
+ const plain = { engine: "diffusers", schema_version: 1 };
+ assert.equal(varyBlockedReason({ ...plain, baked_loras: [] }, true, SDXL, { engine: "diffusers" }), null);
+ assert.match(varyBlockedReason({ ...image, attention_backend: null }, true, SDXL, loaded) ?? "", /attention backend/);
+ assert.match(varyBlockedReason({ ...image, transformer_cache: "static" }, true, SDXL, loaded) ?? "", /transformer cache/);
+ // The status type declares the field the page passes in.
+ assert.match(readSrc("features/images/api.ts"), /baked_loras\?: string\[\] \| null;/);
+});
+
+test("Create with an SDXL ControlNet engaged sends no CFG rescale, and the control says why", () => {
+ const choice = { ...DEFAULT_SD_CONTROLS, cfgRescale: 0.7, variationStrength: 0.2, variationSeed: "5" };
+ assert.deepEqual(sdControlsRequestFields(SDXL, choice, null, true, false), {
+ variation_strength: 0.2,
+ variation_seed: 5,
+ });
+ assert.equal(sdControlsRequestFields(SDXL, choice).cfg_rescale, 0.7);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(
+ src,
+ /const controlnetEngaged =\n\s+controlnetCapable && Boolean\(controlnetId && controlImage\) && workflow === "create" && controlStrength > 0;/,
+ );
+ assert.match(src, /varying,\n\s+!controlnetEngaged,\n\s+\),/);
+ assert.match(src, /disabled=\{controlnetEngaged\}/);
+});
+
+test("pasted A1111 Emphasis: None restores prompt weighting off, and sd.cpp names it as not applied", async () => {
+ const { importNotApplied, parseA1111Parameters, recipeSettings } = await import(
+ "../src/features/images/lib/image-metadata.ts"
+ );
+ const r = parseA1111Parameters("a (fox:1.5), \\(x\\)\nSteps: 20, Seed: 1, Emphasis: None") ?? { prompt: "" };
+ assert.equal(r.prompt_weighting, false);
+ const settings = recipeSettings(r, { width: 1024, height: 1024, steps: 20, guidance: 7 });
+ assert.equal(settings.prompt_weighting, false);
+ assert.equal(sdControlsFromRecipe(settings).promptWeighting, false);
+ // Diffusers SDXL turns weighting off; diffusers without the control reads it literally already.
+ assert.deepEqual(importNotApplied(r, "sdxl", false, SDXL), []);
+ assert.deepEqual(importNotApplied(r, "sdxl", false, null), []);
+ assert.deepEqual(importNotApplied(r, "sdxl", true, null), [
+ "prompt weighting off (the native sd.cpp engine always reads (word:1.2) syntax)",
+ ]);
+});
+
+test("a gallery restore names the SD controls the current load drops or clamps", async () => {
+ const { sdControlsNotRestored } = await import("../src/features/images/lib/sd-controls.ts");
+ const recipe = { clip_skip: 3, cfg_rescale: 0.7, variation_seed: 42, variation_strength: 0.3 };
+ assert.deepEqual(sdControlsNotRestored(recipe, SDXL), []);
+ assert.deepEqual(sdControlsNotRestored({ clip_skip: 1, cfg_rescale: 0 }, null), []);
+ // Same SDXL repo reloaded on the native engine: no sd_controls, so Generate sends none of them.
+ assert.deepEqual(sdControlsNotRestored(recipe, null), [
+ "CLIP skip 3",
+ "CFG rescale 0.7",
+ "variation seed 42 (strength 0.3)",
+ ]);
+ const narrow = { ...SDXL, clip_skip: { ...SDXL.clip_skip, max: 2 }, cfg_rescale: { ...SDXL.cfg_rescale, max: 0.5 } };
+ assert.deepEqual(sdControlsNotRestored(recipe, narrow), ["CLIP skip 3 (clamped to 2)", "CFG rescale 0.7 (clamped to 0.5)"]);
+ const src = readSrc("features/images/images-page.tsx");
+ assert.match(src, /via === "restore" && status\?\.loaded \? sdControlsNotRestored\(image, sdCaps\) : \[\]/);
+ assert.match(src, /description: `Not applied: \$\{notApplied\.join\(", "\)\}\./);
+});
+
+test("an imported CFG rescale outside the accepted range is named with its clamp", async () => {
+ const { importNotApplied } = await import("../src/features/images/lib/image-metadata.ts");
+ assert.deepEqual(importNotApplied({ prompt: "x", cfg_rescale: 0.7 }, "sdxl", false, SDXL), []);
+ assert.deepEqual(importNotApplied({ prompt: "x", cfg_rescale: 1.5 }, "sdxl", false, SDXL), ["CFG rescale 1.5 (clamped to 1)"]);
+ assert.deepEqual(importNotApplied({ prompt: "x", cfg_rescale: -0.2 }, "sdxl", false, SDXL), ["CFG rescale -0.2 (clamped to 0)"]);
+ const narrow = { ...SDXL, cfg_rescale: { ...SDXL.cfg_rescale, max: 0.5 } };
+ assert.deepEqual(importNotApplied({ prompt: "x", cfg_rescale: 0.7 }, "sdxl", false, narrow), ["CFG rescale 0.7 (clamped to 0.5)"]);
+ assert.deepEqual(importNotApplied({ prompt: "x", cfg_rescale: 0.7 }, "sdxl", false, null), ["CFG rescale 0.7"]);
+});
+
+test("Vary is disabled for an image that already used a variation seed", async () => {
+ const { varyBlockedReason } = await import("../src/features/images/lib/sd-controls.ts");
+ const B = { schema_version: 1, engine: "diffusers" };
+ assert.equal(varyBlockedReason({ ...B, variation_strength: 0 }, true, SDXL, B), null);
+ assert.match(varyBlockedReason({ ...B, variation_strength: 1 }, true, SDXL, B) ?? "", /already used a variation seed/);
+ assert.match(varyBlockedReason({ ...B, variation_strength: 0.15 }, true, SDXL, B) ?? "", /already used a variation seed/);
+});