Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: add Qwen-Image-2.1 image editing (#11587)

oobabooga40fd40c593f3
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 5eda9f1a0e0..bf9381ca7c9 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -77,6 +77,15 @@ from .diffusion_device import (
resolve_diffusion_device_target,
resolve_selected_cuda_ordinal,
)
+from .diffusion_conditioning import (
+ LocalizedEdit,
+ check_conditioned_fields,
+ check_output_size,
+ conditioning_capabilities,
+ decode_condition_images,
+ effective_reference_resolution,
+ match_source_size,
+)
from .diffusion_ideogram4 import ideogram4_repo_is_fp8, load_ideogram4_pipeline
from .diffusion_hidream import (
HIDREAM_FAMILY_NAME,
@@ -599,14 +608,29 @@ def _image_variant_hint(
return " ".join(parts)
-def _compile_shape_dims(workflow: str, init_pil: Any, width: int, height: int) -> tuple[int, int]:
+def _is_source_sized(workflow: str, family: Any = None) -> bool:
+ """Whether ``workflow`` takes its output size from the source image rather than width/height.
+ "edit" is source-sized only on an edit-only family (Kontext, Qwen-Image-Edit), or when
+ ``family`` is None; the unified edit (Qwen-Image-2.1) renders at the requested size."""
+ if workflow == "edit":
+ return family is None or bool(getattr(family, "edit", False))
+ return workflow in ("img2img", "inpaint", "upscale")
+
+
+def _compile_shape_dims(
+ workflow: str,
+ init_pil: Any,
+ width: int,
+ height: int,
+ family: Any = None,
+) -> tuple[int, int]:
"""The (width, height) a generation's forward ACTUALLY runs at, for static compile-cache shape
- registration. txt2img / reference / controlnet generate at the requested slider size, but the
- image-conditioned workflows derive the output from the (resized/snapped) input image:
+ registration. txt2img / reference / controlnet / unified edit generate at the requested size,
+ but the source-sized workflows derive the output from the (resized/snapped) input image:
registering the slider values there would mark a shape covered that was never compiled, so
the truly-used shape never re-dirties the bundle and warm restarts keep paying its compile.
Mirrors the width/height kwarg derivation in generate()."""
- if workflow in ("txt2img", "reference", "controlnet") or init_pil is None:
+ if init_pil is None or not _is_source_sized(workflow, family):
return int(width), int(height)
iw, ih = init_pil.size
return int(iw), int(ih)
@@ -6929,8 +6953,9 @@ class DiffusionBackend:
*,
prompt: str,
negative_prompt: Optional[str] = None,
- width: int = 1024,
- height: int = 1024,
+ # None on both (unified edit only) matches Image 1's aspect ratio.
+ width: Optional[int] = 1024,
+ height: Optional[int] = 1024,
# Fallbacks; the route always sends the per-model values the UI seeds.
steps: int = 9,
guidance: float = 0.0,
@@ -6948,8 +6973,12 @@ class DiffusionBackend:
strength: Optional[float] = None,
# Upscale (hires fix): factor > 1 with an init image enlarges then re-denoises at low strength.
upscale: Optional[float] = None,
- # Reference (FLUX.2): additional reference images beyond init_image (a list). Ignored elsewhere.
+ # Reference / unified edit: additional images after init_image, in order. The family bounds the total.
reference_images: Optional[list[str]] = None,
+ # Explicit "edit" / "reference"; None keeps the workflow the other arguments imply.
+ workflow: Optional[str] = None,
+ reference_resolution: Optional[int] = None,
+ localized_edit: Optional[LocalizedEdit] = None,
loras: Optional[list[tuple[str, float]]] = None,
# ControlNet (id, control_image_b64, control_type, strength, guidance_start, guidance_end). None = off.
controlnet: Optional[tuple[str, str, str, float, float, float]] = None,
@@ -7014,10 +7043,14 @@ class DiffusionBackend:
self._apply_loras(state, loras, cancel)
pipe = state.pipe
+ fam = state.family
init_pil = mask_pil = None
control_pil = None
cn_scale = cn_gstart = cn_gend = cn_mode = None
ref_extra: list = []
+ requested_workflow = workflow
+ if requested_workflow not in (None, "edit", "reference"):
+ raise ValueError(f"Unknown workflow '{requested_workflow}'.")
if init_image is None:
if mask_image is not None:
raise ValueError("mask_image requires an input image (init_image).")
@@ -7025,32 +7058,66 @@ class DiffusionBackend:
raise ValueError("upscale requires an input image (init_image).")
if reference_images:
raise ValueError("reference_images require an input image (init_image).")
- if reference_images and not getattr(state.family, "reference", False):
+ if requested_workflow is not None:
+ raise ValueError(
+ f"The {requested_workflow} workflow requires a source image (init_image)."
+ )
+ check_conditioned_fields(
+ requested_workflow,
+ fam,
+ mask_image = mask_image,
+ strength = strength,
+ upscale = upscale,
+ controlnet = controlnet,
+ localized_edit = localized_edit,
+ )
+ if reference_images and not getattr(fam, "reference", False):
raise ValueError(
- f"Reference images are not supported for the '{state.family.name}' "
- "model family."
+ f"Reference images are not supported for the '{fam.name}' model family."
)
- if getattr(state.family, "edit", False):
+ conditioned = False
+ if getattr(fam, "edit", False):
+ if requested_workflow == "reference":
+ raise ValueError(
+ f"The reference workflow is not supported for the '{fam.name}' "
+ "model family."
+ )
if init_image is None:
raise ValueError(
- f"{state.family.name} is an image-editing model: provide an input image."
+ f"{fam.name} is an image-editing model: provide an input image."
)
if mask_image is not None:
raise ValueError(
- f"{state.family.name} is an image-editing model and does not "
+ f"{fam.name} is an image-editing model and does not "
"support masks (mask_image)."
)
workflow = "edit"
init_pil = decode_b64_image(init_image, mode = "RGB")
+ elif requested_workflow == "edit":
+ if not getattr(fam, "unified_edit", False):
+ raise ValueError(
+ f"Instruction editing is not supported for the '{fam.name}' "
+ "model family."
+ )
+ workflow = "edit"
+ conditioned = True
+ elif requested_workflow == "reference":
+ if not getattr(fam, "reference", False):
+ raise ValueError(
+ f"The reference workflow is not supported for the '{fam.name}' "
+ "model family."
+ )
+ workflow = "reference"
+ conditioned = True
elif mask_image is not None and init_image is not None:
workflow = "inpaint"
- pipe = self._workflow_pipe(state, state.family.inpaint_pipeline_class, workflow)
+ pipe = self._workflow_pipe(state, fam.inpaint_pipeline_class, workflow)
init_pil = decode_b64_image(init_image, mode = "RGB")
mask_pil = decode_b64_image(mask_image, mode = "L")
elif init_image is not None and upscale is not None and upscale > 1.0:
# Upscale (hires fix): enlarge with Lanczos, then re-run img2img at low strength to add detail.
workflow = "upscale"
- pipe = self._workflow_pipe(state, state.family.img2img_pipeline_class, workflow)
+ pipe = self._workflow_pipe(state, fam.img2img_pipeline_class, workflow)
init_pil = decode_b64_image(init_image, mode = "RGB")
iw, ih = init_pil.size
# Cap the factor, then the absolute output (longest side 2048); round to a multiple of 16 (VAE
@@ -7071,21 +7138,37 @@ class DiffusionBackend:
init_pil = init_pil.resize((tw, th), Image.LANCZOS)
if strength is None:
strength = 0.35 # hires-fix default: preserve content, add detail
- elif getattr(state.family, "reference", False) and init_image is not None:
- # FLUX.2 reference conditioning: the loaded pipe takes the reference via `image` and generates at
- # the REQUESTED size.
+ elif getattr(fam, "reference", False) and init_image is not None:
workflow = "reference"
- init_pil = decode_b64_image(init_image, mode = "RGB")
- # Additional references (FLUX.2 combines a list); capped to bound VRAM.
- ref_extra = [
- decode_b64_image(x, mode = "RGB") for x in (reference_images or [])[:3]
- ]
+ conditioned = True
elif init_image is not None:
workflow = "img2img"
- pipe = self._workflow_pipe(state, state.family.img2img_pipeline_class, workflow)
+ pipe = self._workflow_pipe(state, fam.img2img_pipeline_class, workflow)
init_pil = decode_b64_image(init_image, mode = "RGB")
else:
workflow = "txt2img"
+ ref_resolution = None
+ if conditioned:
+ images_in = decode_condition_images(
+ fam, init_image, reference_images, localized_edit
+ )
+ init_pil, ref_extra = images_in[0], images_in[1:]
+ ref_resolution = effective_reference_resolution(fam, reference_resolution)
+ elif reference_resolution is not None:
+ raise ValueError(
+ "reference_resolution applies only to the reference and edit workflows."
+ )
+ source_sized = _is_source_sized(workflow, fam)
+ if not source_sized:
+ if width is None or height is None:
+ if not conditioned:
+ raise ValueError("width and height are required for this workflow.")
+ width, height = match_source_size(
+ fam, init_pil.size, ref_resolution or 1024
+ )
+ check_output_size(fam, int(width), int(height))
+ elif width is None or height is None:
+ width = height = 1024
# ControlNet (diffusers): txt2img only. Builds the family CN pipeline around resident modules.
if controlnet is not None:
@@ -7141,7 +7224,7 @@ class DiffusionBackend:
)
# Snap odd-sized inputs (and the mask) to a multiple of 16 where the OUTPUT size comes from the input
# image.
- if init_pil is not None and workflow in ("img2img", "inpaint", "edit"):
+ if init_pil is not None and source_sized and workflow != "upscale":
# img2img/inpaint take output size from the upload, so bound the longest side to 2048 (a phone
# photo would OOM).
if workflow == "img2img":
@@ -7185,15 +7268,21 @@ class DiffusionBackend:
# (cfg_trunc_ratio=0.25); the 1.0 default oversaturates.
kwargs["cfg_trunc_ratio"] = 0.25
if init_pil is not None:
- # Reference passes the whole list (FLUX.2 combines); others take the single image.
+ # Reference / unified edit pass the whole ordered list; others take the single image.
kwargs["image"] = [init_pil, *ref_extra] if ref_extra else init_pil
+ if ref_resolution is not None:
+ if "output_resolution" not in call_params:
+ raise ValueError(
+ "This pipeline does not accept a reference resolution."
+ )
+ kwargs["output_resolution"] = ref_resolution
if mask_pil is not None and "mask_image" in call_params:
kwargs["mask_image"] = mask_pil
if strength is not None and "strength" in call_params:
kwargs["strength"] = strength
# width/height: txt2img uses the slider; image-conditioned pipes must use the INPUT IMAGE's size or
# the latents mismatch, and many drop them, so pass only when accepted.
- if workflow in ("txt2img", "reference", "controlnet"):
+ if not source_sized:
# These generate at the REQUESTED size (reference/control image resized to match).
kwargs["width"] = width
kwargs["height"] = height
@@ -7231,7 +7320,7 @@ class DiffusionBackend:
# their output size from the input image. Same derivation the compile-cache shape registration
# uses further down.
guard_width, guard_height = _compile_shape_dims(
- workflow, init_pil, width, height
+ workflow, init_pil, width, height, fam
)
guard_batch = _activation_guard_batch(chunks)
raise_on_image_activation_shortfall(
@@ -7251,7 +7340,17 @@ class DiffusionBackend:
# img2img is bounded by the Resolution control (see _fit_within), so "generate at a smaller
# resolution" is actionable there; the rest size from the upload alone and get the upload-side
# remedy.
- source_driven = workflow in ("inpaint", "upscale", "edit"),
+ source_driven = source_sized and workflow != "img2img",
+ condition_pixels = (
+ int(
+ (1 + len(ref_extra))
+ * ref_resolution
+ * ref_resolution
+ * getattr(fam, "condition_pixel_weight", 1.0)
+ )
+ if ref_resolution is not None
+ else 0
+ ),
logger = logger,
)
except ValueError:
@@ -7394,7 +7493,9 @@ class DiffusionBackend:
try:
# Register the dims the forward ACTUALLY compiled with, and every distinct chunk size (a static
# compile makes one artifact per batch size too).
- reg_width, reg_height = _compile_shape_dims(workflow, init_pil, width, height)
+ reg_width, reg_height = _compile_shape_dims(
+ workflow, init_pil, width, height, fam
+ )
static_shapes = "compiled" in (
state.speed_optims or ()
) and compiled_shapes_are_static(state.pipe, state.speed_mode)
@@ -7446,6 +7547,8 @@ class DiffusionBackend:
# The workflow this generation ACTUALLY ran, so a conditioned image is not replayed as a plain
# Create recipe.
"workflow": workflow,
+ "reference_resolution": ref_resolution,
+ "localized_edit": localized_edit.mode if localized_edit is not None else None,
}
finally:
with self._generation_cancel_lock:
@@ -7639,6 +7742,7 @@ class DiffusionBackend:
"attention_backend": None,
"transformer_cache": None,
"workflows": [],
+ "conditioning": None,
"supports_lora": False,
"supports_controlnet": False,
"resolved": None,
@@ -7673,6 +7777,9 @@ class DiffusionBackend:
"resolved": state.resolved,
# Workflows the loaded family supports, so the UI can gate its tabs.
"workflows": _family_workflows(state.family),
+ "conditioning": conditioning_capabilities(
+ state.family, _family_workflows(state.family)
+ ),
"supports_lora": diffusion_lora.supports_lora(
engine = "diffusers",
family = state.family.name,
@@ -7701,6 +7808,9 @@ def _family_workflows(fam: DiffusionFamily) -> list[str]:
# Reference families (FLUX.2) add reference conditioning via their pipeline's image arg.
if getattr(fam, "reference", False):
workflows.append("reference")
+ # Unified families (Qwen-Image-2.1) also follow edit instructions through that same pipeline.
+ if getattr(fam, "unified_edit", False):
+ workflows.append("edit")
if getattr(fam, "img2img_pipeline_class", None):
# Upscale runs on the img2img pipeline, so available exactly when img2img is.
workflows.append("img2img")
diff --git a/studio/backend/core/inference/diffusion_conditioning.py b/studio/backend/core/inference/diffusion_conditioning.py
new file mode 100644
index 00000000000..807819e0f5d
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_conditioning.py
@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Condition images and output geometry for the reference and unified-edit workflows, shared by
+the diffusers and sd.cpp engines.
+
+Localized editing follows the official Qwen-Image-2.1 demo conventions, which are generative
+guides rather than inpainting masks: nothing here keeps pixels outside the region.
+
+- ``annotate``: coloured marks (outlines, arrows) drawn ON the source. The instruction names the
+ colours and asks for the marks to be left out of the result.
+- ``paint``: an opaque white region painted ON the source. The instruction refers to the white
+ area.
+- ``mask``: a separate white-on-black mask at the source's geometry, sent as Image 2 right after
+ the source. The instruction refers to the marked area.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Any, Optional, Sequence
+
+LOCALIZED_EDIT_MODES = ("annotate", "paint", "mask")
+
+# Decoded pixels across ALL condition images of one call: ten 4096 px images would be over 600 MB of RGBA.
+MAX_CONDITION_SOURCE_PIXELS = 64_000_000
+
+MIN_OUTPUT_SIDE = 256
+
+# Upstream QwenImage21Pipeline's own ``output_resolution`` default.
+DEFAULT_REFERENCE_RESOLUTION = 1024
+
+
+@dataclass(frozen = True)
+class LocalizedEdit:
+ mode: str
+ image: str
+
+
+def check_conditioned_fields(
+ workflow: Optional[str],
+ fam: Any,
+ *,
+ mask_image: Any = None,
+ strength: Optional[float] = None,
+ upscale: Optional[float] = None,
+ controlnet: Any = None,
+ localized_edit: Optional[LocalizedEdit] = None,
+) -> None:
+ """Refuse request fields an explicit ``workflow`` would drop, on either engine."""
+ if workflow is not None:
+ # One pipeline call over the ordered images would drop these, so refuse rather than ignore them.
+ for present, name in (
+ (mask_image is not None, "mask_image"),
+ (strength is not None, "strength"),
+ (upscale is not None and upscale > 1.0, "upscale"),
+ (controlnet is not None and controlnet[3] not in (None, 0, 0.0), "controlnet"),
+ ):
+ if present:
+ raise ValueError(f"{name} is not supported by the {workflow} workflow.")
+ if localized_edit is not None and not (
+ workflow == "edit" and getattr(fam, "unified_edit", False)
+ ):
+ raise ValueError("localized_edit needs the edit workflow on a model that supports it.")
+
+
+def check_output_size(fam: Any, width: int, height: int) -> None:
+ """Refuse an output size the loaded family cannot render as asked. A size the pipeline would
+ silently floor to its own grid is refused rather than changed behind the caller's back."""
+ multiple = int(getattr(fam, "dimension_multiple", 16) or 16)
+ max_side = int(getattr(fam, "max_output_side", 2048) or 2048)
+ max_pixels = int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048)
+ name = getattr(fam, "name", "this")
+ if min(width, height) < MIN_OUTPUT_SIDE:
+ raise ValueError(
+ f"Width and height must be at least {MIN_OUTPUT_SIDE}px (got {width}x{height})."
+ )
+ if width % multiple or height % multiple:
+ raise ValueError(
+ f"The {name} model needs width and height in multiples of {multiple} "
+ f"(got {width}x{height})."
+ )
+ if max(width, height) > max_side:
+ raise ValueError(
+ f"The {name} model generates at most {max_side}px per side (got {width}x{height})."
+ )
+ if width * height > max_pixels:
+ raise ValueError(
+ f"The {name} model generates at most {max_pixels:,} pixels "
+ f"(got {width}x{height} = {width * height:,})."
+ )
+
+
+def match_source_size(fam: Any, source_size: tuple[int, int], resolution: int) -> tuple[int, int]:
+ """Output (width, height) with the source's aspect ratio at a ``resolution`` squared area, on
+ the family grid and inside its bounds. Upstream's ``calculate_dimensions`` rounding, but from
+ Image 1 rather than the LAST reference. A source too elongated for both bounds keeps the short
+ side at the minimum and caps the long side, so the size is always one the request accepts."""
+ multiple = int(getattr(fam, "dimension_multiple", 16) or 16)
+ max_side = int(getattr(fam, "max_output_side", 2048) or 2048)
+ max_pixels = int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048)
+ sw, sh = source_size
+ ratio = max(1e-6, float(sw) / float(max(1, sh)))
+ area = float(resolution) * float(resolution)
+ for _ in range(64):
+ w = max(multiple, int(round(math.sqrt(area * ratio) / multiple)) * multiple)
+ h = max(multiple, int(round(math.sqrt(area / ratio) / multiple)) * multiple)
+ if max(w, h) <= max_side and w * h <= max_pixels:
+ break
+ area *= 0.9
+ short_min = -(-MIN_OUTPUT_SIDE // multiple) * multiple
+ long_max = max_side // multiple * multiple
+ if min(w, h) < short_min:
+ long_side = int(round(short_min * max(ratio, 1.0 / ratio) / multiple)) * multiple
+ long_side = min(max(long_side, short_min), long_max)
+ while long_side > short_min and long_side * short_min > max_pixels:
+ long_side -= multiple
+ w, h = (long_side, short_min) if ratio >= 1.0 else (short_min, long_side)
+ return w, h
+
+
+def effective_reference_resolution(fam: Any, requested: Optional[int]) -> Optional[int]:
+ """The condition-image preprocessing resolution this call uses, or None for a family without
+ the control. Refuses a value the family does not list rather than rounding it."""
+ allowed = tuple(getattr(fam, "reference_resolutions", ()) or ())
+ if not allowed:
+ if requested is not None:
+ raise ValueError(
+ f"reference_resolution is not supported for the '{getattr(fam, 'name', '')}' "
+ "model family."
+ )
+ return None
+ if requested is None:
+ return (
+ DEFAULT_REFERENCE_RESOLUTION if DEFAULT_REFERENCE_RESOLUTION in allowed else allowed[0]
+ )
+ if requested not in allowed:
+ raise ValueError(
+ "reference_resolution must be one of "
+ + ", ".join(str(v) for v in allowed)
+ + f" for this model (got {requested})."
+ )
+ return int(requested)
+
+
+def _decode_bounded(data: str, mode: str, budget: list[int], what: str) -> Any:
+ from core.inference.diffusion import decode_b64_image
+
+ try:
+ img = decode_b64_image(data, mode = mode, max_pixels = max(1, budget[0]))
+ except ValueError as exc:
+ if "source pixels" in str(exc):
+ raise ValueError(
+ f"The input images are too large together: {what} exceeds the "
+ f"{MAX_CONDITION_SOURCE_PIXELS:,} pixel total for one request. Use smaller images."
+ ) from exc
+ raise ValueError(f"{what}: {exc}") from exc
+ budget[0] -= img.width * img.height
+ return img
+
+
+def _to_source_geometry(img: Any, source: Any, what: str, resample: Any) -> Any:
+ """``img`` at the source's pixel size. Only a pure scale is accepted: a different aspect ratio
+ means the layer was drawn over another image."""
+ if img.size == source.size:
+ return img
+ sw, sh = source.size
+ iw, ih = img.size
+ if abs(iw / float(ih) - sw / float(sh)) > 0.02 * (sw / float(sh)):
+ raise ValueError(
+ f"The {what} is {iw}x{ih} but the source image is {sw}x{sh}; it must be drawn over "
+ "the same image."
+ )
+ return img.resize(source.size, resample)
+
+
+def apply_localized_edit(source: Any, localized: LocalizedEdit, budget: list[int]) -> list[Any]:
+ """The images the localized edit contributes, source first: ``[marked_source]`` for annotate
+ and paint, ``[source, mask]`` for a separate mask. Alpha outside the marks is preserved."""
+ from PIL import Image
+
+ if localized.mode == "annotate":
+ overlay = _decode_bounded(localized.image, "RGBA", budget, "The annotation layer")
+ overlay = _to_source_geometry(overlay, source, "annotation layer", Image.LANCZOS)
+ base = source.convert("RGBA")
+ marked = Image.alpha_composite(base, overlay)
+ return [marked if source.mode == "RGBA" else marked.convert(source.mode)]
+ if localized.mode == "paint":
+ mask = _decode_bounded(localized.image, "L", budget, "The painted region")
+ mask = _to_source_geometry(mask, source, "painted region", Image.LANCZOS)
+ base = source.convert("RGBA")
+ white = Image.new("RGBA", base.size, (255, 255, 255, 255))
+ painted = Image.composite(white, base, mask)
+ return [painted if source.mode == "RGBA" else painted.convert(source.mode)]
+ if localized.mode == "mask":
+ mask = _decode_bounded(localized.image, "L", budget, "The mask")
+ # NEAREST keeps it binary; the threshold then fixes the polarity white = region.
+ mask = _to_source_geometry(mask, source, "mask", Image.NEAREST)
+ mask = mask.point(lambda v: 255 if v >= 128 else 0)
+ if not mask.getbbox():
+ raise ValueError("The mask is empty: paint the region to edit in white.")
+ return [source, mask.convert("RGB")]
+ raise ValueError("localized_edit mode must be one of " + ", ".join(LOCALIZED_EDIT_MODES) + ".")
+
+
+def decode_condition_images(
+ fam: Any,
+ init_image: str,
+ reference_images: Optional[Sequence[str]],
+ localized: Optional[LocalizedEdit] = None,
+) -> list[Any]:
+ """The ordered condition list: the (possibly marked) source, a separate mask if any, then every
+ reference in request order. More images than the family takes is refused, not truncated."""
+ limit = int(getattr(fam, "max_condition_images", 4) or 4)
+ mode = getattr(fam, "condition_image_mode", "RGB") or "RGB"
+ refs = [r for r in (reference_images or []) if r]
+ extra_from_mask = 1 if localized is not None and localized.mode == "mask" else 0
+ total = 1 + extra_from_mask + len(refs)
+ if total > limit:
+ raise ValueError(
+ f"The {getattr(fam, 'name', 'loaded')} model takes at most {limit} input images in "
+ f"total (got {total}" + (", counting the mask" if extra_from_mask else "") + ")."
+ )
+ budget = [MAX_CONDITION_SOURCE_PIXELS]
+ source = _decode_bounded(init_image, mode, budget, "The source image")
+ images = [source]
+ if localized is not None:
+ images = apply_localized_edit(source, localized, budget)
+ for ref in refs:
+ images.append(_decode_bounded(ref, mode, budget, f"Image {len(images) + 1}"))
+ return images
+
+
+def conditioning_capabilities(fam: Any, workflows: Sequence[str]) -> dict[str, Any]:
+ """What the UI may offer for ``fam`` given the ``workflows`` the active engine runs for it.
+ Localized editing is advertised only where the unified edit workflow itself is."""
+ unified = bool(getattr(fam, "unified_edit", False)) and "edit" in workflows
+ return {
+ "max_condition_images": int(getattr(fam, "max_condition_images", 4) or 4),
+ "alpha": (getattr(fam, "condition_image_mode", "RGB") or "RGB") == "RGBA",
+ "dimension_multiple": int(getattr(fam, "dimension_multiple", 16) or 16),
+ "max_output_side": int(getattr(fam, "max_output_side", 2048) or 2048),
+ "max_output_pixels": int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048),
+ "reference_resolutions": list(getattr(fam, "reference_resolutions", ()) or ()),
+ "unified_edit": unified,
+ "localized_edit_modes": list(LOCALIZED_EDIT_MODES) if unified else [],
+ }
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 250830b3241..b08ae0ab9a8 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -99,6 +99,19 @@ class DiffusionFamily:
# True for families whose text-to-image pipeline ALSO accepts reference image(s) (FLUX.2 ``image``): no
# ``strength``, size from width/height.
reference: bool = False
+ # True when the text-to-image pipeline also follows edit instructions over ``image`` (Qwen-Image-2.1), at the
+ # requested size. Never inferred from ``reference``: a reference family is not necessarily trained to edit.
+ unified_edit: bool = False
+ # Condition images per call, INCLUDING the init image; overflow is refused, never sliced.
+ max_condition_images: int = 4
+ condition_image_mode: str = "RGB"
+ dimension_multiple: int = 16
+ max_output_side: int = 2048
+ max_output_pixels: int = 2048 * 2048
+ # Accepted condition-image preprocessing resolutions (square side, by area); empty = no such control.
+ reference_resolutions: tuple[int, ...] = field(default_factory = tuple)
+ # Activation-guard cost of one condition pixel relative to one output pixel.
+ condition_pixel_weight: float = 1.0
# Extra lowercased substrings (besides ``name``) that map a repo id here.
aliases: tuple[str, ...] = field(default_factory = tuple)
# True for families whose activations overflow float16 (-> black image); the backend promotes a resolved float16
@@ -156,6 +169,8 @@ class DiffusionFamily:
# carries the upstream base in its tag name whatever tree was built, so the tag cannot answer
# this, and a custom SD_CLI_PATH build has no record at all.
sd_cpp_arch_marker: Optional[str] = None
+ # Literal an sd.cpp build carries once it can EDIT this family; the arch marker proves text-to-image only.
+ sd_cpp_edit_marker: Optional[str] = None
# True when Unsloth can TRAIN a LoRA on this family; the training-start path refuses a non-trainable family up
# front.
trainable: bool = False
@@ -396,6 +411,18 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# passes. Without this flag diffusion.py refuses reference images for this family and the
# capability ships dark.
reference = True,
+ # Model card: ten inputs, alpha kept for the VAE, 32 px grid (16x VAE, 2x2 token groups), 2K presets up to
+ # 2400 x 1792. ``reference_resolutions`` maps to the pipeline's ``output_resolution``.
+ unified_edit = True,
+ max_condition_images = 10,
+ condition_image_mode = "RGBA",
+ dimension_multiple = 32,
+ max_output_side = 2752,
+ max_output_pixels = 2400 * 1792,
+ reference_resolutions = (512, 1024, 2048),
+ # Condition tokens are prefilled once into a KV cache, not denoised per step. Measured on an RTX 3090: about
+ # 2.5-2.7 GiB per 1024-square condition image against the estimator's 8 GiB per output megapixel.
+ condition_pixel_weight = 0.32,
aliases = ("qwen_image_21", "qwenimage21", "qwen-image-21"),
# Built by us from Qwen/Qwen-Image-2.1 itself: the same 238 tensors under upstream's own
# names, cast fp32 -> bf16 and written as one file, because sd-cli takes --vae as a single
@@ -422,16 +449,15 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# 5.0), with the per-tensor precision the dynamic recipe assigns. Measured on this host
# against the 2.1-capable build, one prompt at 1024 with 20 steps and a shared seed:
# LPIPS 0.02894 / SSIM 0.95908 between the two, 36.5 s against 39.0 s, both coherent.
- # Text to image only, which is all this family exposes (edit is False, and there are no
- # img2img / inpaint pipelines). Upstream's docs/qwen_image_2.1.md requires a separate
- # mmproj through --llm_vision before a GGUF encoder can do image editing; turning editing
- # on here without adding it would load an encoder that logs "vision disabled" and carry on.
+ # The vision projector (--llm_vision) is what native editing needs; text-to-image never reads it (a fixed-seed
+ # render is byte-identical with and without it). Listed here so download, readiness and delete guards see it.
sd_cpp_text_encoders = (
(
"unsloth/Qwen3-VL-8B-Instruct-GGUF",
"Qwen3-VL-8B-Instruct-UD-Q4_K_XL.gguf",
"llm",
),
+ ("unsloth/Qwen3-VL-8B-Instruct-GGUF", "mmproj-F16.gguf", "llm_vision"),
),
sd_cpp_sampling_method = "euler",
# No flow shift on purpose. Qwen-Image pins 3.0, but upstream sd.cpp selects a
@@ -443,6 +469,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
# release (both sd-cli and sd-server, CPU and CUDA) carries this literal, and the five
# older ones, including the previous pin, do not.
sd_cpp_arch_marker = "qwen_image_2_1",
+ # The edit path's no-projector refusal: in the pinned build, absent from the one before it.
+ sd_cpp_edit_marker = "Qwen Image 2.1 editing requires Qwen3-VL vision weights",
),
DiffusionFamily(
name = "z-image",
@@ -1098,6 +1126,11 @@ _GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
("flux.2-klein-base", 50, 4.0),
("flux.2-klein", 4, 1.0),
("flux.2-dev", 28, 4.0), # full (non-distilled)
+ # Qwen-Image-2.1: 40 steps, no guidance. Before the generic qwen-image key.
+ ("qwen-image-2.1", 40, 1.0),
+ ("qwen-image-21", 40, 1.0),
+ ("qwen_image_21", 40, 1.0),
+ ("qwenimage21", 40, 1.0),
("qwen-image", 20, 4.0),
("z-image", 20, 4.0),
# Lumina Image 2.0 card: 50 steps, guidance 4 (plus cfg_trunc_ratio 0.25, which the loader passes itself).
diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py
index 516966bde69..3e5a8182e25 100644
--- a/studio/backend/core/inference/diffusion_memory.py
+++ b/studio/backend/core/inference/diffusion_memory.py
@@ -555,13 +555,16 @@ def estimate_image_runtime_mib(
height: Optional[int],
batch_size: int = 1,
family: Optional[str] = None,
+ condition_pixels: int = 0,
) -> int:
"""Per-call activation / latent headroom for an image gen, scaled by pixel area and batch.
- Distilled / turbo models (few steps, no CFG) need less."""
+ Distilled / turbo models (few steps, no CFG) need less. ``condition_pixels`` (already weighted)
+ join the target's token sequence once per batch image, so they count like target pixels."""
w = max(64, int(width or DEFAULT_IMAGE_WIDTH))
h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
batch = max(1, int(batch_size or 1))
- pixel_scale = (w * h * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)
+ cond = max(0, int(condition_pixels or 0))
+ pixel_scale = ((w * h + cond) * batch) / float(DEFAULT_IMAGE_WIDTH * DEFAULT_IMAGE_HEIGHT)
fam = (family or "").lower()
multiplier = 1.0
if "edit" in fam:
@@ -1070,6 +1073,39 @@ def _enable_vae_saver(pipe: Any, pipe_method: str, vae_method: str, logger: Any)
return False
+def _pin_vision_embedding_device(module: Any) -> int:
+ """Keep a leaf-offloaded Qwen3-VL vision tower's position embeddings on the compute device.
+
+ ``Qwen3VLVisionModel.fast_pos_embed_interpolate`` reads ``self.pos_embed.weight.device`` while
+ group offload still holds that weight on the CPU, so every image-conditioned Qwen-Image-2.1 call
+ on a streamed encoder raised a device mismatch. The result follows ``grid_thw``'s device."""
+ import types
+
+ walk = getattr(module, "modules", None)
+ if not callable(walk):
+ return 0
+ patched = 0
+ for sub in walk():
+ original = getattr(sub, "fast_pos_embed_interpolate", None)
+ if not callable(original) or getattr(original, "_unsloth_device_pinned", False):
+ continue
+
+ def _on_grid_device(
+ self: Any,
+ grid_thw: Any,
+ *,
+ _original: Any = original,
+ ) -> Any:
+ out = _original(grid_thw)
+ device = getattr(grid_thw, "device", None)
+ return out.to(device) if device is not None and out.device != device else out
+
+ _on_grid_device._unsloth_device_pinned = True # type: ignore[attr-defined]
+ sub.fast_pos_embed_interpolate = types.MethodType(_on_grid_device, sub)
+ patched += 1
+ return patched
+
+
def _apply_group_offload(
pipe: Any,
device: str,
@@ -1168,6 +1204,7 @@ def _apply_group_offload(
try:
apply_group_offloading(module, **ekwargs)
installed += 1
+ _pin_vision_embedding_device(module)
except Exception as exc: # noqa: BLE001 -- degrade this encoder, never fail the load
if logger is not None:
logger.warning(
@@ -1231,6 +1268,7 @@ def image_activation_shortfall_message(
family: Optional[str] = None,
base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
source_driven: bool = False,
+ condition_pixels: int = 0,
) -> Optional[str]:
"""A user-facing refusal when this generation's ACTIVATIONS plus the flat base overhead cannot fit
the free device budget, else None.
@@ -1285,6 +1323,7 @@ def image_activation_shortfall_message(
height = height,
batch_size = batch_size,
family = family,
+ condition_pixels = condition_pixels,
)
# What the LOAD budgeted: the same estimator at the default resolution, i.e. the exact call _plan_memory makes.
# Same function and same family hint, so the comparison is between two points on one curve rather than between
@@ -1309,13 +1348,14 @@ def image_activation_shortfall_message(
h = max(64, int(height or DEFAULT_IMAGE_HEIGHT))
batch = max(1, int(batch_size or 1))
batch_note = f" at a batch of {batch}" if batch > 1 else ""
+ cond_note = " with its input images" if condition_pixels else ""
# Two decimals, not one: the refusal is often decided by tens of MiB (the 1088x1920 report needed 13,872 MiB against
# a 13,822 MiB budget), and one decimal prints both as "13.5 GB". The overhead is part of the comparison above, so
# it has to be part of the number reported. Quoting the activations alone printed a refusal that contradicted
# itself: 13.55 GB needed against 13.92 GB usable, refused.
total = int(needed) + max(0, int(base_overhead_mib))
return (
- f"Generating at {w}x{h}{batch_note} needs about {total / 1024:.2f} GB of working memory "
+ f"Generating at {w}x{h}{batch_note}{cond_note} needs about {total / 1024:.2f} GB of working memory "
f"(including about {max(0, int(base_overhead_mib)) / 1024:.2f} GB of fixed overhead), "
f"but only about {int(budget) / 1024:.2f} GB is usable on this device (of the "
f"{int(free) / 1024:.2f} GB currently free, after reserving room for fragmentation and "
@@ -1324,7 +1364,8 @@ def image_activation_shortfall_message(
# Only when the batch is what was budgeted. A refusal measured on ONE image cannot be answered by asking for
# fewer, and pointing there sends the caller at the one change that provably will not help.
f"{'Upload a smaller source image (this workflow takes its output size from the image, not the Resolution setting)' if source_driven else 'Generate at a smaller resolution'}"
- f"{' or a smaller batch size' if batch > 1 else ''}, "
+ f"{' or a smaller batch size' if batch > 1 else ''}"
+ f"{', use fewer input images or a lower reference detail' if condition_pixels else ''}, "
"free device memory by closing other applications, or set "
f"{OVERSIZED_GENERATE_ENV}=1 to attempt it anyway."
)
@@ -1339,6 +1380,7 @@ def raise_on_image_activation_shortfall(
family: Optional[str] = None,
base_overhead_mib: int = DEFAULT_BASE_OVERHEAD_MIB,
source_driven: bool = False,
+ condition_pixels: int = 0,
logger: Any = None,
) -> None:
"""Refuse a generation whose activations cannot fit the free device budget. No-op whenever
@@ -1356,6 +1398,7 @@ def raise_on_image_activation_shortfall(
family = family,
base_overhead_mib = base_overhead_mib,
source_driven = source_driven,
+ condition_pixels = condition_pixels,
)
if message is None:
return
@@ -1418,6 +1461,8 @@ def _apply_streaming_offload(pipe: Any, device: str, logger: Any) -> None:
kwargs["low_cpu_mem_usage"] = True
apply_group_offloading(module, **kwargs)
installed += 1
+ if offload_type == "leaf_level":
+ _pin_vision_embedding_device(module)
except Exception as exc:
if logger is not None:
logger.warning(
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index 22c63857e48..fef1045cb5e 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -332,6 +332,7 @@ def build_sd_cpp_command(
("--clip_g", files.clip_g),
("--t5xxl", files.t5xxl),
("--llm", files.llm),
+ ("--llm_vision", files.llm_vision),
("--qwen2vl", files.qwen2vl),
):
if value:
@@ -555,6 +556,7 @@ def build_sd_cpp_server_command(
("--clip_g", files.clip_g),
("--t5xxl", files.t5xxl),
("--llm", files.llm),
+ ("--llm_vision", files.llm_vision),
("--qwen2vl", files.qwen2vl),
):
if value:
@@ -602,6 +604,7 @@ def build_img_gen_request(
distilled_guidance: Optional[float] = None,
output_format: str = "png",
lora: Optional[list[dict]] = None,
+ ref_images: Optional[list[str]] = None,
) -> dict:
"""Build the ``POST /sdcpp/v1/img_gen`` JSON body for one text-to-image request.
@@ -644,6 +647,9 @@ def build_img_gen_request(
# ``<lora:>`` tags are unsupported server-side), so LoRAs are staged here.
if lora:
req["lora"] = lora
+ # Base64 PNGs in model order; no init_image/strength/mask: this is reference conditioning, not img2img.
+ if ref_images:
+ req["ref_images"] = list(ref_images)
return req
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index c6ebb146742..97632af3a2e 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -18,6 +18,7 @@ selecting it on a CPU box does not drag the heavy GPU stack into the process.
from __future__ import annotations
+import base64
import contextlib
import logging
import os
@@ -1573,7 +1574,9 @@ def _superseded_legacy_server(binary: Optional[str], accelerator: str) -> bool:
_ARCH_MARKER_CACHE: dict[tuple[str, int, int, str], bool] = {}
-def binary_carries_marker(binary: Optional[str], marker: Optional[str]) -> bool:
+def binary_carries_marker(
+ binary: Optional[str], marker: Optional[str], *, unreadable: bool = True
+) -> bool:
"""Whether the sd.cpp executable at ``binary`` contains the literal ``marker``.
An architecture upstream has not implemented leaves no trace in the build, so this answers
@@ -1584,7 +1587,8 @@ def binary_carries_marker(binary: Optional[str], marker: Optional[str]) -> bool:
No marker means no claim, so an unmarked family is True and nothing changes for it. An
unreadable binary is True as well: refusing a load because a stat failed would take the native
route away on a host where it works, and the old behaviour (attempt, fail, surface the error)
- is no worse than what this replaces.
+ is no worse than what this replaces. An optional capability passes ``unreadable = False`` to fail
+ closed instead.
Scanned in chunks with an overlap, since the literal can straddle a boundary, and memoised on
(path, size, mtime) so a reinstall is noticed while repeated loads read 100+ MB once.
@@ -1597,7 +1601,7 @@ def binary_carries_marker(binary: Optional[str], marker: Optional[str]) -> bool:
st = os.stat(binary)
key = (str(binary), int(st.st_size), int(st.st_mtime_ns), marker)
except OSError:
- return True
+ return unreadable
hit = _ARCH_MARKER_CACHE.get(key)
if hit is not None:
return hit
@@ -1616,11 +1620,83 @@ def binary_carries_marker(binary: Optional[str], marker: Optional[str]) -> bool:
break
tail = block[-(len(needle) - 1) :] if len(needle) > 1 else b""
except OSError:
- return True
+ return unreadable
_ARCH_MARKER_CACHE[key] = found
return found
+def _native_output_image(fam: Any, im: Any) -> Any:
+ """A rendered image as the gallery keeps it: alpha for a family that generates transparency, as
+ the diffusers engine returns it, else RGB."""
+ if getattr(fam, "condition_image_mode", "RGB") == "RGBA" and im.mode in ("RGBA", "LA", "P"):
+ return im.convert("RGBA")
+ return im.convert("RGB")
+
+
+# Carried only by sd.cpp builds that keep reference alpha (upstream e112ab5) and no longer centre-crop references
+# on sd-server (78557f8 / e012065). The pinned build predates all three.
+_REFERENCE_FIDELITY_MARKER = "error: allocate memory for channel promotion"
+
+
+def _native_condition_images(
+ fam: Any,
+ init_image: str,
+ reference_images: Optional[list[str]],
+ localized_edit: Any,
+ width: Optional[int],
+ height: Optional[int],
+ *,
+ full_fidelity: bool,
+ pad_to_output: bool,
+) -> tuple[int, int, list[bytes]]:
+ """(width, height, ordered PNG bytes) for one native reference / edit call, decoded through
+ the diffusers engine's helper. A ``full_fidelity`` build gets every image as decoded. An older
+ build reads references as RGB, so each is flattened over white first (else transparent pixels
+ become noise), and its sd-server centre-crops references to the output aspect, so with
+ ``pad_to_output`` each is padded to it instead: white for images, black for a separate mask.
+ """
+ import io
+
+ from PIL import Image
+
+ from core.inference.diffusion_conditioning import (
+ check_output_size,
+ decode_condition_images,
+ match_source_size,
+ )
+
+ images = decode_condition_images(fam, init_image, reference_images, localized_edit)
+ if width is None or height is None:
+ width, height = match_source_size(fam, images[0].size, 1024)
+ check_output_size(fam, int(width), int(height))
+ target = float(width) / float(height)
+ mask_index = 1 if getattr(localized_edit, "mode", None) == "mask" else None
+ blobs: list[bytes] = []
+ for index, img in enumerate(images):
+ if full_fidelity:
+ img = img if img.mode in ("RGB", "RGBA") else img.convert("RGBA")
+ elif img.mode in ("RGBA", "LA", "P"):
+ rgba = img.convert("RGBA")
+ flat = Image.new("RGB", rgba.size, (255, 255, 255))
+ flat.paste(rgba, mask = rgba.getchannel("A"))
+ img = flat
+ else:
+ img = img.convert("RGB")
+ if pad_to_output and not full_fidelity:
+ iw, ih = img.size
+ ratio = iw / float(ih)
+ if abs(ratio - target) > 0.01 * target:
+ pw, ph = (round(ih * target), ih) if ratio < target else (iw, round(iw / target))
+ fill = (0, 0, 0) if index == mask_index else (255, 255, 255)
+ canvas = Image.new("RGB", (max(pw, iw), max(ph, ih)), fill)
+ canvas.paste(img, ((canvas.width - iw) // 2, (canvas.height - ih) // 2))
+ img = canvas
+ buf = io.BytesIO()
+ img.save(buf, format = "PNG")
+ blobs.append(buf.getvalue())
+ return int(width), int(height), blobs
+
+
def sd_cpp_binary_runs_family(binary: Optional[str], fam: Any) -> bool:
"""Whether the sd.cpp build at ``binary`` implements ``fam``'s architecture.
@@ -2583,6 +2659,7 @@ class SdCppDiffusionBackend:
clip_g = paths.get("clip_g"),
t5xxl = paths.get("t5xxl"),
llm = paths.get("llm"),
+ llm_vision = paths.get("llm_vision"),
qwen2vl = paths.get("qwen2vl"),
)
device = resolve_diffusion_device_target().device
@@ -3181,6 +3258,14 @@ class SdCppDiffusionBackend:
# (unreachable) online case rather than relabelled, so nothing changes when the flag is off.
if not local_files_only:
raise
+ if kind == "llm_vision":
+ # Only editing reads the projector, and edit is offered only when it is loaded. A
+ # Qwen-Image-2.1 GGUF cached before the projector was listed still loads for
+ # text-to-image; opening it from the Images page fetches the projector.
+ logger.info(
+ "sd_cpp.llm_vision_not_cached: %s/%s, editing unavailable", repo, fn
+ )
+ continue
raise RuntimeError(
f"'{fn}' is not in the local cache for '{repo}', and this load may not "
f"download (it was not user-initiated). Open the model from the Images "
@@ -3250,13 +3335,41 @@ class SdCppDiffusionBackend:
)
return _with_mirrors(repos)
+ def _native_binary(self, state: _SdState) -> Optional[str]:
+ """The sd.cpp binary this load runs: the resident server's, else the one-shot engine's."""
+ binary = getattr(state.server, "binary", None) if state.server is not None else None
+ return binary or getattr(getattr(self, "_engine", None), "binary", None)
+
+ def _native_reference_fidelity(self, state: Optional[_SdState]) -> bool:
+ """Whether this load's build reads reference images with their alpha and their own size."""
+ if state is None:
+ return False
+ binary = self._native_binary(state)
+ return bool(binary) and binary_carries_marker(
+ binary, _REFERENCE_FIDELITY_MARKER, unreadable = False
+ )
+
+ def _native_edit_ready(self, state: Optional[_SdState]) -> bool:
+ """Whether this load can run the unified edit workflow natively: a unified-edit family, its
+ vision projector loaded, and a build carrying the family's edit marker."""
+ if state is None:
+ return False
+ fam = state.family
+ marker = getattr(fam, "sd_cpp_edit_marker", None)
+ if not (getattr(fam, "unified_edit", False) and marker and state.files.llm_vision):
+ return False
+ binary = self._native_binary(state)
+ if not binary:
+ return False
+ return binary_carries_marker(binary, marker, unreadable = False)
+
def generate(
self,
*,
prompt: str,
negative_prompt: Optional[str] = None,
- width: int = 1024,
- height: int = 1024,
+ width: Optional[int] = 1024,
+ height: Optional[int] = 1024,
steps: int = 9,
guidance: float = 0.0,
seed: Optional[int] = None,
@@ -3271,6 +3384,9 @@ class SdCppDiffusionBackend:
strength: Optional[float] = None,
upscale: Optional[float] = None, # needs an init image; rejected by the guard below
reference_images: Optional[list[str]] = None, # GPU/diffusers-only (FLUX.2)
+ workflow: Optional[str] = None,
+ reference_resolution: Optional[int] = None,
+ localized_edit: Any = None,
# LoRA (id, weight) pairs; resolved up front then applied per path: prompt tags for one-shot sd-cli,
# structured `lora` for sd-server.
loras: Optional[list[tuple[str, float]]] = None,
@@ -3285,10 +3401,23 @@ class SdCppDiffusionBackend:
from core.inference import diffusion_lora
- if (
+ conditioned = workflow in ("edit", "reference")
+ if conditioned:
+ if init_image is None:
+ raise ValueError(f"The {workflow} workflow requires a source image (init_image).")
+ if reference_resolution is not None:
+ # No such control natively: sd.cpp sizes every input image to the output area itself.
+ raise ValueError(
+ "Reference detail is not adjustable on the native sd.cpp engine: it resizes "
+ "each input image to the output area. Leave reference_resolution unset."
+ )
+ elif (
init_image is not None
or mask_image is not None
or reference_images
+ or workflow is not None
+ or reference_resolution is not None
+ or localized_edit is not None
or (upscale is not None and upscale > 1)
):
raise ValueError(
@@ -3337,6 +3466,40 @@ class SdCppDiffusionBackend:
# idle while this holds _generate_lock.
self._gen = _SdGen(total_steps = int(steps))
try:
+ ref_pngs: list[bytes] = []
+ if conditioned:
+ from core.inference.diffusion_conditioning import check_conditioned_fields
+
+ check_conditioned_fields(
+ workflow,
+ state.family,
+ mask_image = mask_image,
+ strength = strength,
+ upscale = upscale,
+ controlnet = controlnet,
+ localized_edit = localized_edit,
+ )
+ if not self._native_edit_ready(state):
+ raise ValueError(
+ f"Image editing is not available for '{state.family.name}' on the native "
+ "sd.cpp engine with this build and its assets."
+ )
+ width, height, ref_pngs = _native_condition_images(
+ state.family,
+ init_image,
+ reference_images,
+ localized_edit,
+ width,
+ height,
+ full_fidelity = self._native_reference_fidelity(state),
+ pad_to_output = state.mode == "server" and state.server is not None,
+ )
+ elif width is None or height is None:
+ raise ValueError("width and height are required for this workflow.")
+ else:
+ from core.inference.diffusion_conditioning import check_output_size
+
+ check_output_size(state.family, int(width), int(height))
if seed is None:
seed = int.from_bytes(os.urandom(6), "big") & ((1 << 53) - 1)
else:
@@ -3378,6 +3541,7 @@ class SdCppDiffusionBackend:
flux_guidance = flux_guidance,
lora_resolved = lora_resolved,
cancel = cancel,
+ ref_pngs = ref_pngs,
)
else:
images, seeds = self._generate_oneshot(
@@ -3393,6 +3557,7 @@ class SdCppDiffusionBackend:
flux_guidance = flux_guidance,
lora_resolved = lora_resolved,
cancel = cancel,
+ ref_pngs = ref_pngs,
)
except RuntimeError as exc:
# The mid-render hipBLAS death the video path records too; not a cancellation.
@@ -3440,6 +3605,11 @@ class SdCppDiffusionBackend:
"offload_policy": (
"active" if without_device_backend_flags(state.offload_flags) else "none"
),
+ "workflow": workflow if conditioned else "txt2img",
+ "reference_resolution": None,
+ "localized_edit": getattr(localized_edit, "mode", None)
+ if conditioned
+ else None,
}
except SdCppCancelled as exc:
raise RuntimeError(DIFFUSION_CANCELLED_MSG) from exc
@@ -3465,6 +3635,7 @@ class SdCppDiffusionBackend:
flux_guidance: Optional[float],
lora_resolved: list,
cancel: threading.Event,
+ ref_pngs: Optional[list[bytes]] = None,
) -> tuple[list, list[int]]:
"""Generate via the resident sd-server (no model reload).
@@ -3533,6 +3704,10 @@ class SdCppDiffusionBackend:
cfg_scale = cfg_scale,
distilled_guidance = flux_guidance,
lora = lora_payload,
+ ref_images = [
+ "data:image/png;base64," + base64.b64encode(b).decode("ascii")
+ for b in ref_pngs or []
+ ],
)
try:
blobs = state.server.img_gen(
@@ -3562,7 +3737,9 @@ class SdCppDiffusionBackend:
raise RuntimeError(
f"sd-server returned {len(blobs)} of {count} requested images in the batch."
)
- images.extend(Image.open(io.BytesIO(b)).convert("RGB") for b in blobs)
+ images.extend(
+ _native_output_image(state.family, Image.open(io.BytesIO(b))) for b in blobs
+ )
# sd.cpp advances the seed per image within a job, so report chunk_seed+i.
seeds.extend((chunk_seed + i) & ((1 << 63) - 1) for i in range(len(blobs)))
finally:
@@ -3644,6 +3821,7 @@ class SdCppDiffusionBackend:
flux_guidance: Optional[float],
lora_resolved: list,
cancel: threading.Event,
+ ref_pngs: Optional[list[bytes]] = None,
) -> tuple[list, list[int]]:
"""Fallback path: re-run one-shot sd-cli per image (reloads the model each time). LoRA on
the one-shot path uses sd-cli's own mechanism: materialize the selected adapters into a
@@ -3675,6 +3853,12 @@ class SdCppDiffusionBackend:
)
eff_prompt = diffusion_lora.inject_prompt_tags(prompt, materialized)
lora_dir = str(Path(tmpdir) / "loras")
+ # Inside the run's temporary directory, so every exit path removes them.
+ ref_paths: list[str] = []
+ for i, blob in enumerate(ref_pngs or []):
+ ref_path = Path(tmpdir) / f"ref_{i + 1:02d}.png"
+ ref_path.write_bytes(blob)
+ ref_paths.append(str(ref_path))
for index in range(max(1, int(batch_size))):
if cancel.is_set():
raise RuntimeError(DIFFUSION_CANCELLED_MSG)
@@ -3695,6 +3879,7 @@ class SdCppDiffusionBackend:
batch_count = 1,
lora_dir = lora_dir,
lora_apply_mode = "auto" if lora_dir else None,
+ ref_images = tuple(ref_paths),
)
# Each sd-cli run executes out of the managed tree, so hold installs off for its duration (and wait
# here if one is already extracting). getattr: an INJECTED engine is the unit-test seam / escape hatch
@@ -3732,7 +3917,7 @@ class SdCppDiffusionBackend:
cancel_event = cancel,
)
with Image.open(out_path) as im:
- images.append(im.copy())
+ images.append(_native_output_image(state.family, im.copy()))
seeds.append(seed_i)
return images, seeds
@@ -3872,10 +4057,37 @@ class SdCppDiffusionBackend:
"supports_lora": False,
"supports_controlnet": False,
"workflows": [],
+ "conditioning": None,
}
from core.inference import diffusion_lora
+ from core.inference.diffusion_conditioning import conditioning_capabilities
from hub.utils.gguf import extract_quant_token
+ workflows = ["txt2img"]
+ if self._native_edit_ready(state):
+ workflows += ["reference", "edit"]
+ conditioning = conditioning_capabilities(state.family, workflows)
+ # No reference detail natively: sd.cpp sizes inputs to the output area.
+ conditioning["reference_resolutions"] = []
+ full_fidelity = self._native_reference_fidelity(state)
+ conditioning["alpha"] = full_fidelity
+ notes: list[str] = []
+ if "edit" in workflows:
+ if not full_fidelity:
+ notes.append(
+ "Transparent parts of input images are filled with white on this native build."
+ )
+ if state.mode == "server":
+ notes.append(
+ "Input images with a different shape from the output are padded to its "
+ "aspect ratio on this native build."
+ )
+ # Measured with upstream c92d73c: 3.7% of pixels stayed transparent at guidance 1, 47.1% at 6.
+ notes.append(
+ "For transparent output on the native engine, raise Guidance above 1 (upstream uses 6)."
+ )
+ conditioning["notes"] = notes
+
return {
"loaded": True,
"repo_id": state.repo_id,
@@ -3912,8 +4124,9 @@ class SdCppDiffusionBackend:
"supports_controlnet": False,
# "server" = resident sd-server (load once); "oneshot" = legacy per-image sd-cli.
"native_mode": state.mode,
- # Native supports txt2img only; advertise it so the UI doesn't disable the Create tab.
- "workflows": ["txt2img"],
+ # txt2img always; reference and edit only where this build and its loaded assets run them.
+ "workflows": workflows,
+ "conditioning": conditioning,
}
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 8b0bb9e9399..de408536e13 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3981,6 +3981,27 @@ class ControlNetSpec(BaseModel):
return self
+class LocalizedEditSpec(BaseModel):
+ """A localized edit for the unified-edit workflow (Qwen-Image-2.1). It guides a generative edit;
+ unlike inpainting it does not keep the pixels outside the region."""
+
+ mode: Literal["annotate", "paint", "mask"] = Field(
+ ...,
+ description = "annotate: `image` is an RGBA layer of marks composited over the source. "
+ "paint: `image` is a mask whose white area is painted white onto the source. mask: `image` "
+ "is a white-on-black mask sent as Image 2, right after the source.",
+ )
+ image: str = Field(
+ ...,
+ max_length = 32 * 1024 * 1024,
+ description = "Base64/data-URL layer at the source image's geometry",
+ )
+
+
+# All images of one request, base64: ten maximal uploads would otherwise buffer ~320 MiB.
+_MAX_CONDITION_PAYLOAD = 128 * 1024 * 1024
+
+
class DiffusionGenerateRequest(BaseModel):
"""Request to generate one image from the loaded diffusion model."""
@@ -3988,9 +4009,18 @@ class DiffusionGenerateRequest(BaseModel):
negative_prompt: Optional[str] = Field(
None, description = "What to avoid (if the model supports it)"
)
- width: int = Field(1024, ge = 256, le = 2048, description = "Image width in pixels (multiple of 16)")
+ # Transport ceiling = the largest 2K preset side; the loaded family enforces its own bounds and grid.
+ width: int = Field(
+ 1024,
+ ge = 256,
+ le = 2752,
+ description = "Image width in pixels (multiple of 16; the loaded model may require more)",
+ )
height: int = Field(
- 1024, ge = 256, le = 2048, description = "Image height in pixels (multiple of 16)"
+ 1024,
+ ge = 256,
+ le = 2752,
+ description = "Image height in pixels (multiple of 16; the loaded model may require more)",
)
steps: int = Field(9, ge = 1, le = 100, description = "Number of denoising steps")
guidance: float = Field(0.0, ge = 0.0, le = 20.0, description = "Classifier-free guidance scale")
@@ -4078,9 +4108,28 @@ class DiffusionGenerateRequest(BaseModel):
)
reference_images: Optional[list[str]] = Field(
None,
- max_length = 3,
- description = "Additional reference images (base64/data-URL) for the FLUX.2 reference "
- "workflow, combined with init_image. Up to 3; ignored by other workflows.",
+ max_length = 9,
+ description = "Additional images (base64/data-URL) for the reference and edit workflows, "
+ "after init_image and in this order. The loaded family bounds the total including "
+ "init_image (FLUX.2: 4, Qwen-Image-2.1: 10); more is refused, never truncated.",
+ )
+ workflow: Optional[Literal["edit", "reference"]] = Field(
+ None,
+ description = "Explicit image-conditioned workflow. edit: follow the prompt as an "
+ "instruction over init_image (and reference_images); reference: generate a new image "
+ "guided by them. Omitted keeps the workflow implied by the other fields.",
+ )
+ reference_resolution: Optional[int] = Field(
+ None,
+ ge = 256,
+ le = 2048,
+ description = "Resolution each condition image is resized to (by area) before encoding, "
+ "for families that expose it (Qwen-Image-2.1: 512, 1024 or 2048; default 1024). "
+ "Separate from the output size.",
+ )
+ localized_edit: Optional[LocalizedEditSpec] = Field(
+ None,
+ description = "Localized edit layer for the edit workflow on a unified-edit model.",
)
loras: Optional[list[LoraSpec]] = Field(
None,
@@ -4120,6 +4169,17 @@ class DiffusionGenerateRequest(BaseModel):
raise ValueError("each reference image must be at most 32 MiB (base64)")
return value
+ @model_validator(mode = "after")
+ def _bounded_condition_payload(self) -> "DiffusionGenerateRequest":
+ total = len(self.init_image or "") + sum(len(r) for r in self.reference_images or [])
+ if self.localized_edit is not None:
+ total += len(self.localized_edit.image)
+ if total > _MAX_CONDITION_PAYLOAD:
+ raise ValueError(
+ "the input images together must be at most 128 MiB (base64); use smaller images"
+ )
+ return self
+
@field_validator("width", "height")
@classmethod
def _multiple_of_16(cls, value: int) -> int:
@@ -4218,7 +4278,15 @@ class GalleryImage(BaseModel):
None, description = "ControlNet guidance interval, formatted as 'start:end'"
)
reference_image_count: Optional[int] = Field(
- None, description = "How many reference images the reference workflow used"
+ None,
+ description = "How many ADDITIONAL images (reference_images, beyond the source) the "
+ "reference or edit workflow used",
+ )
+ reference_resolution: Optional[int] = Field(
+ None, description = "Condition-image preprocessing resolution, when the model has one"
+ )
+ localized_edit: Optional[str] = Field(
+ None, description = "Localized edit convention used: annotate, paint or mask"
)
created_at: float = Field(..., description = "Creation time (epoch seconds)")
# Library state, not recipe: stored beside the PNG, so older files simply read as unset.
@@ -4406,6 +4474,13 @@ class DiffusionStatusResponse(BaseModel):
description = "Image workflows the loaded family supports (drives UI tab gating): "
"txt2img, img2img, inpaint. Empty when nothing is loaded or on the native engine.",
)
+ conditioning: Optional[Dict[str, Any]] = Field(
+ None,
+ description = "Image-conditioning limits of the loaded model on the active engine: "
+ "max_condition_images (total, including the source), alpha, dimension_multiple, "
+ "max_output_side, max_output_pixels, reference_resolutions and localized_edit_modes. "
+ "Null when nothing is loaded.",
+ )
engine: Optional[str] = Field(None, description = "Active diffusion engine: diffusers | sd_cpp")
native_mode: Optional[str] = Field(
None,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 518332f9526..0f27a617733 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -39275,9 +39275,20 @@ async def generate_diffusion_image(
load_identity,
)
+ from core.inference.diffusion_conditioning import LocalizedEdit
+
backend = get_active_diffusion_engine()
if account_access.managed_account():
await asyncio.to_thread(account_access.require_media_adapters, request)
+ # An edit that names no size matches Image 1 in the backend instead of the schema's 1024 square default.
+ size_omitted = request.workflow == "edit" and not (
+ {"width", "height"} & request.model_fields_set
+ )
+ localized_edit = (
+ LocalizedEdit(mode = request.localized_edit.mode, image = request.localized_edit.image)
+ if request.localized_edit is not None
+ else None
+ )
result = None
for attempt in range(2):
expected_load = None
@@ -39299,8 +39310,8 @@ async def generate_diffusion_image(
expected_load = expected_load,
prompt = request.prompt,
negative_prompt = request.negative_prompt,
- width = request.width,
- height = request.height,
+ width = None if size_omitted else request.width,
+ height = None if size_omitted else request.height,
steps = request.steps,
guidance = request.guidance,
seed = request.seed,
@@ -39312,6 +39323,9 @@ async def generate_diffusion_image(
strength = request.strength,
upscale = request.upscale,
reference_images = request.reference_images,
+ workflow = request.workflow,
+ reference_resolution = request.reference_resolution,
+ localized_edit = localized_edit,
loras = [(l.id, l.weight) for l in request.loras] if request.loras else None,
controlnet = (
(
@@ -39414,6 +39428,9 @@ async def generate_diffusion_image(
else None
),
"reference_image_count": len(request.reference_images or []) or None,
+ # From the engine, not the request: an omitted resolution resolves to the family default.
+ "reference_resolution": result.get("reference_resolution"),
+ "localized_edit": result.get("localized_edit"),
"created_at": created_at,
},
)
diff --git a/studio/backend/routes/settings.py b/studio/backend/routes/settings.py
index 2973eb6b043..aba099e0886 100644
--- a/studio/backend/routes/settings.py
+++ b/studio/backend/routes/settings.py
@@ -218,8 +218,8 @@ class ImageGenerationPresetParams(BaseModel):
model_config = ConfigDict(extra = "forbid")
negativePrompt: str = ""
- width: int = Field(default = 1024, ge = 256, le = 2048, multiple_of = 16)
- height: int = Field(default = 1024, ge = 256, le = 2048, multiple_of = 16)
+ width: int = Field(default = 1024, ge = 256, le = 2752, multiple_of = 16)
+ height: int = Field(default = 1024, ge = 256, le = 2752, multiple_of = 16)
steps: int = Field(default = 9, ge = 1, le = 100)
guidance: float = Field(default = 0, ge = 0, le = 20)
batchSize: int = Field(default = 1, ge = 1, le = 32)
diff --git a/studio/backend/tests/test_diffusion_compat_preflight.py b/studio/backend/tests/test_diffusion_compat_preflight.py
index af9db4b1aeb..2aba2fca316 100644
--- a/studio/backend/tests/test_diffusion_compat_preflight.py
+++ b/studio/backend/tests/test_diffusion_compat_preflight.py
@@ -1724,4 +1724,6 @@ def test_qwen_image_2_1_takes_the_dynamic_4bit_text_encoder():
"Qwen3-VL-8B-Instruct-UD-Q4_K_XL.gguf",
"llm",
),
+ # The vision projector native editing needs, pinned by name for the same reason.
+ ("unsloth/Qwen3-VL-8B-Instruct-GGUF", "mmproj-F16.gguf", "llm_vision"),
)
diff --git a/studio/backend/tests/test_diffusion_engine_load_contract.py b/studio/backend/tests/test_diffusion_engine_load_contract.py
index 51c6bfd6483..96aaea20eb3 100644
--- a/studio/backend/tests/test_diffusion_engine_load_contract.py
+++ b/studio/backend/tests/test_diffusion_engine_load_contract.py
@@ -273,6 +273,38 @@ def test_an_uncached_asset_fails_with_a_local_error_naming_it(monkeypatch):
assert "unsloth/FLUX.1-dev" in message
+def test_an_uncached_vision_projector_does_not_fail_an_offline_load(monkeypatch, tmp_path):
+ """Text-to-image never reads --llm_vision, so a Qwen-Image-2.1 GGUF cached before the projector
+ was listed must still restore offline. Only the projector is optional: the encoder it sits
+ beside still fails loudly."""
+ from huggingface_hub.errors import LocalEntryNotFoundError
+
+ import utils.hf_xet_fallback as xet
+ from core.inference.sd_cpp_backend import SdCppDiffusionBackend as Native
+
+ cached = tmp_path / "Qwen3-VL-8B-Instruct-UD-Q4_K_XL.gguf"
+ cached.write_bytes(b"")
+
+ def _download(repo_id, filename, token, **kwargs):
+ if filename == "mmproj-F16.gguf":
+ raise LocalEntryNotFoundError("Cannot find the requested files in the disk cache")
+ return str(cached)
+
+ monkeypatch.setattr(xet, "hf_hub_download_with_xet_fallback", _download)
+ repo = "unsloth/Qwen3-VL-8B-Instruct-GGUF"
+ paths = Native(engine = None)._fetch_assets(
+ [(repo, cached.name, "llm"), (repo, "mmproj-F16.gguf", "llm_vision")],
+ None,
+ local_files_only = True,
+ )
+ assert paths == {"llm": str(cached)}
+
+ with pytest.raises(RuntimeError, match = "mmproj-F16.gguf"):
+ Native(engine = None)._fetch_assets(
+ [(repo, "mmproj-F16.gguf", "llm")], None, local_files_only = True
+ )
+
+
def test_the_default_still_takes_the_xet_fallback_ladder(monkeypatch, tmp_path):
"""Nothing changes with the flag off: the shared Xet -> HTTP path is still the one used, and
``local_files_only`` is not forwarded to a shared layer that may predate it."""
diff --git a/studio/backend/tests/test_diffusion_more_families.py b/studio/backend/tests/test_diffusion_more_families.py
index 8c3246d44e3..b562563b6f7 100644
--- a/studio/backend/tests/test_diffusion_more_families.py
+++ b/studio/backend/tests/test_diffusion_more_families.py
@@ -689,7 +689,9 @@ def test_qwen_image_21_gguf_reaches_sd_cpp_with_its_own_vae_and_a_qwen3vl_encode
assert "2.1" in fam.sd_cpp_vae[1]
encoders = sd_cpp_text_encoders_for(fam, "unsloth/Qwen-Image-2.1-GGUF", None)
- assert len(encoders) == 1
+ # The encoder, then the vision projector native editing reads through --llm_vision.
+ assert len(encoders) == 2
+ assert encoders[1] == ("unsloth/Qwen3-VL-8B-Instruct-GGUF", "mmproj-F16.gguf", "llm_vision")
repo, filename, kind = encoders[0]
assert repo == "unsloth/Qwen3-VL-8B-Instruct-GGUF"
# Which rung is the family's call, pinned by exact name in
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 1db6196d001..e2fc133be46 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -2656,3 +2656,54 @@ def test_the_plan_route_refuses_an_unrecognised_model_before_planning(client):
assert resp.status_code == 400, resp.text
assert "Could not infer a diffusion family" in resp.json()["detail"]
+
+
+def test_edit_without_a_size_lets_the_backend_match_image_1(client, monkeypatch):
+ """An edit that names no size must not be pinned to the schema's 1024 square: the route hands
+ the backend None so it sizes from Image 1 on the family grid. A named size passes through."""
+ _post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")
+ backend = diffusion_module.get_diffusion_backend()
+ seen = []
+ original = backend.generate
+
+ def _record(**kwargs):
+ seen.append(kwargs)
+ out = original(**kwargs)
+ out.update(workflow = "edit", reference_resolution = 512, localized_edit = "mask")
+ return out
+
+ monkeypatch.setattr(backend, "generate", _record)
+ layer = {"mode": "mask", "image": "QUJD"}
+ resp = _post_generate(
+ client,
+ prompt = "p",
+ workflow = "edit",
+ init_image = "QUJD",
+ reference_images = ["QUJD", "QUJD"],
+ reference_resolution = 512,
+ localized_edit = layer,
+ )
+ assert resp.status_code == 200
+ assert seen[-1]["width"] is None and seen[-1]["height"] is None
+ assert seen[-1]["workflow"] == "edit" and seen[-1]["reference_resolution"] == 512
+ assert seen[-1]["localized_edit"].mode == "mask"
+ record = resp.json()["images"][0]
+ # The engaged values are what the recipe keeps; the count stays images BEYOND the source.
+ assert record["reference_resolution"] == 512
+ assert record["localized_edit"] == "mask"
+ assert record["reference_image_count"] == 2
+ _post_generate(client, prompt = "p", workflow = "edit", init_image = "QUJD", width = 1024)
+ assert seen[-1]["width"] == 1024 and seen[-1]["height"] == 1024
+
+
+def test_generate_schema_bounds_for_unified_editing(client):
+ _post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")
+ nine = _post_generate(client, prompt = "p", init_image = "QUJD", reference_images = ["QUJD"] * 9)
+ assert nine.status_code == 200
+ ten = _post_generate(client, prompt = "p", init_image = "QUJD", reference_images = ["QUJD"] * 10)
+ assert ten.status_code == 422
+ assert _post_generate(client, prompt = "p", workflow = "inpaint").status_code == 422
+ assert _post_generate(client, prompt = "p", width = 2752, height = 1536).status_code == 200
+ assert _post_generate(client, prompt = "p", width = 2768).status_code == 422
+ bad_mode = _post_generate(client, prompt = "p", localized_edit = {"mode": "lasso", "image": "QUJD"})
+ assert bad_mode.status_code == 422
diff --git a/studio/backend/tests/test_diffusion_unified_edit.py b/studio/backend/tests/test_diffusion_unified_edit.py
new file mode 100644
index 00000000000..fcd53a6712b
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_unified_edit.py
@@ -0,0 +1,481 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Qwen-Image-2.1 unified editing: the reference / edit dispatch, image order and alpha, output
+geometry, count limits and the localized-edit layers, against the real generate() with a pipeline
+double whose signature matches QwenImage21Pipeline's."""
+
+from __future__ import annotations
+
+import base64
+import io
+import types
+
+import pytest
+from PIL import Image
+
+from core.inference import diffusion_conditioning as cond
+from core.inference.diffusion import DiffusionBackend, _compile_shape_dims
+from core.inference.diffusion_families import detect_family, default_generation_params
+from core.inference.diffusion_memory import estimate_image_runtime_mib
+
+from .test_diffusion_backend import ( # noqa: F401
+ _FakeImage,
+ _FakePipe,
+ _FakeTransformer,
+ _load_into,
+ fake_runtime,
+)
+
+
+class _Fake21Pipe(_FakePipe):
+ """QwenImage21Pipeline's call signature: ``image`` list, explicit size, ``output_resolution``."""
+
+ def __call__(
+ self,
+ *,
+ prompt = None,
+ image = None,
+ negative_prompt = None,
+ true_cfg_scale = 1.0,
+ height = None,
+ width = None,
+ num_inference_steps = 40,
+ num_images_per_prompt = 1,
+ generator = None,
+ callback_on_step_end = None,
+ output_resolution = 1024,
+ use_kv_cache = True,
+ ):
+ self.last_kwargs = {
+ "prompt": prompt,
+ "image": image,
+ "height": height,
+ "width": width,
+ "true_cfg_scale": true_cfg_scale,
+ "output_resolution": output_resolution,
+ "num_images_per_prompt": num_images_per_prompt,
+ }
+ return types.SimpleNamespace(images = [_FakeImage() for _ in range(num_images_per_prompt)])
+
+
+class _Fake21Pipeline:
+ @classmethod
+ def from_pretrained(cls, base, **kwargs):
+ return _Fake21Pipe()
+
+
+def _images(call) -> list:
+ """The ``image`` argument as a list: one image goes to the pipeline bare, as upstream accepts."""
+ image = call["image"]
+ return image if isinstance(image, list) else [image]
+
+
+def _png(
+ size = (64, 64),
+ color = (120, 30, 30, 255),
+ mode = "RGBA",
+ exif = None,
+) -> str:
+ buf = io.BytesIO()
+ img = Image.new(mode, size, color if mode != "RGB" else color[:3])
+ kwargs = {"exif": exif} if exif is not None else {}
+ img.save(buf, format = "PNG", **kwargs)
+ return base64.b64encode(buf.getvalue()).decode()
+
+
+def backend21(fake_runtime, tmp_path):
+ import diffusers
+
+ diffusers.QwenImage21Pipeline = _Fake21Pipeline
+ diffusers.QwenImage21Transformer2DModel = _FakeTransformer
+ (tmp_path / "model.gguf").write_bytes(b"x")
+ backend = DiffusionBackend()
+ _load_into(backend, tmp_path, family_override = "qwen-image-2.1")
+ return backend
+
+
+def _flux2(fake_runtime, tmp_path):
+ import diffusers
+
+ from .test_diffusion_backend import _FakeInpaintPipeline, _FakePipeline
+
+ diffusers.Flux2KleinPipeline = _FakePipeline
+ diffusers.Flux2KleinInpaintPipeline = _FakeInpaintPipeline
+ diffusers.Flux2Transformer2DModel = _FakeTransformer
+ (tmp_path / "model.gguf").write_bytes(b"x")
+ backend = DiffusionBackend()
+ _load_into(backend, tmp_path, family_override = "flux.2-klein")
+ return backend
+
+
+def test_status_advertises_create_reference_and_edit_with_its_limits(backend21):
+ status = backend21.status()
+ assert status["workflows"] == ["txt2img", "reference", "edit"]
+ c = status["conditioning"]
+ assert c["max_condition_images"] == 10 and c["alpha"] is True
+ assert c["dimension_multiple"] == 32
+ assert (c["max_output_side"], c["max_output_pixels"]) == (2752, 2400 * 1792)
+ assert c["reference_resolutions"] == [512, 1024, 2048]
+ assert c["unified_edit"] is True and c["localized_edit_modes"] == ["annotate", "paint", "mask"]
+
+
+def test_edit_only_and_reference_families_keep_their_contracts():
+ kontext = detect_family("black-forest-labs/FLUX.1-Kontext-dev")
+ assert kontext.edit is True and kontext.unified_edit is False
+ flux2 = detect_family("black-forest-labs/FLUX.2-klein-4B")
+ assert flux2.reference is True and flux2.unified_edit is False
+ assert flux2.max_condition_images == 4 and flux2.condition_image_mode == "RGB"
+ from core.inference.diffusion import _family_workflows
+
+ assert _family_workflows(kontext) == ["edit"]
+ assert "edit" not in _family_workflows(flux2)
+
+
+def test_sampling_defaults_resolve_for_every_21_artifact_name():
+ for name in (
+ "Qwen/Qwen-Image-2.1",
+ "unsloth/Qwen-Image-2.1-GGUF",
+ "unsloth/Qwen-Image-2.1-FP8",
+ ):
+ assert default_generation_params(name) == (40, 1.0), name
+ # The generic key still owns the other Qwen-Image checkpoints.
+ assert default_generation_params("Qwen/Qwen-Image-2512") == (20, 4.0)
+
+
+def test_explicit_edit_hands_ordered_rgba_images_and_explicit_geometry(backend21):
+ colors = [(10 * i, 0, 0, 40 + i) for i in range(1, 5)]
+ out = backend21.generate(
+ prompt = "put them together",
+ steps = 4,
+ seed = 1,
+ width = 1024,
+ height = 768,
+ workflow = "edit",
+ init_image = _png(color = colors[0]),
+ reference_images = [_png(color = c) for c in colors[1:]],
+ reference_resolution = 512,
+ )
+ call = backend21._state.pipe.last_kwargs
+ assert [im.mode for im in call["image"]] == ["RGBA"] * 4
+ # Order kept exactly, alpha included.
+ assert [im.getpixel((0, 0)) for im in call["image"]] == colors
+ assert (call["width"], call["height"]) == (1024, 768)
+ assert call["output_resolution"] == 512
+ assert out["workflow"] == "edit" and out["reference_resolution"] == 512
+
+
+def test_omitted_size_matches_image_1_not_the_last_reference(backend21):
+ backend21.generate(
+ prompt = "edit",
+ steps = 4,
+ seed = 1,
+ width = None,
+ height = None,
+ workflow = "edit",
+ init_image = _png(size = (300, 200)),
+ reference_images = [_png(size = (100, 400))],
+ )
+ call = backend21._state.pipe.last_kwargs
+ # 3:2 at the 1024 area on the 32 px grid, as upstream's calculate_dimensions rounds it.
+ assert (call["width"], call["height"]) == (1248, 832)
+ assert cond.match_source_size(detect_family("Qwen/Qwen-Image-2.1"), (300, 200), 1024) == (
+ 1248,
+ 832,
+ )
+
+
+def test_unified_edit_registers_the_requested_shape_not_the_source(backend21):
+ fam = backend21._state.family
+ src = Image.new("RGBA", (96, 64))
+ assert _compile_shape_dims("edit", src, 1024, 768, fam) == (1024, 768)
+ # Edit-only families still size from the source.
+ kontext = detect_family("black-forest-labs/FLUX.1-Kontext-dev")
+ assert _compile_shape_dims("edit", src, 1024, 768, kontext) == (96, 64)
+
+
[email protected]("extras,ok", [(0, True), (3, True), (9, True), (10, False)])
+def test_ten_images_in_total_and_overflow_is_refused(backend21, extras, ok):
+ kwargs = dict(
+ prompt = "p",
+ steps = 2,
+ workflow = "edit",
+ init_image = _png(),
+ reference_images = [_png() for _ in range(extras)],
+ width = 512,
+ height = 512,
+ )
+ if ok:
+ backend21.generate(**kwargs)
+ assert len(_images(backend21._state.pipe.last_kwargs)) == 1 + extras
+ else:
+ with pytest.raises(ValueError, match = "at most 10 input images"):
+ backend21.generate(**kwargs)
+
+
+def test_flux2_keeps_four_images_and_refuses_the_fifth(fake_runtime, tmp_path):
+ backend = _flux2(fake_runtime, tmp_path)
+ backend.generate(
+ prompt = "p",
+ steps = 2,
+ init_image = _png(mode = "RGB"),
+ reference_images = [_png(mode = "RGB") for _ in range(3)],
+ )
+ assert all(im.mode == "RGB" for im in backend._state.pipe.last_kwargs["image"])
+ with pytest.raises(ValueError, match = "at most 4 input images"):
+ backend.generate(
+ prompt = "p",
+ steps = 2,
+ init_image = _png(mode = "RGB"),
+ reference_images = [_png(mode = "RGB") for _ in range(4)],
+ )
+ with pytest.raises(ValueError, match = "Instruction editing is not supported"):
+ backend.generate(prompt = "p", steps = 2, workflow = "edit", init_image = _png())
+ with pytest.raises(ValueError, match = "reference_resolution is not supported"):
+ backend.generate(prompt = "p", steps = 2, init_image = _png(), reference_resolution = 1024)
+
+
+def test_output_bounds_follow_the_family(backend21, fake_runtime, tmp_path):
+ base = dict(prompt = "p", steps = 2)
+ with pytest.raises(ValueError, match = "multiples of 32"):
+ backend21.generate(width = 1040, height = 1024, **base)
+ backend21.generate(width = 2752, height = 1536, **base)
+ with pytest.raises(ValueError, match = "at most 4,300,800 pixels"):
+ backend21.generate(width = 2752, height = 2752, **base)
+ with pytest.raises(ValueError, match = "2752px per side"):
+ backend21.generate(width = 2784, height = 512, **base)
+ # Another family keeps 2048 even though the transport now carries 2752.
+ other = DiffusionBackend()
+ _load_into(other, tmp_path)
+ with pytest.raises(ValueError, match = "2048px per side"):
+ other.generate(width = 2752, height = 1536, **base)
+
+
+ "extra,match",
+ [
+ ({"strength": 0.5}, "strength is not supported"),
+ ({"mask_image": _png(mode = "L", color = (255,))}, "mask_image is not supported"),
+ ({"upscale": 2.0}, "upscale is not supported"),
+ (
+ {"controlnet": ("cn", _png(), "canny", 0.7, 0.0, 1.0)},
+ "controlnet is not supported",
+ ),
+ ],
+)
+def test_explicit_workflows_refuse_what_they_would_drop(backend21, extra, match):
+ with pytest.raises(ValueError, match = match):
+ backend21.generate(prompt = "p", steps = 2, workflow = "edit", init_image = _png(), **extra)
+
+
+def test_missing_source_and_misplaced_settings_are_refused(backend21):
+ with pytest.raises(ValueError, match = "requires a source image"):
+ backend21.generate(prompt = "p", steps = 2, workflow = "edit")
+ with pytest.raises(ValueError, match = "applies only to the reference and edit"):
+ backend21.generate(prompt = "p", steps = 2, reference_resolution = 1024)
+ with pytest.raises(ValueError, match = "must be one of 512, 1024, 2048"):
+ backend21.generate(
+ prompt = "p", steps = 2, workflow = "edit", init_image = _png(), reference_resolution = 768
+ )
+ with pytest.raises(ValueError, match = "localized_edit needs the edit workflow"):
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ workflow = "reference",
+ init_image = _png(),
+ localized_edit = cond.LocalizedEdit("paint", _png(mode = "L", color = (255,))),
+ )
+
+
+def test_omitted_workflow_keeps_the_reference_path(backend21):
+ out = backend21.generate(prompt = "p", steps = 2, width = 512, height = 512, init_image = _png())
+ assert out["workflow"] == "reference"
+ call = backend21._state.pipe.last_kwargs
+ assert (call["width"], call["height"]) == (512, 512)
+ assert call["output_resolution"] == 1024
+
+
+def test_annotate_composites_marks_and_keeps_alpha_elsewhere(backend21):
+ overlay = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
+ overlay.putpixel((5, 5), (255, 0, 0, 255))
+ buf = io.BytesIO()
+ overlay.save(buf, format = "PNG")
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(color = (0, 0, 200, 0)),
+ localized_edit = cond.LocalizedEdit("annotate", base64.b64encode(buf.getvalue()).decode()),
+ )
+ marked = _images(backend21._state.pipe.last_kwargs)
+ assert len(marked) == 1 and marked[0].mode == "RGBA"
+ assert marked[0].getpixel((5, 5)) == (255, 0, 0, 255)
+ assert marked[0].getpixel((30, 30)) == (0, 0, 200, 0)
+
+
+def test_paint_whitens_the_region_at_the_source_geometry(backend21):
+ mask = Image.new("L", (32, 32), 0)
+ for x in range(16):
+ for y in range(16):
+ mask.putpixel((x, y), 255)
+ buf = io.BytesIO()
+ mask.save(buf, format = "PNG")
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(color = (10, 20, 30, 255)),
+ localized_edit = cond.LocalizedEdit("paint", base64.b64encode(buf.getvalue()).decode()),
+ )
+ painted = _images(backend21._state.pipe.last_kwargs)[0]
+ assert painted.size == (64, 64) # the half-size layer was scaled to the source
+ assert painted.getpixel((4, 4)) == (255, 255, 255, 255)
+ assert painted.getpixel((60, 60)) == (10, 20, 30, 255)
+
+
+def test_mask_is_image_2_binary_white_region_and_counts_toward_the_limit(backend21):
+ mask = Image.new("L", (64, 64), 0)
+ mask.putpixel((1, 1), 200)
+ buf = io.BytesIO()
+ mask.save(buf, format = "PNG")
+ layer = base64.b64encode(buf.getvalue()).decode()
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(color = (9, 9, 9, 255)),
+ reference_images = [_png(color = (1, 2, 3, 255))],
+ localized_edit = cond.LocalizedEdit("mask", layer),
+ )
+ images = backend21._state.pipe.last_kwargs["image"]
+ assert [im.getpixel((0, 0))[:3] for im in images] == [(9, 9, 9), (0, 0, 0), (1, 2, 3)]
+ assert images[1].getpixel((1, 1))[:3] == (255, 255, 255)
+ with pytest.raises(ValueError, match = "counting the mask"):
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(),
+ reference_images = [_png() for _ in range(9)],
+ localized_edit = cond.LocalizedEdit("mask", layer),
+ )
+ empty = io.BytesIO()
+ Image.new("L", (64, 64), 0).save(empty, format = "PNG")
+ with pytest.raises(ValueError, match = "mask is empty"):
+ backend21.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(),
+ localized_edit = cond.LocalizedEdit("mask", base64.b64encode(empty.getvalue()).decode()),
+ )
+
+
+def test_layer_for_another_image_is_refused():
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ with pytest.raises(ValueError, match = "same image"):
+ cond.decode_condition_images(
+ fam,
+ _png(size = (64, 64)),
+ None,
+ cond.LocalizedEdit("paint", _png(size = (64, 32), mode = "L", color = (255,))),
+ )
+
+
+def test_transparent_palette_png_and_exif_rotation_decode_for_the_family():
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ pal = Image.new("P", (40, 20))
+ pal.putpalette([255, 0, 0] + [0, 0, 0] * 255)
+ pal.info["transparency"] = 0
+ buf = io.BytesIO()
+ pal.save(buf, format = "PNG", transparency = 0)
+ (img,) = cond.decode_condition_images(fam, base64.b64encode(buf.getvalue()).decode(), None)
+ assert img.mode == "RGBA" and img.getpixel((0, 0))[3] == 0
+
+ exif = Image.Exif()
+ exif[0x0112] = 6 # rotate 90 CW on display
+ jpg = io.BytesIO()
+ Image.new("RGB", (40, 20)).save(jpg, format = "JPEG", exif = exif.tobytes())
+ (rot,) = cond.decode_condition_images(fam, base64.b64encode(jpg.getvalue()).decode(), None)
+ assert rot.size == (20, 40)
+
+
+def test_aggregate_source_pixels_are_bounded_before_decoding(monkeypatch):
+ fam = detect_family("Qwen/Qwen-Image-2.1")
+ monkeypatch.setattr(cond, "MAX_CONDITION_SOURCE_PIXELS", 64 * 64 * 2)
+ with pytest.raises(ValueError, match = "too large together"):
+ cond.decode_condition_images(fam, _png(), [_png(), _png()])
+
+
+def test_condition_images_count_toward_working_memory():
+ base = estimate_image_runtime_mib(width = 1024, height = 1024)
+ weight = detect_family("Qwen/Qwen-Image-2.1").condition_pixel_weight
+ four = estimate_image_runtime_mib(
+ width = 1024, height = 1024, condition_pixels = int(4 * 1024 * 1024 * weight)
+ )
+ # Measured: about 2.5 GiB per 1024-square reference on top of the output's own.
+ assert 4 * 2400 <= four - base <= 4 * 2900
+
+
+def test_offloaded_vision_tower_builds_position_embeddings_on_the_compute_device():
+ """Under leaf-level group offload Qwen3-VL read ``pos_embed.weight.device`` while the weight was
+ still on the CPU, so every image-conditioned call on a streamed encoder died with a device
+ mismatch. The pin moves the result to ``grid_thw``'s device, per instance."""
+ from core.inference.diffusion_memory import _pin_vision_embedding_device
+
+ class _Out:
+ def __init__(self, device):
+ self.device = device
+
+ def to(self, device):
+ return _Out(device)
+
+ class _Vision:
+ def fast_pos_embed_interpolate(self, grid_thw):
+ return _Out("cpu")
+
+ def modules(self):
+ return iter([self])
+
+ vision = _Vision()
+ assert _pin_vision_embedding_device(vision) == 1
+ assert (
+ vision.fast_pos_embed_interpolate(types.SimpleNamespace(device = "cuda:0")).device == "cuda:0"
+ )
+ # Idempotent, and a module without the method or without modules() is left alone.
+ assert _pin_vision_embedding_device(vision) == 0
+ assert _pin_vision_embedding_device(object()) == 0
+ assert _Vision().fast_pos_embed_interpolate(None).device == "cpu"
+
+
+ "source,resolution",
+ [((4096, 256), 512), ((256, 4096), 512), ((20000, 10), 1024), ((4096, 256), 2048)],
+)
+def test_an_elongated_source_matches_to_a_size_the_request_accepts(source, resolution):
+ """4096x256 at the 512 tier rounded to 2048x128, which the API refuses (both sides >= 256).
+ The short side now stays at the minimum and the long side is capped at the family's bound."""
+ for name in ("Qwen/Qwen-Image-2.1", "Tongyi-MAI/Z-Image-Turbo"):
+ fam = detect_family(name)
+ w, h = cond.match_source_size(fam, source, resolution)
+ cond.check_output_size(fam, w, h)
+ assert cond.match_source_size(detect_family("Qwen/Qwen-Image-2.1"), (4096, 256), 512) == (
+ 2752,
+ 256,
+ )
+
+
+def test_an_output_side_below_the_minimum_is_refused():
+ with pytest.raises(ValueError, match = "at least 256px"):
+ cond.check_output_size(detect_family("Qwen/Qwen-Image-2.1"), 2048, 128)
diff --git a/studio/backend/tests/test_sd_cpp_backend.py b/studio/backend/tests/test_sd_cpp_backend.py
index b84d3dcdc1d..fdd29e5c6ba 100644
--- a/studio/backend/tests/test_sd_cpp_backend.py
+++ b/studio/backend/tests/test_sd_cpp_backend.py
@@ -764,7 +764,7 @@ def test_status_loaded_shape():
def test_generate_returns_images_and_seed():
eng = _FakeEngine()
b = _loaded_backend(engine = eng)
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 8, seed = 123, batch_size = 2)
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 8, seed = 123, batch_size = 2)
assert out["seed"] == 123
assert out["repo_id"] == "unsloth/Z-Image-Turbo-GGUF"
assert len(out["images"]) == 2
@@ -938,7 +938,7 @@ def test_generate_publishes_progress_before_lora_resolution(monkeypatch):
monkeypatch.setattr(diffusion_lora, "resolve_specs", _resolve)
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 8, loras = [("some/lora", 1.0)])
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 8, loras = [("some/lora", 1.0)])
assert out["images"]
assert seen["progress"]["active"] is True
assert seen["progress"]["total_steps"] == 8
@@ -1475,7 +1475,7 @@ def test_server_generate_uses_one_request_for_whole_batch(monkeypatch):
b = SdCppDiffusionBackend()
servers: list = []
_run_server_load(monkeypatch, b, servers)
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 8, seed = 7, batch_size = 3)
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 8, seed = 7, batch_size = 3)
assert len(out["images"]) == 3
assert all(isinstance(im, Image.Image) for im in out["images"])
# ONE job for the whole batch (no per-image model reload), unlike the one-shot path.
@@ -1502,7 +1502,7 @@ def test_server_generation_restarts_on_the_cpu_backend_after_a_ggml_abort(monkey
_run_server_load(monkeypatch, b, servers, device = "mps")
servers[0].img_gen_error = RuntimeError(_GGML_ABORT)
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 4, seed = 3)
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 4, seed = 3)
assert len(out["images"]) == 1 # the retry produced the image
assert len(servers) == 2 and servers[0].stopped is True
@@ -1558,7 +1558,7 @@ def test_server_generate_splits_batches_above_server_limit(monkeypatch):
b = SdCppDiffusionBackend()
servers: list = []
_run_server_load(monkeypatch, b, servers)
- out = b.generate(prompt = "x", width = 64, height = 64, steps = 4, seed = 100, batch_size = 10)
+ out = b.generate(prompt = "x", width = 256, height = 256, steps = 4, seed = 100, batch_size = 10)
assert len(out["images"]) == 10
counts = [p["batch_count"] for p in servers[0].payloads]
assert counts == [bk._MAX_SERVER_BATCH, 10 - bk._MAX_SERVER_BATCH] # [8, 2]
@@ -1578,7 +1578,7 @@ def test_server_generate_masks_large_seed(monkeypatch):
b = SdCppDiffusionBackend()
servers: list = []
_run_server_load(monkeypatch, b, servers)
- out = b.generate(prompt = "x", width = 64, height = 64, steps = 4, seed = 2**64 - 1, batch_size = 1)
+ out = b.generate(prompt = "x", width = 256, height = 256, steps = 4, seed = 2**64 - 1, batch_size = 1)
assert servers[0].payloads[0]["seed"] <= (1 << 63) - 1
assert all(s <= (1 << 63) - 1 for s in out["seeds"])
@@ -2210,7 +2210,7 @@ def test_generate_reports_the_build_the_recipe_persists():
gguf_filename = "z-image-turbo-Q4_K_M.gguf",
offload_flags = ("--vae-on-cpu", "--clip-on-cpu"),
)
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 4, seed = 1)
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 4, seed = 1)
assert out["model_kind"] == "gguf"
assert out["gguf_filename"] == "z-image-turbo-Q4_K_M.gguf"
assert out["offload_policy"] == "active"
@@ -2237,11 +2237,11 @@ def test_a_completed_native_generation_stops_advertising_itself_as_cancellable(m
monkeypatch.setattr(b, "_generate_oneshot", _oneshot)
with pytest.raises(RuntimeError, match = "cancelled"):
- b.generate(prompt = "a fox", width = 64, height = 64, steps = 4, seed = 1)
+ b.generate(prompt = "a fox", width = 256, height = 256, steps = 4, seed = 1)
# And once a run completes, the event is gone before the result is handed back.
b2 = _loaded_backend()
- out = b2.generate(prompt = "a fox", width = 64, height = 64, steps = 4, seed = 1)
+ out = b2.generate(prompt = "a fox", width = 256, height = 256, steps = 4, seed = 1)
assert out["images"]
seen.append(b2.cancel_generate())
assert seen == [False]
@@ -2286,7 +2286,7 @@ def test_a_card_pick_is_not_reported_as_an_offload():
status = b.status()
assert status["cpu_offload"] is False
assert status["offload_policy"] == "none"
- out = b.generate(prompt = "a fox", width = 64, height = 64, steps = 4, seed = 1)
+ out = b.generate(prompt = "a fox", width = 256, height = 256, steps = 4, seed = 1)
assert out["offload_policy"] == "none"
@@ -2411,7 +2411,7 @@ def test_generation_in_flight_tracks_a_generation(monkeypatch):
monkeypatch.setattr(diffusion_lora, "resolve_specs", _resolve)
assert bk.generation_in_flight() is False
- b.generate(prompt = "a fox", width = 64, height = 64, steps = 8, loras = [("some/lora", 1.0)])
+ b.generate(prompt = "a fox", width = 256, height = 256, steps = 8, loras = [("some/lora", 1.0)])
assert (
seen["in_flight"] is True
), "liveness cannot tell this backend from a dead one while the native engine renders"
diff --git a/studio/backend/tests/test_sd_cpp_qwen21_edit.py b/studio/backend/tests/test_sd_cpp_qwen21_edit.py
new file mode 100644
index 00000000000..f0218b1ef1d
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_qwen21_edit.py
@@ -0,0 +1,326 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Native (sd.cpp) Qwen-Image-2.1 editing: the vision projector asset, the --llm_vision and
+reference-image transport on both the one-shot and the server path, the readiness gate, and the
+pinned build's alpha and crop workarounds."""
+
+from __future__ import annotations
+
+import base64
+import dataclasses
+import io
+import types
+
+import pytest
+from PIL import Image
+
+from core.inference import sd_cpp_backend as bk
+from core.inference.diffusion_conditioning import LocalizedEdit
+from core.inference.diffusion_families import detect_family, sd_cpp_text_encoders_for
+from core.inference.sd_cpp_args import (
+ SdCppGenParams,
+ SdCppModelFiles,
+ build_img_gen_request,
+ build_sd_cpp_command,
+ build_sd_cpp_server_command,
+)
+from core.inference.sd_cpp_backend import SdCppDiffusionBackend
+
+from .test_sd_cpp_backend import _FakeEngine, _FakeServer
+
+FAM = detect_family("unsloth/Qwen-Image-2.1-GGUF")
+FILES = SdCppModelFiles(
+ diffusion_model = "/m/q.gguf",
+ vae = "/m/vae.safetensors",
+ llm = "/m/llm.gguf",
+ llm_vision = "/m/mmproj.gguf",
+)
+
+
+def _png(size = (64, 32), color = (200, 10, 10, 255)) -> str:
+ buf = io.BytesIO()
+ Image.new("RGBA", size, color).save(buf, format = "PNG")
+ return base64.b64encode(buf.getvalue()).decode()
+
+
+def _decode(blob: bytes) -> Image.Image:
+ return Image.open(io.BytesIO(blob))
+
+
+def test_the_projector_is_a_planned_companion_asset():
+ specs = SdCppDiffusionBackend(engine = _FakeEngine())._asset_specs(
+ "unsloth/Qwen-Image-2.1-GGUF", "qwen-image-2.1-Q4_K_M.gguf", FAM
+ )
+ assert ("unsloth/Qwen3-VL-8B-Instruct-GGUF", "mmproj-F16.gguf", "llm_vision") in specs
+ assert any(k == "llm_vision" for _r, _f, k in sd_cpp_text_encoders_for(FAM))
+
+
+def test_both_command_builders_pass_the_projector_and_the_cli_keeps_ref_order():
+ cli = build_sd_cpp_command(
+ "sd-cli",
+ FILES,
+ SdCppGenParams(prompt = "p", width = 512, height = 512, ref_images = ("/a.png", "/b.png")),
+ output_path = "/o.png",
+ )
+ assert cli[cli.index("--llm_vision") + 1] == "/m/mmproj.gguf"
+ refs = [cli[i + 1] for i, a in enumerate(cli) if a == "--ref-image"]
+ assert refs == ["/a.png", "/b.png"]
+ assert "--init-img" not in cli and "--mask" not in cli
+ server = build_sd_cpp_server_command("sd-server", FILES, host = "127.0.0.1", port = 1)
+ assert server[server.index("--llm_vision") + 1] == "/m/mmproj.gguf"
+
+
+def test_the_server_request_carries_refs_and_nothing_img2img():
+ req = build_img_gen_request(prompt = "p", ref_images = ["data:image/png;base64,AA", "B"])
+ assert req["ref_images"] == ["data:image/png;base64,AA", "B"]
+ assert "init_image" not in req and "strength" not in req and "mask_image" not in req
+ assert "ref_images" not in build_img_gen_request(prompt = "p")
+
+
+def test_condition_images_flatten_alpha_over_white_and_pad_only_for_the_server():
+ clear = _png(size = (64, 32), color = (0, 0, 255, 0))
+ w, h, blobs = bk._native_condition_images(
+ FAM, _png(size = (64, 32)), [clear], None, None, None, full_fidelity = False, pad_to_output = True
+ )
+ # Image 1's 2:1 aspect ratio at the 1024 area, the same size the diffusers engine picks.
+ assert (w, h) == (1440, 736)
+ ref = _decode(blobs[1])
+ assert ref.mode == "RGB" and ref.getpixel((32, 16)) == (255, 255, 255)
+ # A reference with another aspect ratio is padded to the output's instead of being cropped.
+ _w, _h, padded = bk._native_condition_images(
+ FAM,
+ _png(size = (64, 32)),
+ [_png(size = (32, 32))],
+ None,
+ 1024,
+ 512,
+ full_fidelity = False,
+ pad_to_output = True,
+ )
+ assert _decode(padded[1]).size == (64, 32)
+ _w, _h, plain = bk._native_condition_images(
+ FAM,
+ _png(size = (64, 32)),
+ [_png(size = (32, 32))],
+ None,
+ 1024,
+ 512,
+ full_fidelity = False,
+ pad_to_output = False,
+ )
+ assert _decode(plain[1]).size == (32, 32)
+
+
+def test_native_outputs_keep_alpha_for_the_family_only():
+ rgba = Image.new("RGBA", (2, 2), (1, 2, 3, 100))
+ assert bk._native_output_image(FAM, rgba).mode == "RGBA"
+ assert bk._native_output_image(detect_family("z-image"), rgba).mode == "RGB"
+
+
+def _state(
+ files = FILES,
+ mode = "oneshot",
+ server = None,
+):
+ return bk._SdState(
+ repo_id = "unsloth/Qwen-Image-2.1-GGUF",
+ base_repo = FAM.base_repo,
+ family = FAM,
+ device = "cpu",
+ files = files,
+ sampling_method = FAM.sd_cpp_sampling_method,
+ mode = mode,
+ server = server,
+ )
+
+
+def test_edit_is_advertised_only_with_the_projector_and_an_edit_capable_build(tmp_path):
+ capable = tmp_path / "sd-new"
+ capable.write_bytes(b"xx" + FAM.sd_cpp_edit_marker.encode() + b"yy qwen_image_2_1")
+ older = tmp_path / "sd-old"
+ older.write_bytes(b"qwen_image_2_1 only")
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ for binary, files, expect in (
+ (capable, FILES, True),
+ (older, FILES, False),
+ (capable, SdCppModelFiles(diffusion_model = "/m/q.gguf", llm = "/m/llm.gguf"), False),
+ ):
+ b._state = _state(
+ files,
+ mode = "server",
+ server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True),
+ )
+ status = b.status()
+ assert ("edit" in status["workflows"]) is expect, binary
+ assert ("reference" in status["workflows"]) is expect
+ if expect:
+ c = status["conditioning"]
+ assert c["reference_resolutions"] == [] and c["alpha"] is False
+ assert any("padded to its aspect ratio" in n for n in c["notes"])
+ assert any("raise Guidance above 1" in n for n in c["notes"])
+
+
+def test_an_unreadable_build_advertises_no_edit_and_no_fidelity(tmp_path):
+ # The arch gate lets an unreadable binary through; these optional capabilities must not.
+ binary = tmp_path / "sd-server"
+ binary.write_bytes(FAM.sd_cpp_edit_marker.encode() + b" " + bk._REFERENCE_FIDELITY_MARKER.encode())
+ binary.chmod(0o111)
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state(
+ mode = "server",
+ server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True),
+ )
+ try:
+ assert "edit" not in b.status()["workflows"]
+ assert b._native_reference_fidelity(b._state) is False
+ assert bk.binary_carries_marker(str(binary), "anything") is True
+ finally:
+ binary.chmod(0o644)
+
+
+def test_oneshot_edit_stages_ordered_pngs_and_records_the_workflow(monkeypatch):
+ engine = _FakeEngine()
+ b = SdCppDiffusionBackend(engine = engine)
+ b._state = _state()
+ monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: True)
+ seen = {}
+
+ def _capture(files, params, output_path, **kw):
+ seen["refs"] = [Image.open(p).getpixel((0, 0)) for p in params.ref_images]
+ seen["size"] = (params.width, params.height)
+ Image.new("RGBA", (1, 1), (1, 2, 3, 4)).save(output_path)
+
+ monkeypatch.setattr(engine, "generate", _capture)
+ out = b.generate(
+ prompt = "p",
+ steps = 2,
+ workflow = "edit",
+ width = None,
+ height = None,
+ init_image = _png(color = (10, 0, 0, 255)),
+ reference_images = [_png(color = (20, 0, 0, 255)), _png(color = (30, 0, 0, 255))],
+ localized_edit = None,
+ )
+ assert [c[:3] for c in seen["refs"]] == [(10, 0, 0), (20, 0, 0), (30, 0, 0)]
+ assert seen["size"][0] / seen["size"][1] == pytest.approx(2.0, rel = 0.05)
+ assert out["workflow"] == "edit" and out["images"][0].mode == "RGBA"
+
+
+def test_server_edit_sends_the_mask_as_image_2(monkeypatch):
+ server = _FakeServer("sd-server")
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state(mode = "server", server = server)
+ monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: True)
+ mask = io.BytesIO()
+ Image.new("L", (64, 32), 255).save(mask, format = "PNG")
+ b.generate(
+ prompt = "p",
+ steps = 2,
+ workflow = "edit",
+ width = 512,
+ height = 256,
+ init_image = _png(),
+ localized_edit = LocalizedEdit("mask", base64.b64encode(mask.getvalue()).decode()),
+ )
+ refs = server.payloads[-1]["ref_images"]
+ assert len(refs) == 2
+ second = _decode(base64.b64decode(refs[1].split(",", 1)[1]))
+ assert second.getpixel((0, 0)) == (255, 255, 255)
+
+
+def test_native_refuses_what_it_cannot_honour(monkeypatch):
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state()
+ with pytest.raises(ValueError, match = "Reference detail is not adjustable"):
+ b.generate(prompt = "p", workflow = "edit", init_image = _png(), reference_resolution = 1024)
+ with pytest.raises(ValueError, match = "not yet supported"):
+ b.generate(prompt = "p", init_image = _png()) # an omitted workflow keeps its refusal
+ # Same refusal as the diffusers engine: a localized edit belongs to the edit workflow only.
+ with pytest.raises(ValueError, match = "localized_edit needs the edit workflow"):
+ b.generate(
+ prompt = "p",
+ workflow = "reference",
+ init_image = _png(),
+ width = 512,
+ height = 256,
+ localized_edit = LocalizedEdit("paint", _png()),
+ )
+ # Without an edit-capable build and projector, the workflow is refused rather than sent.
+ monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: False)
+ with pytest.raises(ValueError, match = "Image editing is not available"):
+ b.generate(prompt = "p", workflow = "edit", init_image = _png(), width = 512, height = 512)
+
+
+ "family, width, height, match",
+ [
+ ("z-image", 2560, 1024, "at most 2048px"),
+ ("qwen-image-2.1", 1040, 1024, "multiples of 32"),
+ ("qwen-image-2.1", 2752, 2752, "pixels"),
+ ],
+)
+def test_native_text_to_image_keeps_the_family_size_bounds(family, width, height, match):
+ # The request schema admits 2752 for Qwen-Image-2.1's 2K presets; every other bound is the family's.
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = dataclasses.replace(_state(), family = detect_family(family))
+ with pytest.raises(ValueError, match = match):
+ b.generate(prompt = "p", steps = 2, width = width, height = height)
+
+
+def test_a_padded_mask_adds_no_region_and_stays_aligned_with_the_source():
+ """Padding the separate mask with white marked the added border as part of the region to
+ edit. It is padded black, by the same amount as the source, so the region stays where it was
+ drawn."""
+ mask = Image.new("L", (64, 32), 0)
+ for x in range(8):
+ for y in range(8):
+ mask.putpixel((x, y), 255)
+ buf = io.BytesIO()
+ mask.save(buf, format = "PNG")
+ _w, _h, blobs = bk._native_condition_images(
+ FAM,
+ _png(size = (64, 32), color = (9, 9, 9, 255)),
+ None,
+ LocalizedEdit("mask", base64.b64encode(buf.getvalue()).decode()),
+ 512,
+ 512,
+ full_fidelity = False,
+ pad_to_output = True,
+ )
+ source, padded = _decode(blobs[0]), _decode(blobs[1]).convert("L")
+ assert source.size == padded.size == (64, 64)
+ # The border the padding added is not part of the region...
+ assert padded.getpixel((32, 2)) == 0 and padded.getpixel((32, 60)) == 0
+ # ...the drawn region moved with the source, and the source border is white as before.
+ assert padded.getpixel((2, 18)) == 255 and padded.getpixel((2, 26)) == 0
+ assert source.getpixel((32, 2))[:3] == (255, 255, 255)
+ assert source.getpixel((2, 18))[:3] == (9, 9, 9)
+
+
+def test_a_full_fidelity_build_gets_the_images_as_decoded(tmp_path):
+ """A build carrying the upstream reference fixes reads alpha and keeps each image's shape, so
+ neither workaround applies and status reports alpha."""
+ clear = _png(size = (32, 32), color = (0, 0, 255, 0))
+ _w, _h, blobs = bk._native_condition_images(
+ FAM, _png(size = (64, 32)), [clear], None, 1024, 512, full_fidelity = True, pad_to_output = True
+ )
+ ref = _decode(blobs[1])
+ assert ref.mode == "RGBA" and ref.size == (32, 32) and ref.getpixel((4, 4)) == (0, 0, 255, 0)
+
+ binary = tmp_path / "sd-server"
+ binary.write_bytes(
+ b"qwen_image_2_1 "
+ + FAM.sd_cpp_edit_marker.encode()
+ + b" "
+ + bk._REFERENCE_FIDELITY_MARKER.encode()
+ )
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state(
+ mode = "server", server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True)
+ )
+ c = b.status()["conditioning"]
+ assert c["alpha"] is True
+ assert c["notes"] == [
+ "For transparent output on the native engine, raise Guidance above 1 (upstream uses 6)."
+ ]
diff --git a/studio/frontend/src/components/image-dropzone.tsx b/studio/frontend/src/components/image-dropzone.tsx
index 7efaa9c38f0..2bcb62811c5 100644
--- a/studio/frontend/src/components/image-dropzone.tsx
+++ b/studio/frontend/src/components/image-dropzone.tsx
@@ -120,7 +120,17 @@ export function ImageDropzone({
data-reload-snapshot-sensitive
className={cn("relative overflow-hidden rounded-[10px] border border-border", className)}
>
- <img src={value} alt="Source" className="max-h-44 w-full object-contain bg-muted/30" />
+ {/* Checkerboard behind the preview, so a transparent upload shows its alpha. */}
+ <img
+ src={value}
+ alt="Source"
+ className="max-h-44 w-full object-contain bg-muted/30"
+ style={{
+ backgroundImage:
+ "repeating-conic-gradient(rgb(128 128 128 / 0.28) 0% 25%, transparent 0% 50%)",
+ backgroundSize: "16px 16px",
+ }}
+ />
<Tooltip>
<TooltipTrigger asChild={true}>
<Button
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 87abe421366..88de7ba3bf3 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -53,6 +53,8 @@ export interface DiffusionStatus {
// Image workflows the loaded family supports (drives tab gating). Absent when nothing is loaded or
// on the native engine.
workflows?: string[];
+ // Absent on an older backend: callers keep the historical limits (4 images, RGB, 16 px, 2048).
+ conditioning?: DiffusionConditioning | null;
// Whether the loaded model + quantisation can apply LoRA adapters (drives the LoRA picker enabled state).
supports_lora?: boolean;
// Whether the loaded model can apply a ControlNet. Diffusers only, for families with a ControlNet pipeline.
@@ -62,6 +64,22 @@ export interface DiffusionStatus {
resolved?: Record<string, DiffusionResolvedControl> | null;
}
+export interface DiffusionConditioning {
+ // Total input images per call, INCLUDING the source.
+ max_condition_images: number;
+ alpha: boolean;
+ dimension_multiple: number;
+ max_output_side: number;
+ max_output_pixels: number;
+ reference_resolutions: number[];
+ unified_edit: boolean;
+ localized_edit_modes: LocalizedEditMode[];
+ // Engine-specific caveats shown next to the inputs.
+ notes?: string[];
+}
+
+export type LocalizedEditMode = "annotate" | "paint" | "mask";
+
export interface DiffusionGenerateProgress {
active: boolean;
step: number;
@@ -141,8 +159,12 @@ export interface DiffusionGenerateRequest {
strength?: number;
// Upscale (hires fix): factor > 1 with an init_image enlarges the source and re-denoises at low strength.
upscale?: number;
- // Additional reference images for the FLUX.2 reference workflow, combined with init_image.
+ // Additional images after init_image, in order, for the reference and edit workflows.
reference_images?: string[];
+ workflow?: "edit" | "reference";
+ reference_resolution?: number;
+ // Unified edit only: annotate/paint composite onto the source, mask is sent as Image 2.
+ localized_edit?: { mode: LocalizedEditMode; image: string };
// LoRA adapters for this generation (discovery id + weight, 0..2). Rejected with a 400 when the
// loaded model cannot apply LoRA.
loras?: LoraSpecInput[];
@@ -223,7 +245,10 @@ export interface GalleryImage {
strength?: number | null;
upscale?: number | null;
controlnet_guidance?: string | null;
+ // Images beyond the source (reference_images), for the reference and edit workflows.
reference_image_count?: number | null;
+ reference_resolution?: number | null;
+ localized_edit?: LocalizedEditMode | null;
created_at: number;
// Library state, not recipe: stored beside the PNG, absent on records written before this existed.
pinned?: boolean;
diff --git a/studio/frontend/src/features/images/edit-conditioning.ts b/studio/frontend/src/features/images/edit-conditioning.ts
new file mode 100644
index 00000000000..8d153d22a23
--- /dev/null
+++ b/studio/frontend/src/features/images/edit-conditioning.ts
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type { CSSProperties } from "react";
+
+import type {
+ DiffusionConditioning,
+ DiffusionGenerateRequest,
+ LocalizedEditMode,
+} from "./api";
+import { type SizeLimits, fitSize, matchSourceSize } from "./image-size.ts";
+
+/** Colours offered for annotations, named the way an instruction refers to them. */
+export const ANNOTATION_COLORS: ReadonlyArray<{ name: string; value: string }> =
+ [
+ { name: "red", value: "#ff2020" },
+ { name: "blue", value: "#1f6bff" },
+ { name: "green", value: "#1fc23a" },
+ { name: "yellow", value: "#ffd400" },
+ ];
+
+/** Checkerboard behind images that may be transparent, so alpha reads as alpha. */
+export const TRANSPARENCY_CHECKER: CSSProperties = {
+ backgroundImage:
+ "repeating-conic-gradient(rgb(128 128 128 / 0.28) 0% 25%, transparent 0% 50%)",
+ backgroundSize: "16px 16px",
+};
+
+/** Total inputs when the backend does not say: the FLUX.2 reference limit it has always had. */
+export const DEFAULT_MAX_CONDITION_IMAGES = 4;
+
+/** Images the page may add after the source; a separate mask takes one slot. */
+export function maxAdditionalImages(
+ conditioning: DiffusionConditioning | null | undefined,
+ localizedMode: LocalizedEditMode | null,
+): number {
+ const total =
+ conditioning?.max_condition_images ?? DEFAULT_MAX_CONDITION_IMAGES;
+ return Math.max(0, total - 1 - (localizedMode === "mask" ? 1 : 0));
+}
+
+/** Image number of additional slot ``index`` (after the source and any separate mask). */
+export function additionalImageNumber(
+ index: number,
+ localizedMode: LocalizedEditMode | null,
+): number {
+ return index + (localizedMode === "mask" ? 3 : 2);
+}
+
+export const REFERENCE_DETAIL_LABELS: Record<number, string> = {
+ 512: "Low (512)",
+ 1024: "Standard (1024)",
+ 2048: "High (2048)",
+};
+
+/** The build's canvas tier when listed, else 1024; never an automatic 2048. */
+export function seedReferenceResolution(
+ allowed: readonly number[],
+ tier: number,
+): number | null {
+ if (!allowed.length) return null;
+ if (allowed.includes(tier)) return tier;
+ if (allowed.includes(1024)) return 1024;
+ return allowed[0];
+}
+
+/** The official Qwen-Image-2.1 2K sizes, offered when the loaded model allows them. */
+export const OFFICIAL_2K_PRESETS: ReadonlyArray<{
+ label: string;
+ width: number;
+ height: number;
+}> = [
+ { label: "1:1", width: 2048, height: 2048 },
+ { label: "4:3", width: 2400, height: 1792 },
+ { label: "3:4", width: 1792, height: 2400 },
+ { label: "3:2", width: 2528, height: 1696 },
+ { label: "2:3", width: 1696, height: 2528 },
+ { label: "16:9", width: 2752, height: 1536 },
+ { label: "9:16", width: 1536, height: 2752 },
+];
+
+export function presetsWithin(limits: SizeLimits) {
+ return OFFICIAL_2K_PRESETS.filter(
+ (p) =>
+ Math.max(p.width, p.height) <= limits.maxSide &&
+ p.width * p.height <= limits.maxPixels &&
+ p.width % limits.multiple === 0 &&
+ p.height % limits.multiple === 0,
+ );
+}
+
+// The model card's prompt format for transparent output.
+export const TRANSPARENCY_PREFIX = "This is an RGBA image with transparency.";
+export const TRANSPARENCY_SUFFIX =
+ "The image has alpha channel and the background is transparent.";
+
+export function withTransparencyPrompt(prompt: string): string {
+ let body = prompt.trim();
+ if (!body.startsWith(TRANSPARENCY_PREFIX)) {
+ body = `${TRANSPARENCY_PREFIX} ${body}`.trim();
+ }
+ if (!body.endsWith(TRANSPARENCY_SUFFIX)) {
+ body = `${body} ${TRANSPARENCY_SUFFIX}`.trim();
+ }
+ return body;
+}
+
+/** Region wording from the official demo cases: a leading phrase naming the marked area, or for
+ * annotations a closing sentence asking for the marks to be left out. */
+export function withLocalizedHint(
+ prompt: string,
+ mode: LocalizedEditMode,
+ colors: readonly string[] = [],
+): string {
+ const body = prompt.trim();
+ if (mode === "annotate") {
+ const named = colors.length ? joinWords(colors) : "annotation";
+ const suffix = `Do not render the ${named} annotation lines in the image.`;
+ return body.endsWith(suffix) ? body : `${body} ${suffix}`.trim();
+ }
+ const prefix =
+ mode === "paint"
+ ? "In the area marked with white paint,"
+ : "In the area marked by the mask,";
+ return body.startsWith(prefix) ? body : `${prefix} ${body}`.trim();
+}
+
+function joinWords(words: readonly string[]): string {
+ if (words.length < 2) return words[0] ?? "";
+ return `${words.slice(0, -1).join(", ")} and ${words[words.length - 1]}`;
+}
+
+export type EditSizing = "source" | "custom";
+
+/** Unified edit output size: Image 1's aspect ratio at the chosen area, or the custom size fitted. */
+export function resolveEditSize(
+ sizing: EditSizing,
+ source: { width: number; height: number } | null,
+ matchResolution: number,
+ custom: { width: number; height: number },
+ limits: SizeLimits,
+): { width: number; height: number } {
+ if (sizing === "source" && source && source.width > 0 && source.height > 0) {
+ return matchSourceSize(
+ source.width,
+ source.height,
+ matchResolution,
+ limits,
+ );
+ }
+ return fitSize(custom.width, custom.height, limits);
+}
+
+/** The conditioned half of a Reference / unified Edit request. Empty slots are dropped in order. */
+export function conditionedRequestFields(opts: {
+ workflow: "edit" | "reference";
+ initImage: string;
+ extras: readonly string[];
+ referenceResolution: number | null;
+ conditioning: DiffusionConditioning | null | undefined;
+ localized: { mode: LocalizedEditMode; image: string } | null;
+}): Pick<
+ DiffusionGenerateRequest,
+ | "workflow"
+ | "init_image"
+ | "reference_images"
+ | "reference_resolution"
+ | "localized_edit"
+> {
+ const allowed = opts.conditioning?.reference_resolutions ?? [];
+ const extras = opts.extras.filter(Boolean);
+ const unified = Boolean(opts.conditioning?.unified_edit);
+ const localized =
+ opts.workflow === "edit" && unified && opts.localized
+ ? opts.localized
+ : undefined;
+ return {
+ workflow: opts.workflow,
+ init_image: opts.initImage,
+ reference_images:
+ extras.length && (opts.workflow === "reference" || unified)
+ ? extras
+ : undefined,
+ reference_resolution:
+ opts.referenceResolution != null &&
+ allowed.includes(opts.referenceResolution)
+ ? opts.referenceResolution
+ : undefined,
+ localized_edit: localized,
+ };
+}
+
+/** What a restored conditioned recipe needs supplied again, for the restore toast. */
+export function restoreInputsNote(image: {
+ workflow?: string | null;
+ reference_image_count?: number | null;
+ localized_edit?: LocalizedEditMode | null;
+}): string | null {
+ const extras = image.reference_image_count ?? 0;
+ const layer =
+ image.localized_edit === "mask"
+ ? "the mask"
+ : image.localized_edit === "annotate"
+ ? "the annotations"
+ : image.localized_edit === "paint"
+ ? "the painted region"
+ : null;
+ if (image.workflow === "edit" || image.workflow === "reference") {
+ const parts = ["the source image"];
+ if (layer) parts.push(layer);
+ if (extras > 0)
+ parts.push(
+ `${extras} additional image${extras === 1 ? "" : "s"} in the same order`,
+ );
+ return parts.length > 1
+ ? `${parts.slice(0, -1).join(", ")} and ${parts[parts.length - 1]}`
+ : parts[0];
+ }
+ return null;
+}
diff --git a/studio/frontend/src/features/images/image-generation-defaults.ts b/studio/frontend/src/features/images/image-generation-defaults.ts
index 55ebc49ff43..dd018dc6a30 100644
--- a/studio/frontend/src/features/images/image-generation-defaults.ts
+++ b/studio/frontend/src/features/images/image-generation-defaults.ts
@@ -22,6 +22,11 @@ const MODEL_DEFAULTS: Array<{
{ match: "flux.2-klein-base", steps: 50, guidance: 4 },
{ match: "flux.2-klein", steps: 4, guidance: 1 },
{ match: "flux.2-dev", steps: 28, guidance: 4 },
+ // Qwen-Image-2.1 and its aliases, before the generic key.
+ { match: "qwen-image-2.1", steps: 40, guidance: 1 },
+ { match: "qwen-image-21", steps: 40, guidance: 1 },
+ { match: "qwen_image_21", steps: 40, guidance: 1 },
+ { match: "qwenimage21", steps: 40, guidance: 1 },
{ match: "qwen-image", steps: 20, guidance: 4 },
{ match: "z-image", steps: 20, guidance: 4 },
{ match: "ideogram", steps: 48, guidance: 7 },
diff --git a/studio/frontend/src/features/images/image-size.ts b/studio/frontend/src/features/images/image-size.ts
index 5fc49cb99e6..436ef5a8ca0 100644
--- a/studio/frontend/src/features/images/image-size.ts
+++ b/studio/frontend/src/features/images/image-size.ts
@@ -1,13 +1,94 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// Z-Image's range, which ImageGenerationPresetParams also enforces on the persisted recipe.
+import type { DiffusionConditioning } from "./api";
+
+// Z-Image's range, which every family but Qwen-Image-2.1 keeps.
export const MIN_DIM = 256;
export const MAX_DIM = 2048;
-export function snapDim(value: number): number {
+/** The output grid and bounds of the loaded model. */
+export interface SizeLimits {
+ multiple: number;
+ maxSide: number;
+ maxPixels: number;
+}
+
+export const DEFAULT_SIZE_LIMITS: SizeLimits = {
+ multiple: 16,
+ maxSide: MAX_DIM,
+ maxPixels: MAX_DIM * MAX_DIM,
+};
+
+/** The loaded model's limits, or the historical ones when the backend reports none. */
+export function sizeLimitsFrom(
+ conditioning: DiffusionConditioning | null | undefined,
+): SizeLimits {
+ if (!conditioning) return DEFAULT_SIZE_LIMITS;
+ return {
+ multiple: conditioning.dimension_multiple || DEFAULT_SIZE_LIMITS.multiple,
+ maxSide: conditioning.max_output_side || DEFAULT_SIZE_LIMITS.maxSide,
+ maxPixels: conditioning.max_output_pixels || DEFAULT_SIZE_LIMITS.maxPixels,
+ };
+}
+
+export function snapDim(
+ value: number,
+ limits: SizeLimits = DEFAULT_SIZE_LIMITS,
+): number {
if (!Number.isFinite(value)) return 1024;
- return Math.min(MAX_DIM, Math.max(MIN_DIM, Math.round(value / 16) * 16));
+ const m = limits.multiple;
+ const top = Math.floor(limits.maxSide / m) * m;
+ const bottom = Math.ceil(MIN_DIM / m) * m;
+ return Math.min(top, Math.max(bottom, Math.round(value / m) * m));
+}
+
+/** A (width, height) pair on the grid and inside both bounds, shrunk together to keep its ratio. */
+export function fitSize(
+ width: number,
+ height: number,
+ limits: SizeLimits = DEFAULT_SIZE_LIMITS,
+): { width: number; height: number } {
+ let w = snapDim(width, limits);
+ let h = snapDim(height, limits);
+ if (w * h <= limits.maxPixels) return { width: w, height: h };
+ const scale = Math.sqrt(limits.maxPixels / (w * h));
+ const m = limits.multiple;
+ w = Math.max(snapDim(MIN_DIM, limits), Math.floor((w * scale) / m) * m);
+ h = Math.max(snapDim(MIN_DIM, limits), Math.floor((h * scale) / m) * m);
+ return { width: w, height: h };
+}
+
+/** Mirrors the backend's match_source_size, so the size shown is the size generated. */
+export function matchSourceSize(
+ sourceWidth: number,
+ sourceHeight: number,
+ resolution: number,
+ limits: SizeLimits = DEFAULT_SIZE_LIMITS,
+): { width: number; height: number } {
+ const m = limits.multiple;
+ const ratio = Math.max(1e-6, sourceWidth / Math.max(1, sourceHeight));
+ let area = resolution * resolution;
+ let w = m;
+ let h = m;
+ for (let i = 0; i < 64; i++) {
+ w = Math.max(m, Math.round(Math.sqrt(area * ratio) / m) * m);
+ h = Math.max(m, Math.round(Math.sqrt(area / ratio) / m) * m);
+ if (Math.max(w, h) <= limits.maxSide && w * h <= limits.maxPixels) break;
+ area *= 0.9;
+ }
+ const shortMin = Math.ceil(MIN_DIM / m) * m;
+ const longMax = Math.floor(limits.maxSide / m) * m;
+ if (Math.min(w, h) < shortMin) {
+ let longSide = Math.round((shortMin * Math.max(ratio, 1 / ratio)) / m) * m;
+ longSide = Math.min(Math.max(longSide, shortMin), longMax);
+ while (longSide > shortMin && longSide * shortMin > limits.maxPixels)
+ longSide -= m;
+ return ratio >= 1
+ ? { width: longSide, height: shortMin }
+ : { width: shortMin, height: longSide };
+ }
+ return { width: w, height: h };
}
/** A gallery record's size as the Create form can hold it. Scaled as a pair, so the recipe's
@@ -22,9 +103,10 @@ export function restorableSize(
width: number,
height: number,
workflow?: string | null,
+ limits: SizeLimits = DEFAULT_SIZE_LIMITS,
): { width: number; height: number } {
if (workflow === "img2img") {
- return { width: snapDim(width), height: snapDim(height) };
+ return { width: snapDim(width, limits), height: snapDim(height, limits) };
}
if (
!Number.isFinite(width) ||
@@ -32,11 +114,15 @@ export function restorableSize(
width <= 0 ||
height <= 0
) {
- return { width: snapDim(width), height: snapDim(height) };
+ return { width: snapDim(width, limits), height: snapDim(height, limits) };
}
const upTo = Math.max(MIN_DIM / width, MIN_DIM / height);
- const downTo = Math.min(MAX_DIM / width, MAX_DIM / height);
+ const downTo = Math.min(
+ limits.maxSide / width,
+ limits.maxSide / height,
+ Math.sqrt(limits.maxPixels / (width * height)),
+ );
// A ratio too extreme to fit both bounds at any scale falls back to per-side clamping.
const scale = upTo > downTo ? 1 : Math.min(Math.max(1, upTo), downTo);
- return { width: snapDim(width * scale), height: snapDim(height * scale) };
+ return fitSize(width * scale, height * scale, limits);
}
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index eee9ac49c87..99a27d6ebaf 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -139,7 +139,31 @@ import {
import { toast } from "@/lib/toast";
import { subscribeModelEjected } from "@/lib/model-lifecycle-events";
import { DEFAULT_GEN, defaultsFor, resolutionFor } from "./image-generation-defaults";
-import { MAX_DIM, MIN_DIM, restorableSize, snapDim } from "./image-size";
+import {
+ MIN_DIM,
+ type SizeLimits,
+ DEFAULT_SIZE_LIMITS,
+ fitSize,
+ restorableSize,
+ sizeLimitsFrom,
+ snapDim,
+} from "./image-size";
+import {
+ ANNOTATION_COLORS,
+ type EditSizing,
+ REFERENCE_DETAIL_LABELS,
+ TRANSPARENCY_CHECKER,
+ additionalImageNumber,
+ conditionedRequestFields,
+ maxAdditionalImages,
+ presetsWithin,
+ resolveEditSize,
+ restoreInputsNote,
+ seedReferenceResolution,
+ withLocalizedHint,
+ withTransparencyPrompt,
+} from "./edit-conditioning";
+import { LocalizedEditCanvas } from "./localized-edit-canvas";
import {
type ControlNetSpecInput,
@@ -150,6 +174,7 @@ import {
type DiffusionLoadRequest,
type DiffusionLoraInfo,
type DiffusionStatus,
+ type LocalizedEditMode,
type GalleryImage,
type LoraSpecInput,
GenerateResponseLostError,
@@ -246,9 +271,16 @@ const RUNS_SLIDER_MAX = 128;
// Offered sizes; a locked ratio can derive one off-list, so the current value is added in.
const DIM_OPTIONS = [
256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1152, 1280,
- 1408, 1536, 1664, 1792, 1920, 2048,
+ 1408, 1536, 1664, 1792, 1920, 2048, 2304, 2560, 2752,
];
+// Larger limits than this unlock the 2K presets.
+const MAX_OUTPUT_DEFAULT = DEFAULT_SIZE_LIMITS.maxSide;
+
+function dimOptions(limits: SizeLimits): number[] {
+ return DIM_OPTIONS.filter((n) => n <= limits.maxSide && n % limits.multiple === 0);
+}
+
function DimensionSelect({
icon,
label,
@@ -256,6 +288,7 @@ function DimensionSelect({
open,
onOpenChange,
onChange,
+ limits = DEFAULT_SIZE_LIMITS,
}: {
icon: IconSvgElement;
label: string;
@@ -263,6 +296,7 @@ function DimensionSelect({
open: boolean;
onOpenChange: (open: boolean) => void;
onChange: (value: number) => void;
+ limits?: SizeLimits;
}) {
// Typing is held in a draft so a half-entered number is not snapped mid-keystroke.
const [draft, setDraft] = useState(String(value));
@@ -273,7 +307,7 @@ function DimensionSelect({
}
const commit = () => {
const typed = Number(draft);
- const next = snapDim(Number.isFinite(typed) && typed > 0 ? typed : value);
+ const next = snapDim(Number.isFinite(typed) && typed > 0 ? typed : value, limits);
setDraft(String(next));
setLastValue(next);
if (next !== value) onChange(next);
@@ -312,7 +346,7 @@ function DimensionSelect({
<ChevronDown className="size-4" />
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="max-h-72 overflow-y-auto">
- {DIM_OPTIONS.map((n) => (
+ {dimOptions(limits).map((n) => (
<DropdownMenuItem key={n} onSelect={() => pick(n)}>
<span className="tabular-nums">{n}</span>
</DropdownMenuItem>
@@ -1263,8 +1297,16 @@ export function ImagesPage({
// Upscale (hires fix): the enlargement factor and the low denoise strength that re-details the result.
const [upscaleFactor, setUpscaleFactor] = useState(2);
const [upscaleStrength, setUpscaleStrength] = useState(0.35);
- // Reference (FLUX.2): up to 3 ADDITIONAL reference images beyond the primary one.
+ // Reference and Edit: the ADDITIONAL images after the source. "" holds a cleared slot so others do not renumber.
const [referenceImages, setReferenceImages] = useState<string[]>([]);
+ const [referenceResolution, setReferenceResolution] = useState<number | null>(null);
+ const [editSizing, setEditSizing] = useState<EditSizing>("source");
+ const [matchResolution, setMatchResolution] = useState(1024);
+ const [localizedMode, setLocalizedMode] = useState<LocalizedEditMode | null>(null);
+ const [localizedLayer, setLocalizedLayer] = useState<string | null>(null);
+ const [localizedColor, setLocalizedColor] = useState(ANNOTATION_COLORS[0].value);
+ const [localizedColors, setLocalizedColors] = useState<string[]>([]);
+ const [localizedResetKey, setLocalizedResetKey] = useState(0);
// LoRA adapters selected for the next generation (id + weight), plus the list the picker offers.
const [loras, setLoras] = useState<LoraSpecInput[]>([]);
const [availableLoras, setAvailableLoras] = useState<DiffusionLoraInfo[]>([]);
@@ -1340,6 +1382,11 @@ export function ImagesPage({
// setInterval, so returning fires one immediate poll.
const genVisibilityListener = useRef<(() => void) | null>(null);
const [status, setStatus] = useState<DiffusionStatus | null>(null);
+ const conditioning = status?.loaded ? (status.conditioning ?? null) : null;
+ const sizeLimits = useMemo(() => sizeLimitsFrom(conditioning), [conditioning]);
+ const unifiedEdit = Boolean(conditioning?.unified_edit);
+ const referenceResolutions = conditioning?.reference_resolutions ?? [];
+ const maxExtras = maxAdditionalImages(conditioning, workflow === "edit" && unifiedEdit ? localizedMode : null);
// Controlled so the body-portaled overlays force-close while this page is mounted but off-tab.
const [selectorOpen, setSelectorOpen] = useState(false);
const [aspectOpen, setAspectOpen] = useState(false);
@@ -1998,7 +2045,7 @@ export function ImagesPage({
// Restore from the BASE batch seed, not this image's derived seed, or a replay with batch_size
// advances it again.
setSeed(String(image.batch_seed ?? image.seed));
- const restored = restorableSize(image.width, image.height, image.workflow);
+ const restored = restorableSize(image.width, image.height, image.workflow, sizeLimits);
setWidth(restored.width);
setHeight(restored.height);
// The batch shared one base seed, so a batch_index>0 image only reproduces by replaying the whole batch.
@@ -2023,12 +2070,24 @@ export function ImagesPage({
else setStrength(image.strength);
}
if (typeof image.upscale === "number") setUpscaleFactor(image.upscale);
- // None of the conditioning images are persisted, so a restore must clear the Transform /
- // Inpaint / Edit uploads and return to Create.
- setWorkflow("create");
+ // Conditioning images are not persisted, so every upload is cleared. Edit and Reference reopen
+ // their own workflow, so Generate stays blocked until the inputs are supplied again instead of
+ // replaying as text-to-image; the others return to Create.
+ const reopened: WorkflowId =
+ image.workflow === "edit" ? "edit" : image.workflow === "reference" ? "reference" : "create";
+ setWorkflow(reopened);
setInitImage(null);
setMaskImage(null);
- setReferenceImages([]);
+ setReferenceImages(
+ reopened === "create" ? [] : Array.from({ length: image.reference_image_count ?? 0 }, () => ""),
+ );
+ if (typeof image.reference_resolution === "number") {
+ setReferenceResolution(image.reference_resolution);
+ }
+ setLocalizedMode(reopened === "edit" ? (image.localized_edit ?? null) : null);
+ setLocalizedLayer(null);
+ setLocalizedColors([]);
+ if (reopened === "edit") setEditSizing("custom");
// The control image is not persisted, so clear any stale ControlNet selection.
setControlnetId("");
setControlImage(null);
@@ -2036,16 +2095,17 @@ export function ImagesPage({
// so rather than leaving the two silently disagreeing.
const rescaled =
restored.width !== image.width || restored.height !== image.height
- ? { description: `Size scaled to ${restored.width} × ${restored.height} to fit the ${MIN_DIM}-${MAX_DIM} range.` }
+ ? { description: `Size scaled to ${restored.width} × ${restored.height} to fit the ${MIN_DIM}-${sizeLimits.maxSide} range.` }
: undefined;
// Say so, rather than letting a conditioned image restore as a plain Create that generates something unrelated.
- const conditioned = CONDITIONED_WORKFLOW_INPUTS[image.workflow ?? ""];
+ const conditioned =
+ restoreInputsNote(image) ?? CONDITIONED_WORKFLOW_INPUTS[image.workflow ?? ""];
if (conditioned) {
toast.success(`Settings restored. Add ${conditioned} again to reproduce this image.`, rescaled);
} else {
toast.success("Settings restored to inputs", rescaled);
}
- }, [setWorkflow]);
+ }, [setWorkflow, sizeLimits]);
// A locked ratio keeps the paired dimension in step; "custom" frees both, Flip swaps W/H. ratioHW is h/w for [a,b].
const ratioHW = (a: number, b: number) => (portrait ? a / b : b / a);
@@ -2053,19 +2113,19 @@ export function ImagesPage({
setAspect(key);
if (key === "custom") return;
const [a, b] = ASPECT_RATIOS[key];
- setHeight(snapDim(width * ratioHW(a, b)));
+ setHeight(snapDim(width * ratioHW(a, b), sizeLimits));
};
const changeWidth = (v: number) => {
setWidth(v);
if (aspect === "custom") return;
const [a, b] = ASPECT_RATIOS[aspect];
- setHeight(snapDim(v * ratioHW(a, b)));
+ setHeight(snapDim(v * ratioHW(a, b), sizeLimits));
};
const changeHeight = (v: number) => {
setHeight(v);
if (aspect === "custom") return;
const [a, b] = ASPECT_RATIOS[aspect];
- setWidth(snapDim(v / ratioHW(a, b)));
+ setWidth(snapDim(v / ratioHW(a, b), sizeLimits));
};
const flipDimensions = () => {
setWidth(height);
@@ -3349,6 +3409,154 @@ export function ImagesPage({
cancelLoadRef.current = () => void handleCancelLoad();
}, [handleCancelLoad]);
+ // Seed reference detail and match-source area from the build's canvas tier when the loaded build changes.
+ const buildKey = status?.loaded
+ ? [
+ status.repo_id,
+ status.base_repo,
+ status.model_kind,
+ status.transformer_quant,
+ (status.conditioning?.reference_resolutions ?? []).join(","),
+ ].join("|")
+ : null;
+ const [seededBuild, setSeededBuild] = useState<string | null>(null);
+ if (buildKey !== seededBuild) {
+ setSeededBuild(buildKey);
+ if (status?.loaded) {
+ const tier = resolutionFor(status.base_repo ?? status.repo_id ?? "", {
+ modelKind: status.model_kind,
+ transformerQuant: status.transformer_quant,
+ }).width;
+ setReferenceResolution(
+ seedReferenceResolution(status.conditioning?.reference_resolutions ?? [], tier),
+ );
+ setMatchResolution(tier);
+ }
+ }
+
+ // Keep the size on the loaded model's grid and inside its bounds, so the value shown is the value sent.
+ const limitsKey = `${sizeLimits.multiple}|${sizeLimits.maxSide}|${sizeLimits.maxPixels}`;
+ const [fittedLimits, setFittedLimits] = useState(limitsKey);
+ if (limitsKey !== fittedLimits) {
+ setFittedLimits(limitsKey);
+ const fitted = fitSize(width, height, sizeLimits);
+ if (fitted.width !== width) setWidth(fitted.width);
+ if (fitted.height !== height) setHeight(fitted.height);
+ }
+
+ // Keyed to the image it was read from, so a stale read never sizes a newer source.
+ const [sourceRead, setSourceRead] = useState<{
+ src: string;
+ width: number;
+ height: number;
+ } | null>(null);
+ const sourceDims = sourceRead && sourceRead.src === initImage ? sourceRead : null;
+ useEffect(() => {
+ if (!initImage) return;
+ let live = true;
+ loadImage(initImage)
+ .then((img) => {
+ if (live) setSourceRead({ src: initImage, width: img.naturalWidth, height: img.naturalHeight });
+ })
+ .catch(() => {});
+ return () => {
+ live = false;
+ };
+ }, [initImage]);
+
+ const editSize = useMemo(
+ () =>
+ resolveEditSize(editSizing, sourceDims, matchResolution, { width, height }, sizeLimits),
+ [editSizing, sourceDims, matchResolution, width, height, sizeLimits],
+ );
+ const officialPresets = useMemo(() => presetsWithin(sizeLimits), [sizeLimits]);
+ const showOfficialPresets = sizeLimits.maxSide > MAX_OUTPUT_DEFAULT && officialPresets.length > 0;
+ const unifiedEditActive = workflow === "edit" && unifiedEdit;
+ const onLocalizedLayer = useCallback((dataUrl: string | null) => setLocalizedLayer(dataUrl), []);
+ const onLocalizedColors = useCallback((names: string[]) => setLocalizedColors(names), []);
+
+ // `numberOf` is the image number the model sees for slot i.
+ const renderAdditionalImages = (numberOf: (i: number) => number, hint: string) => (
+ <>
+ {referenceImages.map((img, i) => (
+ <Field key={i} label={`Image ${numberOf(i)}`} hint={hint}>
+ <div className="space-y-1.5">
+ <ImageDropzone
+ value={img}
+ onChange={(v) =>
+ // Keep the slot in place (empty string when cleared) so other slots do not renumber mid-edit;
+ // empty slots are dropped at send time.
+ setReferenceImages((prev) => prev.map((p, j) => (j === i ? (v ?? "") : p)))
+ }
+ removeLabel={`Remove image ${numberOf(i)}`}
+ />
+ <Button
+ type="button"
+ variant="secondary"
+ size="sm"
+ className="w-full"
+ onClick={() => setReferenceImages((prev) => prev.filter((_, j) => j !== i))}
+ >
+ <HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
+ Remove image {numberOf(i)}
+ </Button>
+ </div>
+ </Field>
+ ))}
+ {referenceImages.length > maxExtras && (
+ <p className="text-xs text-destructive">
+ This model takes {maxExtras} additional image{maxExtras === 1 ? "" : "s"} here. Remove{" "}
+ {referenceImages.length - maxExtras} to generate.
+ </p>
+ )}
+ {referenceImages.length < maxExtras && (
+ <Button
+ type="button"
+ variant="secondary"
+ size="sm"
+ className="w-full"
+ disabled={!initImage}
+ onClick={() => setReferenceImages((prev) => [...prev, ""])}
+ >
+ <HugeiconsIcon icon={ImageAdd02Icon} className="size-3.5" />
+ Add image {numberOf(referenceImages.length)}
+ </Button>
+ )}
+ </>
+ );
+
+ const engineNotes = conditioning?.notes?.length ? (
+ <ul className="space-y-1 text-ui-11 leading-snug text-muted-foreground">
+ {conditioning.notes.map((note) => (
+ <li key={note}>{note}</li>
+ ))}
+ </ul>
+ ) : null;
+
+ const referenceDetailControl =
+ referenceResolutions.length > 0 && referenceResolution != null ? (
+ <Field
+ label="Reference detail"
+ hint="The resolution every input image is resized to (by area) before the model reads it. Separate from the output size: higher keeps more detail from the inputs and costs more memory and time for every image. High (2048) with many images needs a large GPU."
+ >
+ <Select
+ value={String(referenceResolution)}
+ onValueChange={(v) => setReferenceResolution(Number(v))}
+ >
+ <SelectTrigger aria-label="Reference detail">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {referenceResolutions.map((r) => (
+ <SelectItem key={r} value={String(r)}>
+ {REFERENCE_DETAIL_LABELS[r] ?? String(r)}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </Field>
+ ) : null;
+
const handleGenerate = useCallback(async () => {
if (!prompt.trim()) {
toast.error("Prompt is empty");
@@ -3380,6 +3588,27 @@ export function ImagesPage({
toast.error("Paint a mask over the region to regenerate");
return;
}
+ const unifiedEditRun = isEdit && unifiedEdit;
+ if (unifiedEditRun && localizedMode && !localizedLayer) {
+ toast.error(
+ localizedMode === "annotate"
+ ? "Draw the annotations on the source image, or turn Localize off"
+ : "Paint the region on the source image, or turn Localize off",
+ );
+ return;
+ }
+ const emptySlot = (isReference || unifiedEditRun) ? referenceImages.findIndex((img) => !img) : -1;
+ if (emptySlot >= 0) {
+ const n = isReference ? emptySlot + 2 : additionalImageNumber(emptySlot, localizedMode);
+ toast.error(`Image ${n} is empty. Add it, or remove that slot.`);
+ return;
+ }
+ if ((isReference || unifiedEditRun) && referenceImages.filter(Boolean).length > maxExtras) {
+ toast.error(
+ `This model takes at most ${maxExtras + 1 + (unifiedEditRun && localizedMode === "mask" ? 1 : 0)} input images in total. Remove ${referenceImages.filter(Boolean).length - maxExtras}.`,
+ );
+ return;
+ }
if (isExtend && !(extendSides.left || extendSides.right || extendSides.top || extendSides.bottom)) {
toast.error("Pick at least one side to extend");
return;
@@ -3391,7 +3620,7 @@ export function ImagesPage({
let condMask: string | undefined;
let condStrength: number | undefined;
let condUpscale: number | undefined;
- let condRefImages: string[] | undefined;
+ let condFields: ReturnType<typeof conditionedRequestFields> | undefined;
try {
if (isTransform) {
condInit = initImage ?? undefined;
@@ -3411,15 +3640,21 @@ export function ImagesPage({
condInit = initImage ?? undefined;
condUpscale = upscaleFactor;
condStrength = upscaleStrength;
- } else if (isReference) {
- // FLUX.2 reference conditioning: send the primary plus extra references. A fresh image is
- // generated at the slider size.
- condInit = initImage ?? undefined;
- const extras = referenceImages.filter(Boolean);
- if (extras.length) condRefImages = extras;
+ } else if (isReference || unifiedEditRun) {
+ condFields = conditionedRequestFields({
+ workflow: isReference ? "reference" : "edit",
+ initImage: initImage!,
+ extras: referenceImages,
+ referenceResolution,
+ conditioning,
+ localized:
+ unifiedEditRun && localizedMode && localizedLayer
+ ? { mode: localizedMode, image: localizedLayer }
+ : null,
+ });
} else if (isEdit) {
- // Instruction editing: send the source image; the prompt IS the instruction. No mask, no strength.
- condInit = initImage ?? undefined;
+ // Edit-only model: the source alone; the prompt IS the instruction and the output takes its size.
+ condFields = { workflow: "edit", init_image: initImage ?? undefined };
}
} catch {
toast.error("Could not prepare the source image");
@@ -3438,9 +3673,14 @@ export function ImagesPage({
baseSeed = Math.floor(Math.random() * 2 ** 32);
}
- // Snap custom dims to the model's grid so a half-typed value cannot 422.
- const w = snapDim(width);
- const h = snapDim(height);
+ // Snap to the model's grid and bounds so a half-typed value cannot 400.
+ const sent = unifiedEditRun ? editSize : fitSize(width, height, sizeLimits);
+ const w = sent.width;
+ const h = sent.height;
+ if (!unifiedEditRun || editSizing === "custom") {
+ if (w !== width) setWidth(w);
+ if (h !== height) setHeight(h);
+ }
// A large run count is legitimate, so no upper cap: floor at 1 and ignore non-numeric input.
const runs = Number.isFinite(count) && count >= 1 ? Math.floor(count) : 1;
@@ -3535,7 +3775,7 @@ export function ImagesPage({
mask_image: condMask,
strength: condStrength,
upscale: condUpscale,
- reference_images: condRefImages,
+ ...condFields,
// Drop empty and zero-weight rows and trim hand-typed repo ids, so the recipe records only
// adapters that applied. Gated on loraCapable, since a restore can leave adapters in state.
loras: (() => {
@@ -3612,7 +3852,7 @@ export function ImagesPage({
setGenDone(null);
setGenStep(null);
}
- }, [prompt, negativePrompt, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus]);
+ }, [prompt, negativePrompt, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
// Stop the in-flight generation. Latch FIRST, so a multi-run request stops even if the POST
// races the run that is already finishing.
@@ -4248,66 +4488,221 @@ export function ImagesPage({
{workflow === "reference" && (
<>
<Field
- label="Reference image"
+ label={unifiedEdit ? "Image 1" : "Reference image"}
hint="A reference the model draws on (subject, style, or composition) while generating a NEW image from your prompt at the size below. Unlike Transform, it is not a redraw of this image, so there is no strength."
>
<ImageDropzone value={initImage} onChange={handleInitChange} />
</Field>
- {referenceImages.map((img, i) => (
+ {renderAdditionalImages(
+ (i) => i + 2,
+ "An extra reference combined with the others (e.g. one for the subject, one for the style). Refer to it in the prompt by its number.",
+ )}
+ {referenceDetailControl}
+ {engineNotes}
+ </>
+ )}
+
+ {workflow === "edit" && !unifiedEdit && (
+ <Field
+ label="Source image"
+ hint="The image to edit. Describe the change in the prompt below (e.g. 'make it night', 'add a red hat', 'change the background to a beach')."
+ >
+ <ImageDropzone value={initImage} onChange={handleInitChange} />
+ </Field>
+ )}
+
+ {unifiedEditActive && (
+ <>
+ {initImage && localizedMode ? (
<Field
- key={i}
- label={`Reference ${i + 2}`}
- hint="An extra reference combined with the others (e.g. one for the subject, one for the style)."
+ label="Image 1 (source)"
+ hint={
+ localizedMode === "annotate"
+ ? "Draw outlines or marks around what to change, in the colours your instruction names."
+ : localizedMode === "paint"
+ ? "Paint the area to change in white. The model sees the white paint on the source."
+ : "Paint the area to change. It is sent as a white-on-black mask, Image 2, right after the source."
+ }
>
<div className="space-y-1.5">
- <ImageDropzone
- value={img}
- onChange={(v) =>
- // Keep the slot in place (empty string when cleared) so other slots do not renumber mid-edit;
- // empty slots are dropped at send time.
- setReferenceImages((prev) =>
- prev.map((p, j) => (j === i ? (v ?? "") : p)),
- )
- }
+ <LocalizedEditCanvas
+ image={initImage}
+ mode={localizedMode}
+ color={localizedColor}
+ brushPct={brushPct}
+ resetKey={localizedResetKey}
+ onLayerChange={onLocalizedLayer}
+ onColorsChange={onLocalizedColors}
/>
<Button
type="button"
variant="secondary"
size="sm"
className="w-full"
- onClick={() => setReferenceImages((prev) => prev.filter((_, j) => j !== i))}
+ onClick={() => handleInitChange(null)}
>
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
- Remove reference {i + 2}
+ Remove source image
</Button>
</div>
</Field>
- ))}
- {referenceImages.length < 3 && (
+ ) : (
+ <Field
+ label="Image 1 (source)"
+ hint="The image to edit. Describe the change in the instruction below; add more images to combine them, and refer to them as Image 2, Image 3 and so on."
+ >
+ <ImageDropzone value={initImage} onChange={handleInitChange} />
+ </Field>
+ )}
+ {conditioning?.localized_edit_modes?.length ? (
+ <Field
+ label="Localize"
+ hint="Point the edit at a region. It guides a generative edit: the model redraws the whole image and is asked to change the marked area, so pixels outside it are not guaranteed to stay identical."
+ >
+ <Select
+ value={localizedMode ?? "off"}
+ onValueChange={(v) => {
+ setLocalizedMode(v === "off" ? null : (v as LocalizedEditMode));
+ setLocalizedLayer(null);
+ setLocalizedColors([]);
+ }}
+ >
+ <SelectTrigger aria-label="Localize">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="off">Off (whole image)</SelectItem>
+ {conditioning.localized_edit_modes.includes("annotate") && (
+ <SelectItem value="annotate">Colour annotations</SelectItem>
+ )}
+ {conditioning.localized_edit_modes.includes("paint") && (
+ <SelectItem value="paint">White paint on the source</SelectItem>
+ )}
+ {conditioning.localized_edit_modes.includes("mask") && (
+ <SelectItem value="mask">Separate mask (Image 2)</SelectItem>
+ )}
+ </SelectContent>
+ </Select>
+ </Field>
+ ) : null}
+ {localizedMode && (
+ <>
+ {localizedMode === "annotate" && (
+ <Field label="Annotation colour">
+ <div className="flex gap-2">
+ {ANNOTATION_COLORS.map((c) => (
+ <button
+ key={c.value}
+ type="button"
+ aria-label={c.name}
+ aria-pressed={localizedColor === c.value}
+ onClick={() => setLocalizedColor(c.value)}
+ className={cn(
+ "size-7 rounded-full border-2",
+ localizedColor === c.value ? "border-foreground" : "border-transparent",
+ )}
+ style={{ backgroundColor: c.value }}
+ />
+ ))}
+ </div>
+ </Field>
+ )}
+ <SliderField
+ label="Brush size"
+ value={brushPct}
+ min={1}
+ max={25}
+ step={1}
+ onChange={setBrushPct}
+ />
+ <div className="flex gap-2">
+ <Button
+ type="button"
+ variant="secondary"
+ size="sm"
+ className="flex-1"
+ disabled={!localizedLayer}
+ onClick={() => setLocalizedResetKey((k) => k + 1)}
+ >
+ Clear
+ </Button>
+ <Button
+ type="button"
+ variant="secondary"
+ size="sm"
+ className="flex-1"
+ onClick={() =>
+ setPrompt((p) => withLocalizedHint(p, localizedMode, localizedColors))
+ }
+ >
+ Add region wording
+ </Button>
+ </div>
+ </>
+ )}
+ {renderAdditionalImages(
+ (i) => additionalImageNumber(i, localizedMode),
+ "Another input the instruction can refer to by its number, such as a person, product or style to bring into the source.",
+ )}
+ <Field
+ label="Output size"
+ hint="Match Image 1 keeps the source's proportions at the chosen size. Custom uses the aspect ratio and resolution below. The size shown is the size generated."
+ >
+ <div className="flex items-center gap-2">
+ <Select
+ value={editSizing}
+ onValueChange={(v) => setEditSizing(v as EditSizing)}
+ >
+ <SelectTrigger aria-label="Output size" className="flex-1">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ <SelectItem value="source">Match Image 1</SelectItem>
+ <SelectItem value="custom">Custom</SelectItem>
+ </SelectContent>
+ </Select>
+ {editSizing === "source" && (
+ <Select
+ value={String(matchResolution)}
+ onValueChange={(v) => setMatchResolution(Number(v))}
+ >
+ <SelectTrigger aria-label="Match size" className="w-[120px]">
+ <SelectValue />
+ </SelectTrigger>
+ <SelectContent>
+ {[512, 768, 1024, 1536, 2048]
+ .filter((r) => r <= sizeLimits.maxSide)
+ .map((r) => (
+ <SelectItem key={r} value={String(r)}>
+ {r === 2048 ? "2K" : `${r} px`}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ )}
+ </div>
+ <p className="text-ui-11 tabular-nums text-muted-foreground">
+ {editSizing === "source" && !sourceDims
+ ? "Add Image 1 to size the output from it."
+ : `${editSize.width} × ${editSize.height}`}
+ </p>
+ </Field>
+ {referenceDetailControl}
+ {engineNotes}
+ {unifiedEdit && (
<Button
type="button"
variant="secondary"
size="sm"
className="w-full"
- disabled={!initImage}
- onClick={() => setReferenceImages((prev) => [...prev, ""])}
+ onClick={() => setPrompt((p) => withTransparencyPrompt(p))}
>
- <HugeiconsIcon icon={ImageAdd02Icon} className="size-3.5" />
- Add another reference
+ Ask for a transparent background
</Button>
)}
</>
)}
- {workflow === "edit" && (
- <Field
- label="Source image"
- hint="The image to edit. Describe the change in the prompt below (e.g. 'make it night', 'add a red hat', 'change the background to a beach')."
- >
- <ImageDropzone value={initImage} onChange={handleInitChange} />
- </Field>
- )}
-
<Field label={workflow === "edit" ? "Instruction" : "Prompt"}>
<Textarea
rows={4}
@@ -4461,6 +4856,8 @@ export function ImagesPage({
</div>
</Field>
)}
+ {!(unifiedEditActive && editSizing === "source") && (
+ <>
<Field
label="Aspect ratio"
hint="Pick a ratio to lock the proportions, then set the size below. Flip swaps width and height."
@@ -4520,9 +4917,9 @@ export function ImagesPage({
: workflow === "inpaint" ||
workflow === "extend" ||
workflow === "upscale" ||
- workflow === "edit"
+ (workflow === "edit" && !unifiedEdit)
? "Not used by this workflow: the output size comes from the source image. Upload a smaller image to generate at a smaller size."
- : "Width and height in pixels. Sizes run from 256 to 2048 in steps of 16. Z-Image is trained around 1 megapixel, so much larger sizes can look worse."
+ : `Width and height in pixels. Sizes run from ${MIN_DIM} to ${sizeLimits.maxSide} in steps of ${sizeLimits.multiple}${sizeLimits.maxPixels < sizeLimits.maxSide * sizeLimits.maxSide ? `, up to ${(sizeLimits.maxPixels / 1e6).toFixed(1)} megapixels` : ""}. Most models are trained around 1 megapixel, so much larger sizes can look worse.`
}
>
<div className="flex items-center gap-2">
@@ -4533,6 +4930,7 @@ export function ImagesPage({
open={active && widthOpen}
onOpenChange={(o) => setWidthOpen(active && o)}
onChange={changeWidth}
+ limits={sizeLimits}
/>
<DimensionSelect
icon={ArrowUpDownIcon}
@@ -4541,9 +4939,42 @@ export function ImagesPage({
open={active && heightOpen}
onOpenChange={(o) => setHeightOpen(active && o)}
onChange={changeHeight}
+ limits={sizeLimits}
/>
</div>
</Field>
+ {showOfficialPresets && (
+ <Field
+ label="2K presets"
+ hint="The model's native 2K sizes. They take several times the memory and time of a 1 megapixel image."
+ >
+ <Select
+ value=""
+ onValueChange={(v) => {
+ const preset = officialPresets.find((p) => `${p.width}x${p.height}` === v);
+ if (!preset) return;
+ setWidth(preset.width);
+ setHeight(preset.height);
+ const m = matchAspect(preset.width, preset.height);
+ setAspect(m.key);
+ setPortrait(m.portrait);
+ }}
+ >
+ <SelectTrigger aria-label="2K presets">
+ <SelectValue placeholder="Choose a 2K size" />
+ </SelectTrigger>
+ <SelectContent>
+ {officialPresets.map((p) => (
+ <SelectItem key={`${p.width}x${p.height}`} value={`${p.width}x${p.height}`}>
+ {`${p.label} (${p.width} × ${p.height})`}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </Field>
+ )}
+ </>
+ )}
{/* First of the one-line sliders, so it takes a bigger break than the gap gives. */}
<div className="pt-2">
@@ -4637,6 +5068,7 @@ export function ImagesPage({
<img
src={selectedSrc}
alt={selected.prompt}
+ style={TRANSPARENCY_CHECKER}
className="max-h-full max-w-full object-contain shadow-sm"
/>
{/* Actions grouped in one glass toolbar so they stay legible over any image. Size and seed
diff --git a/studio/frontend/src/features/images/localized-edit-canvas.tsx b/studio/frontend/src/features/images/localized-edit-canvas.tsx
new file mode 100644
index 00000000000..01eecd705dc
--- /dev/null
+++ b/studio/frontend/src/features/images/localized-edit-canvas.tsx
@@ -0,0 +1,197 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ type PointerEvent,
+ useCallback,
+ useEffect,
+ useRef,
+ useState,
+} from "react";
+
+import type { LocalizedEditMode } from "./api";
+import { ANNOTATION_COLORS, TRANSPARENCY_CHECKER } from "./edit-conditioning";
+
+/** Draws the localized-edit layer over the source at its NATIVE resolution: an RGBA layer of
+ * coloured strokes for annotate, a white-on-black mask for paint and mask. */
+export function LocalizedEditCanvas({
+ image,
+ mode,
+ color,
+ brushPct,
+ resetKey,
+ onLayerChange,
+ onColorsChange,
+}: {
+ image: string;
+ mode: LocalizedEditMode;
+ color: string;
+ brushPct: number;
+ resetKey: number;
+ onLayerChange: (dataUrl: string | null) => void;
+ onColorsChange: (names: string[]) => void;
+}) {
+ const dispRef = useRef<HTMLCanvasElement | null>(null);
+ const layerRef = useRef<HTMLCanvasElement | null>(null);
+ const dims = useRef<{ w: number; h: number }>({ w: 0, h: 0 });
+ const drawing = useRef(false);
+ const dirty = useRef(false);
+ const last = useRef<{ x: number; y: number } | null>(null);
+ const used = useRef<Set<string>>(new Set());
+ // Drawing waits until the layer for THIS source, mode and clear has been sized.
+ const layerKey = `${resetKey}|${mode}|${image}`;
+ const [readyFor, setReadyFor] = useState<string | null>(null);
+ const ready = readyFor === layerKey;
+
+ // A new source, a new mode or Clear starts a blank layer at the source's natural size. The old
+ // layer is dropped at once, not on load, so Generate never pairs it with the new source.
+ useEffect(() => {
+ let live = true;
+ drawing.current = false;
+ dirty.current = false;
+ used.current = new Set();
+ onLayerChange(null);
+ onColorsChange([]);
+ const img = new Image();
+ img.onload = () => {
+ if (!live) return;
+ const w = img.naturalWidth;
+ const h = img.naturalHeight;
+ dims.current = { w, h };
+ const disp = dispRef.current;
+ const layer = layerRef.current ?? document.createElement("canvas");
+ layerRef.current = layer;
+ if (!disp) return;
+ disp.width = w;
+ disp.height = h;
+ layer.width = w;
+ layer.height = h;
+ const lctx = layer.getContext("2d");
+ const dctx = disp.getContext("2d");
+ if (!lctx || !dctx) return;
+ lctx.clearRect(0, 0, w, h);
+ if (mode !== "annotate") {
+ lctx.fillStyle = "#000";
+ lctx.fillRect(0, 0, w, h);
+ }
+ dctx.clearRect(0, 0, w, h);
+ setReadyFor(`${resetKey}|${mode}|${image}`);
+ };
+ img.src = image;
+ return () => {
+ live = false;
+ };
+ }, [image, mode, resetKey, onLayerChange, onColorsChange]);
+
+ const radius = useCallback(() => {
+ const base = Math.min(dims.current.w, dims.current.h) || 1024;
+ // Annotations are outlines, so a fraction of the region brush.
+ const pct = mode === "annotate" ? brushPct / 4 : brushPct;
+ return Math.max(1.5, (pct / 100) * base);
+ }, [brushPct, mode]);
+
+ const toNatural = (e: PointerEvent<HTMLCanvasElement>) => {
+ const disp = dispRef.current;
+ if (!disp) return { x: 0, y: 0 };
+ const r = disp.getBoundingClientRect();
+ return {
+ x: ((e.clientX - r.left) / r.width) * dims.current.w,
+ y: ((e.clientY - r.top) / r.height) * dims.current.h,
+ };
+ };
+
+ const stroke = (
+ from: { x: number; y: number } | null,
+ to: { x: number; y: number },
+ ) => {
+ const disp = dispRef.current;
+ const layer = layerRef.current;
+ if (!disp || !layer) return;
+ const r = radius();
+ const shown =
+ mode === "annotate"
+ ? color
+ : mode === "paint"
+ ? "#ffffff"
+ : "rgba(244,114,114,0.55)";
+ const stored = mode === "annotate" ? color : "#ffffff";
+ const layers: Array<[CanvasRenderingContext2D | null, string]> = [
+ [disp.getContext("2d"), shown],
+ [layer.getContext("2d"), stored],
+ ];
+ for (const [ctx, style] of layers) {
+ if (!ctx) continue;
+ ctx.strokeStyle = style;
+ ctx.fillStyle = style;
+ ctx.lineWidth = r * 2;
+ ctx.lineCap = "round";
+ ctx.lineJoin = "round";
+ ctx.beginPath();
+ ctx.arc(to.x, to.y, r, 0, Math.PI * 2);
+ ctx.fill();
+ if (from) {
+ ctx.beginPath();
+ ctx.moveTo(from.x, from.y);
+ ctx.lineTo(to.x, to.y);
+ ctx.stroke();
+ }
+ }
+ dirty.current = true;
+ if (mode === "annotate") {
+ const name = ANNOTATION_COLORS.find((c) => c.value === color)?.name;
+ if (name && !used.current.has(name)) {
+ used.current.add(name);
+ onColorsChange([...used.current]);
+ }
+ }
+ };
+
+ const onDown = (e: PointerEvent<HTMLCanvasElement>) => {
+ if (!ready) return;
+ drawing.current = true;
+ try {
+ e.currentTarget.setPointerCapture(e.pointerId);
+ } catch {
+ // setPointerCapture can throw for synthetic events; safe to ignore.
+ }
+ const p = toNatural(e);
+ last.current = p;
+ stroke(null, p);
+ };
+ const onMove = (e: PointerEvent<HTMLCanvasElement>) => {
+ if (!drawing.current) return;
+ const p = toNatural(e);
+ stroke(last.current, p);
+ last.current = p;
+ };
+ const onUp = () => {
+ if (!drawing.current) return;
+ drawing.current = false;
+ last.current = null;
+ const layer = layerRef.current;
+ if (layer && dirty.current) onLayerChange(layer.toDataURL("image/png"));
+ };
+
+ return (
+ <div
+ className="relative overflow-hidden rounded-[10px] border border-border"
+ style={TRANSPARENCY_CHECKER}
+ >
+ <img
+ src={image}
+ alt="Edit source"
+ className="block w-full select-none"
+ draggable={false}
+ />
+ <canvas
+ ref={dispRef}
+ data-testid="localized-edit-canvas"
+ onPointerDown={onDown}
+ onPointerMove={onMove}
+ onPointerUp={onUp}
+ onPointerLeave={onUp}
+ className="absolute inset-0 h-full w-full cursor-crosshair touch-none"
+ />
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/images/workflows.ts b/studio/frontend/src/features/images/workflows.ts
index ee7c6968681..ed003d4927e 100644
--- a/studio/frontend/src/features/images/workflows.ts
+++ b/studio/frontend/src/features/images/workflows.ts
@@ -42,6 +42,13 @@ export const WORKFLOW_TABS: Array<{
icon: SparklesIcon,
hint: "Generate a new image from a prompt",
},
+ {
+ id: "edit",
+ label: "Edit",
+ icon: Edit03Icon,
+ requires: "edit",
+ hint: "Change an image with an instruction",
+ },
{
id: "transform",
label: "Transform",
@@ -77,11 +84,4 @@ export const WORKFLOW_TABS: Array<{
requires: "reference",
hint: "Generate guided by a reference image",
},
- {
- id: "edit",
- label: "Edit",
- icon: Edit03Icon,
- requires: "edit",
- hint: "Change an image with an instruction",
- },
];
diff --git a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
index 3b77e6a6919..5370eabec4e 100644
--- a/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
+++ b/studio/frontend/src/features/model-picker/components/model-selector/model-catalog.ts
@@ -182,9 +182,8 @@ export const IMAGE_CATALOG: CatalogGroup[] = [
{
canonicalId: "unsloth/Qwen-Image-2.1",
displayName: "Qwen-Image 2.1",
- // Text-to-image that also accepts reference images on the same call (the family's `reference`
- // flag), rather than a separate edit pipeline the way Qwen-Image-Edit is.
- description: "Text-to-image",
+ // One pipeline for text-to-image and editing (`unified_edit`); the Images page follows the engine's status.
+ description: "Text-to-image and image editing",
scope: "image",
// Same reason as the 2512 row below: the int8 half of the prequant repo is reached through
// prequant_variant_repos and has no artifact row, so alias it to keep a pasted id finding it.
diff --git a/studio/frontend/tests/image-restore-settings-size.test.ts b/studio/frontend/tests/image-restore-settings-size.test.ts
index 61f81c810f4..96fbccd68b4 100644
--- a/studio/frontend/tests/image-restore-settings-size.test.ts
+++ b/studio/frontend/tests/image-restore-settings-size.test.ts
@@ -87,7 +87,7 @@ test("restoreSettings puts the record through restorableSize", () => {
const body = restoreSettingsBody();
assert.match(
body,
- /restorableSize\(image\.width, image\.height, image\.workflow\)/,
+ /restorableSize\(image\.width, image\.height, image\.workflow, sizeLimits\)/,
);
assert.doesNotMatch(
body,
diff --git a/studio/frontend/tests/image-unified-edit.test.ts b/studio/frontend/tests/image-unified-edit.test.ts
new file mode 100644
index 00000000000..d1485b96f38
--- /dev/null
+++ b/studio/frontend/tests/image-unified-edit.test.ts
@@ -0,0 +1,247 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Qwen-Image-2.1 editing on the Images page: the size the form shows is the size the backend
+// renders, the request keeps image order and drops nothing silently, and a restored edit reopens
+// Edit with its inputs requested again instead of replaying as text-to-image.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { DiffusionConditioning } from "../src/features/images/api.ts";
+import {
+ additionalImageNumber,
+ conditionedRequestFields,
+ maxAdditionalImages,
+ presetsWithin,
+ resolveEditSize,
+ restoreInputsNote,
+ seedReferenceResolution,
+ withLocalizedHint,
+ withTransparencyPrompt,
+} from "../src/features/images/edit-conditioning.ts";
+import { defaultsFor } from "../src/features/images/image-generation-defaults.ts";
+import {
+ DEFAULT_SIZE_LIMITS,
+ fitSize,
+ matchSourceSize,
+ restorableSize,
+ sizeLimitsFrom,
+ snapDim,
+} from "../src/features/images/image-size.ts";
+
+import { readSrc } from "./helpers/kit.ts";
+
+const QWEN21: DiffusionConditioning = {
+ max_condition_images: 10,
+ alpha: true,
+ dimension_multiple: 32,
+ max_output_side: 2752,
+ max_output_pixels: 2400 * 1792,
+ reference_resolutions: [512, 1024, 2048],
+ unified_edit: true,
+ localized_edit_modes: ["annotate", "paint", "mask"],
+};
+const LIMITS = sizeLimitsFrom(QWEN21);
+
+test("match-source sizes agree with the backend's match_source_size", () => {
+ // Same pairs the backend tests pin, so the form never shows a size the engine does not render.
+ assert.deepEqual(matchSourceSize(300, 200, 1024, LIMITS), {
+ width: 1248,
+ height: 832,
+ });
+ assert.deepEqual(matchSourceSize(64, 32, 1024, LIMITS), {
+ width: 1440,
+ height: 736,
+ });
+ const big = matchSourceSize(4000, 1000, 2048, LIMITS);
+ assert.ok(
+ big.width <= 2752 && big.width * big.height <= 2400 * 1792,
+ JSON.stringify(big),
+ );
+ assert.equal(big.width % 32, 0);
+ assert.equal(big.height % 32, 0);
+});
+
+test("an elongated source still matches to a size the request accepts", () => {
+ // 4096x256 at the 512 tier rounded to 2048x128, which the API refuses (both sides >= 256).
+ for (const [w, h, r] of [
+ [4096, 256, 512],
+ [256, 4096, 512],
+ [20000, 10, 1024],
+ ]) {
+ for (const limits of [LIMITS, DEFAULT_SIZE_LIMITS]) {
+ const out = matchSourceSize(w, h, r, limits);
+ assert.ok(Math.min(out.width, out.height) >= 256, JSON.stringify(out));
+ assert.ok(
+ Math.max(out.width, out.height) <= limits.maxSide,
+ JSON.stringify(out),
+ );
+ assert.equal(out.width % limits.multiple, 0);
+ assert.equal(out.height % limits.multiple, 0);
+ }
+ }
+ assert.deepEqual(matchSourceSize(4096, 256, 512, LIMITS), {
+ width: 2752,
+ height: 256,
+ });
+});
+
+test("the grid and bounds follow the loaded model, and default to the historical ones", () => {
+ assert.deepEqual(sizeLimitsFrom(null), DEFAULT_SIZE_LIMITS);
+ assert.equal(snapDim(1040), 1040);
+ assert.equal(snapDim(1040, LIMITS), 1056);
+ assert.equal(snapDim(2752), 2048);
+ assert.equal(snapDim(2752, LIMITS), 2752);
+ const fitted = fitSize(2752, 2752, LIMITS);
+ assert.ok(fitted.width * fitted.height <= LIMITS.maxPixels);
+ assert.equal(fitted.width, fitted.height);
+ // A 2K recipe restores whole on this model and scaled on another.
+ assert.deepEqual(restorableSize(2752, 1536, "txt2img", LIMITS), {
+ width: 2752,
+ height: 1536,
+ });
+ const elsewhere = restorableSize(2752, 1536, "txt2img");
+ assert.ok(elsewhere.width <= 2048);
+});
+
+test("official 2K presets are offered only where they fit", () => {
+ assert.equal(presetsWithin(LIMITS).length, 7);
+ assert.equal(presetsWithin(DEFAULT_SIZE_LIMITS).length, 1); // only 2048 x 2048
+});
+
+test("the unified edit sends the size it shows", () => {
+ const source = { width: 1536, height: 1024 };
+ assert.deepEqual(
+ resolveEditSize(
+ "source",
+ source,
+ 1024,
+ { width: 512, height: 512 },
+ LIMITS,
+ ),
+ { width: 1248, height: 832 },
+ );
+ assert.deepEqual(
+ resolveEditSize(
+ "custom",
+ source,
+ 1024,
+ { width: 1040, height: 768 },
+ LIMITS,
+ ),
+ fitSize(1040, 768, LIMITS),
+ );
+});
+
+test("slots count the mask, and additional images are numbered after it", () => {
+ assert.equal(maxAdditionalImages(QWEN21, null), 9);
+ assert.equal(maxAdditionalImages(QWEN21, "mask"), 8);
+ assert.equal(maxAdditionalImages(null, null), 3);
+ assert.equal(additionalImageNumber(0, null), 2);
+ assert.equal(additionalImageNumber(0, "mask"), 3);
+});
+
+test("reference detail seeds from the build tier and never picks 2048 on its own", () => {
+ assert.equal(seedReferenceResolution([512, 1024, 2048], 512), 512);
+ assert.equal(seedReferenceResolution([512, 1024, 2048], 1024), 1024);
+ assert.equal(seedReferenceResolution([512, 1024, 2048], 768), 1024);
+ assert.equal(seedReferenceResolution([], 1024), null);
+});
+
+test("the request keeps order, drops empty slots and only sends what the model takes", () => {
+ const fields = conditionedRequestFields({
+ workflow: "edit",
+ initImage: "SRC",
+ extras: ["A", "", "B"],
+ referenceResolution: 2048,
+ conditioning: QWEN21,
+ localized: { mode: "mask", image: "M" },
+ });
+ assert.deepEqual(fields, {
+ workflow: "edit",
+ init_image: "SRC",
+ reference_images: ["A", "B"],
+ reference_resolution: 2048,
+ localized_edit: { mode: "mask", image: "M" },
+ });
+ const flux = conditionedRequestFields({
+ workflow: "reference",
+ initImage: "SRC",
+ extras: ["A"],
+ referenceResolution: 1024,
+ conditioning: { ...QWEN21, reference_resolutions: [], unified_edit: false },
+ localized: { mode: "paint", image: "P" },
+ });
+ assert.equal(flux.reference_resolution, undefined);
+ assert.equal(flux.localized_edit, undefined);
+ assert.deepEqual(flux.reference_images, ["A"]);
+});
+
+test("prompt helpers edit the visible instruction idempotently", () => {
+ const once = withTransparencyPrompt("A red apple.");
+ assert.match(
+ once,
+ /^This is an RGBA image with transparency\. A red apple\. The image has alpha channel/,
+ );
+ assert.equal(withTransparencyPrompt(once), once);
+ const marked = withLocalizedHint("Change the hair to black.", "annotate", [
+ "red",
+ "blue",
+ ]);
+ assert.match(
+ marked,
+ /Do not render the red and blue annotation lines in the image\.$/,
+ );
+ assert.equal(withLocalizedHint(marked, "annotate", ["red", "blue"]), marked);
+ assert.match(
+ withLocalizedHint("add a diver", "paint"),
+ /^In the area marked with white paint, add a diver$/,
+ );
+});
+
+test("Qwen-Image-2.1 samples at 40 steps without guidance under every artifact name", () => {
+ for (const id of [
+ "Qwen/Qwen-Image-2.1",
+ "unsloth/Qwen-Image-2.1-GGUF",
+ "unsloth/Qwen-Image-2.1-FP8",
+ ]) {
+ assert.deepEqual(defaultsFor(id), { steps: 40, guidance: 1 }, id);
+ }
+ assert.deepEqual(defaultsFor("Qwen/Qwen-Image-2512"), {
+ steps: 20,
+ guidance: 4,
+ });
+});
+
+test("a restored edit names every input it needs again", () => {
+ assert.equal(
+ restoreInputsNote({
+ workflow: "edit",
+ reference_image_count: 2,
+ localized_edit: "mask",
+ }),
+ "the source image, the mask and 2 additional images in the same order",
+ );
+ assert.equal(
+ restoreInputsNote({ workflow: "reference" }),
+ "the source image",
+ );
+ assert.equal(restoreInputsNote({ workflow: "img2img" }), null);
+});
+
+test("restoreSettings reopens Edit and Reference instead of Create", () => {
+ // Structural: the page component is not renderable under node:test. The workflow decision and
+ // the cleared uploads are what keep a restored edit from generating as text-to-image.
+ const src = readSrc("features/images/images-page.tsx");
+ const body = src.slice(src.indexOf("const restoreSettings = useCallback"));
+ assert.match(
+ body,
+ /image\.workflow === "edit" \? "edit" : image\.workflow === "reference" \? "reference" : "create"/,
+ );
+ assert.match(body, /setInitImage\(null\)/);
+ assert.match(body, /setReferenceImages\(/);
+ // Generation stays blocked until the source and every restored slot are supplied again.
+ assert.match(src, /usesInit && !initImage/);
+ assert.match(src, /is empty\. Add it, or remove that slot\./);
+});