Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Reference/edit image modes (annotate, paint, mask) are generative guides, not inpainting masks."""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Any, Optional, Sequence
LOCALIZED_EDIT_MODES = ("annotate", "paint", "mask")
# Decoded pixels across ALL condition images of one call: ten 4096 px images would be over 600 MB of RGBA.
MAX_CONDITION_SOURCE_PIXELS = 64_000_000
MIN_OUTPUT_SIDE = 256
# Largest family max_output_side; per-family bounds live in check_output_size.
MAX_REQUEST_SIDE = 6272
# Upstream QwenImage21Pipeline's own ``output_resolution`` default.
DEFAULT_REFERENCE_RESOLUTION = 1024
@dataclass(frozen = True)
class LocalizedEdit:
mode: str
image: str
def check_conditioned_fields(
workflow: Optional[str],
fam: Any,
*,
mask_image: Any = None,
strength: Optional[float] = None,
upscale: Optional[float] = None,
controlnet: Any = None,
localized_edit: Optional[LocalizedEdit] = None,
) -> None:
"""Refuse request fields an explicit ``workflow`` would drop, on either engine."""
if workflow is not None:
# One pipeline call over the ordered images would drop these, so refuse rather than ignore them.
for present, name in (
(mask_image is not None, "mask_image"),
(strength is not None, "strength"),
(upscale is not None and upscale > 1.0, "upscale"),
(controlnet is not None and controlnet[3] not in (None, 0, 0.0), "controlnet"),
):
if present:
raise ValueError(f"{name} is not supported by the {workflow} workflow.")
if localized_edit is not None and not (
workflow == "edit" and getattr(fam, "unified_edit", False)
):
raise ValueError("localized_edit needs the edit workflow on a model that supports it.")
def check_output_size(fam: Any, width: int, height: int) -> None:
"""Refuse an output size the loaded family cannot render as asked. A size the pipeline would
silently floor to its own grid is refused rather than changed behind the caller's back."""
multiple = int(getattr(fam, "dimension_multiple", 16) or 16)
max_side = int(getattr(fam, "max_output_side", 2048) or 2048)
max_pixels = int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048)
name = getattr(fam, "name", "this")
if min(width, height) < MIN_OUTPUT_SIDE:
raise ValueError(
f"Width and height must be at least {MIN_OUTPUT_SIDE}px (got {width}x{height})."
)
if width % multiple or height % multiple:
raise ValueError(
f"The {name} model needs width and height in multiples of {multiple} "
f"(got {width}x{height})."
)
if max(width, height) > max_side:
raise ValueError(
f"The {name} model generates at most {max_side}px per side (got {width}x{height})."
)
if width * height > max_pixels:
raise ValueError(
f"The {name} model generates at most {max_pixels:,} pixels "
f"(got {width}x{height} = {width * height:,})."
)
def match_source_size(fam: Any, source_size: tuple[int, int], resolution: int) -> tuple[int, int]:
"""Source aspect ratio at the target area, snapped to the family grid and bounds, from Image 1."""
multiple = int(getattr(fam, "dimension_multiple", 16) or 16)
max_side = int(getattr(fam, "max_output_side", 2048) or 2048)
max_pixels = int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048)
sw, sh = source_size
ratio = max(1e-6, float(sw) / float(max(1, sh)))
area = float(resolution) * float(resolution)
for _ in range(64):
w = max(multiple, int(round(math.sqrt(area * ratio) / multiple)) * multiple)
h = max(multiple, int(round(math.sqrt(area / ratio) / multiple)) * multiple)
if max(w, h) <= max_side and w * h <= max_pixels:
break
area *= 0.9
short_min = -(-MIN_OUTPUT_SIDE // multiple) * multiple
long_max = max_side // multiple * multiple
if min(w, h) < short_min:
long_side = int(round(short_min * max(ratio, 1.0 / ratio) / multiple)) * multiple
long_side = min(max(long_side, short_min), long_max)
while long_side > short_min and long_side * short_min > max_pixels:
long_side -= multiple
w, h = (long_side, short_min) if ratio >= 1.0 else (short_min, long_side)
return w, h
def effective_reference_resolution(fam: Any, requested: Optional[int]) -> Optional[int]:
"""The condition-image preprocessing resolution this call uses, or None for a family without
the control. Refuses a value the family does not list rather than rounding it."""
allowed = tuple(getattr(fam, "reference_resolutions", ()) or ())
if not allowed:
if requested is not None:
raise ValueError(
f"reference_resolution is not supported for the '{getattr(fam, 'name', '')}' "
"model family."
)
return None
if requested is None:
return (
DEFAULT_REFERENCE_RESOLUTION if DEFAULT_REFERENCE_RESOLUTION in allowed else allowed[0]
)
if requested not in allowed:
raise ValueError(
"reference_resolution must be one of "
+ ", ".join(str(v) for v in allowed)
+ f" for this model (got {requested})."
)
return int(requested)
def reference_request_pixels(ref_resolution: Optional[int], extra_images: int = 0) -> int:
"""Shared by the load's edit placement and the render-time guard, so both price the same pixels."""
if not ref_resolution:
return 0
return int((1 + max(0, int(extra_images))) * int(ref_resolution) * int(ref_resolution))
def default_reference_pixels(fam: Any) -> int:
return reference_request_pixels(effective_reference_resolution(fam, None))
def _decode_bounded(data: str, mode: str, budget: list[int], what: str) -> Any:
from core.inference.diffusion import decode_b64_image
try:
img = decode_b64_image(data, mode = mode, max_pixels = max(1, budget[0]))
except ValueError as exc:
if "source pixels" in str(exc):
raise ValueError(
f"The input images are too large together: {what} exceeds the "
f"{MAX_CONDITION_SOURCE_PIXELS:,} pixel total for one request. Use smaller images."
) from exc
raise ValueError(f"{what}: {exc}") from exc
budget[0] -= img.width * img.height
return img
def _to_source_geometry(img: Any, source: Any, what: str, resample: Any) -> Any:
"""``img`` at the source's pixel size. Only a pure scale is accepted: a different aspect ratio
means the layer was drawn over another image."""
if img.size == source.size:
return img
sw, sh = source.size
iw, ih = img.size
if abs(iw / float(ih) - sw / float(sh)) > 0.02 * (sw / float(sh)):
raise ValueError(
f"The {what} is {iw}x{ih} but the source image is {sw}x{sh}; it must be drawn over "
"the same image."
)
return img.resize(source.size, resample)
def apply_localized_edit(source: Any, localized: LocalizedEdit, budget: list[int]) -> list[Any]:
"""The images the localized edit contributes, source first: ``[marked_source]`` for annotate
and paint, ``[source, mask]`` for a separate mask. Alpha outside the marks is preserved."""
from PIL import Image
if localized.mode == "annotate":
overlay = _decode_bounded(localized.image, "RGBA", budget, "The annotation layer")
overlay = _to_source_geometry(overlay, source, "annotation layer", Image.LANCZOS)
base = source.convert("RGBA")
marked = Image.alpha_composite(base, overlay)
return [marked if source.mode == "RGBA" else marked.convert(source.mode)]
if localized.mode == "paint":
mask = _decode_bounded(localized.image, "L", budget, "The painted region")
mask = _to_source_geometry(mask, source, "painted region", Image.LANCZOS)
base = source.convert("RGBA")
white = Image.new("RGBA", base.size, (255, 255, 255, 255))
painted = Image.composite(white, base, mask)
return [painted if source.mode == "RGBA" else painted.convert(source.mode)]
if localized.mode == "mask":
mask = _decode_bounded(localized.image, "L", budget, "The mask")
# NEAREST keeps it binary; the threshold then fixes the polarity white = region.
mask = _to_source_geometry(mask, source, "mask", Image.NEAREST)
mask = mask.point(lambda v: 255 if v >= 128 else 0)
if not mask.getbbox():
raise ValueError("The mask is empty: paint the region to edit in white.")
return [source, mask.convert("RGB")]
raise ValueError("localized_edit mode must be one of " + ", ".join(LOCALIZED_EDIT_MODES) + ".")
def decode_condition_images(
fam: Any,
init_image: str,
reference_images: Optional[Sequence[str]],
localized: Optional[LocalizedEdit] = None,
) -> list[Any]:
"""The ordered condition list: the (possibly marked) source, a separate mask if any, then every
reference in request order. More images than the family takes is refused, not truncated."""
limit = int(getattr(fam, "max_condition_images", 4) or 4)
mode = getattr(fam, "condition_image_mode", "RGB") or "RGB"
refs = [r for r in (reference_images or []) if r]
extra_from_mask = 1 if localized is not None and localized.mode == "mask" else 0
total = 1 + extra_from_mask + len(refs)
if total > limit:
raise ValueError(
f"The {getattr(fam, 'name', 'loaded')} model takes at most {limit} input images in "
f"total (got {total}" + (", counting the mask" if extra_from_mask else "") + ")."
)
budget = [MAX_CONDITION_SOURCE_PIXELS]
source = _decode_bounded(init_image, mode, budget, "The source image")
images = [source]
if localized is not None:
images = apply_localized_edit(source, localized, budget)
for ref in refs:
images.append(_decode_bounded(ref, mode, budget, f"Image {len(images) + 1}"))
return images
# QwenImageEditPlusPipeline's VAE_IMAGE_SIZE: each image of a multi-image edit is VAE-encoded at this area (pixels).
EDIT_EXTRA_IMAGE_AREA = 1024 * 1024
def edit_takes_extra_images(fam: Any) -> bool:
"""An instruction-edit family that reads more than the source in one edit (Qwen-Image-Edit-Plus): its edit
workflow takes ``reference_images`` after ``init_image``, up to ``max_condition_images`` in total."""
return (
bool(getattr(fam, "edit", False)) and int(getattr(fam, "max_condition_images", 4) or 4) > 1
)
def conditioning_capabilities(fam: Any, workflows: Sequence[str]) -> dict[str, Any]:
"""What the UI may offer for ``fam`` given the ``workflows`` the active engine runs for it.
Localized editing is advertised only where the unified edit workflow itself is."""
localized = bool(getattr(fam, "unified_edit", False)) and "edit" in workflows
# The page's unified Edit form (Image 1 + extras, output sized from Image 1) fits a reference-edit family too.
unified = localized or (bool(getattr(fam, "reference_edit", False)) and "edit" in workflows)
return {
"max_condition_images": int(getattr(fam, "max_condition_images", 4) or 4),
"alpha": (getattr(fam, "condition_image_mode", "RGB") or "RGB") == "RGBA",
"dimension_multiple": int(getattr(fam, "dimension_multiple", 16) or 16),
"max_output_side": int(getattr(fam, "max_output_side", 2048) or 2048),
"max_output_pixels": int(getattr(fam, "max_output_pixels", 2048 * 2048) or 2048 * 2048),
"reference_resolutions": list(getattr(fam, "reference_resolutions", ()) or ()),
"unified_edit": unified,
# Edit-only families whose edit takes the extra images too (the UI shows them on the Edit tab).
"multi_image_edit": edit_takes_extra_images(fam) and "edit" in workflows,
"localized_edit_modes": list(LOCALIZED_EDIT_MODES) if localized else [],
}