Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Qwen-Image-Layered on the native sd.cpp engine (#12777)
Daniel Han55c79f19fa63
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex 415cfed2db5..eaa890ff5ff 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -401,6 +401,20 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (condition_image_mode = "RGBA",# same DiT as qwen-imagefp16_incompatible = True,+ # Native: its own 4-channel VAE (the RGB one cannot decode layers; pixel-identical to the ComfyUI repack) and+ # qwen-image's encoder, no projector (sd.cpp: enable_vision = version != VERSION_QWEN_IMAGE_LAYERED).+ sd_cpp_vae = ("Qwen/Qwen-Image-Layered", "vae/diffusion_pytorch_model.safetensors"),+ sd_cpp_text_encoders = (+ (+ "unsloth/Qwen2.5-VL-7B-Instruct-GGUF",+ "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf",+ "qwen2vl",+ ),+ ),+ sd_cpp_sampling_method = "euler",+ sd_cpp_flow_shift = 1.0,+ # Layered support landed upstream in master-744 (556f04b); an older reused build has no such literal.+ sd_cpp_arch_marker = "qwen_image_layers",),DiffusionFamily(name = "qwen-image",diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.pyindex d5968bf2278..9f226a873ae 100644--- a/studio/backend/core/inference/sd_cpp_args.py+++ b/studio/backend/core/inference/sd_cpp_args.py@@ -32,6 +32,7 @@ _TE_FLAGS_BY_FAMILY: dict[str, tuple[str, ...]] = {"flux.2-dev": ("--llm",),"qwen-image": ("--qwen2vl",),"qwen-image-edit": ("--qwen2vl",),+ "qwen-image-layered": ("--qwen2vl",),# 2.1 conditions on Qwen3-VL, which sd-cli takes through --llm. --qwen2vl is an alias of --llm# that also turns on Qwen2-VL's vision path, so it is the wrong door even though both names# reach the same field.@@ -94,6 +95,8 @@ class SdCppGenParams:ref_images: tuple[str, ...] = ()lora_dir: Optional[str] = Nonelora_apply_mode: Optional[str] = None+ # Qwen-Image-Layered layer count; sd.cpp returns layers + 1 images (first = input reconstruction). None = its default 3.+ qwen_image_layers: Optional[int] = None@dataclass(frozen = True)@@ -351,6 +354,8 @@ def build_sd_cpp_command(cmd += ["--mask", params.mask]for ref in params.ref_images:cmd += ["--ref-image", ref]+ if params.qwen_image_layers is not None:+ cmd += ["--qwen-image-layers", str(int(params.qwen_image_layers))]# LoRA: the directory to scan; individual LoRAs are <lora:name:w> tags in prompt.if params.lora_dir:cmd += ["--lora-model-dir", params.lora_dir]@@ -609,6 +614,7 @@ def build_img_gen_request(output_format: str = "png",lora: Optional[list[dict]] = None,ref_images: Optional[list[str]] = None,+ qwen_image_layers: Optional[int] = None,) -> dict:"""Build the ``POST /sdcpp/v1/img_gen`` JSON body for one text-to-image request.@@ -654,9 +660,23 @@ def build_img_gen_request(# Base64 PNGs in model order; no init_image/strength/mask: this is reference conditioning, not img2img.if ref_images:req["ref_images"] = list(ref_images)+ if qwen_image_layers is not None:+ req["qwen_image_layers"] = int(qwen_image_layers)return req+def sd_cli_output_paths(output_path: str, num_results: int) -> list[str]:+ """The files one ``sd-cli`` image run writes for ``--output output_path``: the path itself for a+ single result, else ``<stem>_<i><ext>`` from 0 (``examples/cli/main.cpp`` ``save_results``).+ A layered generation is the multi-result case: layers + 1 images from one run."""+ if num_results <= 1:+ return [output_path]+ stem, dot, ext = output_path.rpartition(".")+ if not dot or "/" in ext or "\\" in ext:+ stem, ext = output_path, "png"+ return [f"{stem}_{i}.{ext}" for i in range(num_results)]++def h3_server_eligible(params: "SdCppVideoGenParams") -> bool:"""False for reference videos / audio: path-only sd-cli flags with no ``vid_gen`` JSON field."""return not (params.ref_videos or params.ref_video_audios or params.ref_audios)diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.pyindex 52820d00749..004102922fc 100644--- a/studio/backend/core/inference/sd_cpp_backend.py+++ b/studio/backend/core/inference/sd_cpp_backend.py@@ -70,6 +70,7 @@ from core.inference.sd_cpp_args import (device_backend_flags,is_ggml_unsupported_op_abort,offload_flags,+ sd_cli_output_paths,without_device_backend_flags,)from core.inference.sd_cpp_engine import (@@ -1809,6 +1810,30 @@ def _native_output_image(fam: Any, im: Any) -> Any:return im.convert("RGB")+def _layer_count(fam: Any) -> int:+ return int(getattr(fam, "layer_count", 0) or 0)+++def _layered_canvas_size(fam: Any, size: tuple[int, int]) -> tuple[int, int]:+ """Copy of diffusion._layered_canvas (the pipeline's calculate_dimensions); this module never imports torch."""+ import math++ iw, ih = size+ area = float(getattr(fam, "layer_resolution", 640) or 640) ** 2+ ratio = float(iw) / float(max(1, ih))+ width = math.sqrt(area * ratio)+ height = width / ratio+ return max(32, int(round(width / 32)) * 32), max(32, int(round(height / 32)) * 32)+++def _keep_layers(items: list, layers: int) -> list:+ """Drop sd.cpp's input reconstruction (first of each layers + 1 group), as the diffusers pipeline does."""+ if not layers:+ return list(items)+ per = layers + 1+ return [item for i, item in enumerate(items) if i % per]++def _family_reads_vision(fam: Any) -> bool:"""Whether ``fam``'s native encoders include a vision projector (``llm_vision``)."""return any(kind == "llm_vision" for _r, _f, kind in getattr(fam, "sd_cpp_text_encoders", ()))@@ -1851,7 +1876,14 @@ def _native_condition_images()images = decode_condition_images(fam, init_image, reference_images, localized_edit)- if source_sized:+ if source_sized and _layer_count(fam):+ # Canvas from the 16 px snapped source, as the diffusers engine picks it.+ sw, sh = images[0].size+ snapped = (max(16, int(round(sw / 16)) * 16), max(16, int(round(sh / 16)) * 16))+ width, height = _layered_canvas_size(fam, snapped)+ if images[0].size != (width, height):+ images[0] = images[0].resize((width, height), Image.LANCZOS)+ elif source_sized:multiple = int(getattr(fam, "dimension_multiple", 16) or 16)sw, sh = images[0].sizemax_side = int(getattr(fam, "max_output_side", 2048) or 2048)@@ -3845,6 +3877,7 @@ class SdCppDiffusionBackend:lora_resolved = lora_resolved,cancel = cancel,ref_pngs = ref_pngs,+ layers = _layer_count(state.family) or None,)else:images, seeds = self._generate_oneshot(@@ -3861,6 +3894,7 @@ class SdCppDiffusionBackend:lora_resolved = lora_resolved,cancel = cancel,ref_pngs = ref_pngs,+ layers = _layer_count(state.family) or None,)except RuntimeError as exc:# The mid-render hipBLAS death the video path records too; not a cancellation.@@ -3952,6 +3986,7 @@ class SdCppDiffusionBackend:lora_resolved: list,cancel: threading.Event,ref_pngs: Optional[list[bytes]] = None,+ layers: Optional[int] = None,) -> tuple[list, list[int]]:"""Generate via the resident sd-server (no model reload).@@ -4024,6 +4059,7 @@ class SdCppDiffusionBackend:"data:image/png;base64," + base64.b64encode(b).decode("ascii")for b in ref_pngs or []],+ qwen_image_layers = layers,)try:blobs = state.server.img_gen(@@ -4048,16 +4084,21 @@ class SdCppDiffusionBackend:cancel_event = cancel,total_timeout = max(deadline - time.monotonic(), 1.0),)- # All-or-nothing per chunk: fail rather than silently drop images from the batch.- if not cancel.is_set() and len(blobs) != count:+ # All-or-nothing per chunk; a layered generation decodes layers + 1 images.+ expected = count * ((layers + 1) if layers else 1)+ if not cancel.is_set() and len(blobs) != expected:raise RuntimeError(- f"sd-server returned {len(blobs)} of {count} requested images in the batch."+ f"sd-server returned {len(blobs)} of {expected} requested images in the batch.")+ kept = _keep_layers(blobs, layers or 0)images.extend(- _native_output_image(state.family, Image.open(io.BytesIO(b))) for b in blobs+ _native_output_image(state.family, Image.open(io.BytesIO(b))) for b in kept+ )+ # sd.cpp advances the seed per generation; every layer of one carries that seed.+ per = len(kept) // count if count else 1+ seeds.extend(+ (chunk_seed + i // max(1, per)) & ((1 << 63) - 1) for i in range(len(kept)))- # sd.cpp advances the seed per image within a job, so report chunk_seed+i.- seeds.extend((chunk_seed + i) & ((1 << 63) - 1) for i in range(len(blobs)))finally:if lora_stage is not None:shutil.rmtree(lora_stage, ignore_errors = True)@@ -4139,6 +4180,7 @@ class SdCppDiffusionBackend:lora_resolved: list,cancel: threading.Event,ref_pngs: Optional[list[bytes]] = None,+ layers: Optional[int] = None,) -> tuple[list, list[int]]:"""Fallback path: re-run one-shot sd-cli per image (reloads the model each time). LoRA onthe one-shot path uses sd-cli's own mechanism: materialize the selected adapters into a@@ -4197,6 +4239,7 @@ class SdCppDiffusionBackend:lora_dir = lora_dir,lora_apply_mode = "auto" if lora_dir else None,ref_images = tuple(ref_paths),+ qwen_image_layers = layers,)# Each sd-cli run executes out of the managed tree, so hold installs off for its duration (and wait# here if one is already extracting). getattr: an INJECTED engine is the unit-test seam / escape hatch@@ -4234,9 +4277,11 @@ class SdCppDiffusionBackend:on_log = self._on_log,cancel_event = cancel,)- with Image.open(out_path) as im:- images.append(_native_output_image(state.family, im.copy()))- seeds.append(seed_i)+ outputs = sd_cli_output_paths(out_path, (layers + 1) if layers else 1)+ for path in _keep_layers(outputs, layers or 0):+ with Image.open(path) as im:+ images.append(_native_output_image(state.family, im.copy()))+ seeds.append(seed_i)return images, seedsdef _on_log(self, line: str) -> None:diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.pyindex aa2a0fd7e57..4d5169412c1 100644--- a/studio/backend/core/inference/sd_cpp_engine.py+++ b/studio/backend/core/inference/sd_cpp_engine.py@@ -44,6 +44,7 @@ from core.inference.sd_cpp_args import (build_sd_cpp_upscale_command,build_sd_cpp_video_command,native_speed_flags,+ sd_cli_output_paths,)logger = logging.getLogger(__name__)@@ -719,9 +720,13 @@ class SdCppEngine:verbose = verbose,extra_args = merged_extra,)+ # A layered run writes numbered files, never output_path itself.+ expected = output_path+ if params.qwen_image_layers is not None:+ expected = sd_cli_output_paths(str(output_path), int(params.qwen_image_layers) + 1)[0]return self._run(cmd,- output_path,+ expected,timeout = timeout,env = env,on_log = on_log,diff --git a/studio/backend/tests/test_sd_cpp_edit_families.py b/studio/backend/tests/test_sd_cpp_edit_families.pyindex 6b75ba5b89e..968fd642b5d 100644--- a/studio/backend/tests/test_sd_cpp_edit_families.py+++ b/studio/backend/tests/test_sd_cpp_edit_families.py@@ -128,8 +128,10 @@ def test_the_new_companions_stay_fetch_only_and_bases_stay_loadable():assert "black-forest-labs/flux.1-schnell" not in companions-def test_layered_and_inpaint_variants_are_still_refused():- assert detect_family("unsloth/Qwen-Image-Layered-GGUF") is None+def test_inpaint_and_other_layered_variants_are_still_refused():+ # Qwen-Image-Layered has a family of its own; a layered or inpaint variant of any other family does not.+ assert detect_family("unsloth/Qwen-Image-Layered-GGUF").name == "qwen-image-layered"+ assert detect_family("someone/FLUX.1-dev-layered-GGUF") is Noneassert detect_family("someone/FLUX.1-dev-inpaint-GGUF") is Nonediff --git a/studio/backend/tests/test_sd_cpp_qwen_layered.py b/studio/backend/tests/test_sd_cpp_qwen_layered.pynew file mode 100644index 00000000000..c85516ff05c--- /dev/null+++ b/studio/backend/tests/test_sd_cpp_qwen_layered.py@@ -0,0 +1,214 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Native (sd.cpp) route for Qwen-Image-Layered: the asset mapping, the layer count on the one-shot+argv and in the sd-server body, the canvas the input is decomposed at, and the layers + 1 images+sd.cpp decodes, of which the first (its reconstruction of the input) is dropped as the diffusers+pipeline drops it."""++from __future__ import annotations++import base64+import io+from pathlib import Path++import pytest+from PIL import Image++from core.inference import sd_cpp_backend as bk+from core.inference.diffusion_families import (+ detect_family,+ detect_family_for_pick,+ family_sd_cpp_supported,+)+from core.inference.sd_cpp_args import (+ SdCppGenParams,+ SdCppModelFiles,+ build_img_gen_request,+ build_sd_cpp_command,+ sd_cli_output_paths,+ text_encoder_flags_for_family,+)+from core.inference.sd_cpp_backend import SdCppDiffusionBackend++from .test_sd_cpp_backend import _FakeEngine, _FakeServer++LAYERED = detect_family("unsloth/Qwen-Image-Layered-GGUF")+FILES = SdCppModelFiles(+ diffusion_model = "/m/qwen-image-layered-Q4_K_M.gguf",+ vae = "/m/qwen_image_layered_vae.safetensors",+ qwen2vl = "/m/Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf",+)+++class _LayerServer(_FakeServer):+ """A server answering with the blobs a test set: sd.cpp's layers + 1 images per generation."""++ next_blobs: list = []++ def img_gen(self, payload, **kw):+ super().img_gen(payload, **kw)+ return list(self.next_blobs)+++def _png(size = (900, 600), color = (200, 10, 10)) -> str:+ buf = io.BytesIO()+ Image.new("RGB", size, color).save(buf, format = "PNG")+ return base64.b64encode(buf.getvalue()).decode()+++def _blob(color) -> bytes:+ buf = io.BytesIO()+ Image.new("RGBA", (4, 4), color).save(buf, format = "PNG")+ return buf.getvalue()+++def _state(*, mode = "oneshot", server = None):+ return bk._SdState(+ repo_id = "unsloth/Qwen-Image-Layered-GGUF",+ base_repo = LAYERED.base_repo,+ family = LAYERED,+ device = "cpu",+ files = FILES,+ sampling_method = LAYERED.sd_cpp_sampling_method,+ flow_shift = LAYERED.sd_cpp_flow_shift,+ mode = mode,+ server = server,+ )+++def test_layered_maps_to_its_own_vae_and_the_qwen_encoder():+ fam = detect_family_for_pick(+ "unsloth/Qwen-Image-Layered-GGUF", "qwen-image-layered-Q4_K_M.gguf"+ )+ assert fam is LAYERED and family_sd_cpp_supported(fam)+ specs = SdCppDiffusionBackend(engine = _FakeEngine())._asset_specs(+ "unsloth/Qwen-Image-Layered-GGUF", "qwen-image-layered-Q4_K_M.gguf", fam+ )+ # The vendor base's own VAE file (no community repack), the one the diffusers route fetches too.+ assert specs[1] == ("Qwen/Qwen-Image-Layered", "vae/diffusion_pytorch_model.safetensors", "vae")+ assert fam.sd_cpp_vae[0] == fam.base_repo+ # sd.cpp never runs the vision tower for this model, so no projector is fetched.+ assert [k for _r, _f, k in specs] == ["diffusion_model", "vae", "qwen2vl"]+ assert text_encoder_flags_for_family(fam.name) == ("--qwen2vl",)+ # The diffusers route's ComfyUI shift, so both engines sample the same schedule.+ assert (fam.sd_cpp_sampling_method, fam.sd_cpp_flow_shift) == ("euler", fam.comfy_flow_shift)+++def test_both_builders_carry_the_layer_count_only_when_set():+ params = SdCppGenParams(prompt = "p", width = 640, height = 640, ref_images = ("/r.png",))+ argv = build_sd_cpp_command("sd-cli", FILES, params, output_path = "/o.png")+ assert "--qwen-image-layers" not in argv+ layered = SdCppGenParams(+ prompt = "p", width = 640, height = 640, ref_images = ("/r.png",), qwen_image_layers = 2+ )+ argv = build_sd_cpp_command("sd-cli", FILES, layered, output_path = "/o.png")+ assert argv[argv.index("--qwen-image-layers") + 1] == "2"+ assert "qwen_image_layers" not in build_img_gen_request(prompt = "p")+ assert build_img_gen_request(prompt = "p", qwen_image_layers = 2)["qwen_image_layers"] == 2+++def test_sd_cli_output_paths_follow_its_naming():+ assert sd_cli_output_paths("/t/img_0.png", 1) == ["/t/img_0.png"]+ assert sd_cli_output_paths("/t/img_0.png", 3) == [+ "/t/img_0_0.png",+ "/t/img_0_1.png",+ "/t/img_0_2.png",+ ]+++def test_canvas_matches_the_diffusers_engine():+ from core.inference.diffusion import _layered_canvas+ for size in ((900, 600), (768, 768), (600, 1400), (1024, 1000)):+ assert bk._layered_canvas_size(LAYERED, size) == _layered_canvas(LAYERED, size), size+++def test_workflow_is_edit_only_and_ready_without_a_projector():+ b = SdCppDiffusionBackend(engine = _FakeEngine())+ b._state = _state()+ assert b.status()["workflows"] == ["edit"]+++def test_server_body_sends_the_layer_count_and_drops_the_reconstruction():+ server = _LayerServer("sd-server")+ layers = LAYERED.layer_count+ colors = [(i * 10, 0, 0, 255) for i in range(layers + 1)]+ server.next_blobs = [_blob(c) for c in colors]+ b = SdCppDiffusionBackend(engine = _FakeEngine())+ b._state = _state(mode = "server", server = server)+ out = b.generate(prompt = "p", steps = 20, guidance = 2.5, seed = 5, init_image = _png())+ body = server.payloads[-1]+ assert body["qwen_image_layers"] == layers+ assert (body["width"], body["height"]) == bk._layered_canvas_size(LAYERED, (900, 600))+ assert body["sample_params"]["guidance"] == {"txt_cfg": 2.5}+ assert body["sample_params"]["flow_shift"] == LAYERED.sd_cpp_flow_shift+ ref = Image.open(io.BytesIO(base64.b64decode(body["ref_images"][0].split(",", 1)[1])))+ assert ref.size == (body["width"], body["height"])+ assert [im.getpixel((0, 0)) for im in out["images"]] == colors[1:]+ assert all(im.mode == "RGBA" for im in out["images"])+ assert out["seeds"] == [5] * layers and out["workflow"] == "edit"+++def test_server_refuses_a_short_layer_set():+ server = _LayerServer("sd-server")+ server.next_blobs = [_blob((1, 1, 1, 255))]+ b = SdCppDiffusionBackend(engine = _FakeEngine())+ b._state = _state(mode = "server", server = server)+ with pytest.raises(RuntimeError, match = r"returned 1 of \d+ requested images"):+ b.generate(prompt = "p", steps = 2, init_image = _png())+++def test_oneshot_collects_every_numbered_layer_file(monkeypatch):+ engine = _FakeEngine()+ layers = LAYERED.layer_count+ colors = [(0, i * 10, 0, 255) for i in range(layers + 1)]+ seen = {}++ def _write(files, params, output_path, **kw):+ seen["argv"] = build_sd_cpp_command("sd-cli", files, params, output_path = output_path)+ for path, color in zip(sd_cli_output_paths(output_path, layers + 1), colors):+ Path(path).write_bytes(_blob(color))+ return Path(output_path)++ monkeypatch.setattr(engine, "generate", _write)+ b = SdCppDiffusionBackend(engine = engine)+ b._state = _state()+ out = b.generate(prompt = "p", steps = 20, guidance = 2.5, seed = 9, init_image = _png((768, 768)))+ argv = seen["argv"]+ assert argv[argv.index("--qwen-image-layers") + 1] == str(layers)+ assert argv.count("--ref-image") == 1 and argv[argv.index("--width") + 1] == "640"+ assert [im.getpixel((0, 0)) for im in out["images"]] == colors[1:]+ assert out["seeds"] == [9] * layers++[email protected]("size", [(500, 521), (770, 500), (901, 603), (768, 768)])+def test_off_grid_source_gets_the_same_canvas_as_the_diffusers_engine(size):+ # The diffusers engine snaps the source to 16 px before the pipeline picks its canvas (500x521 -> 608x672, not+ # the 640x640 the raw size gives), so the native engine must pick from the snapped size too.+ from core.inference.diffusion import _layered_canvas, _snap_to_multiple++ buf = io.BytesIO()+ Image.new("RGBA", size, (10, 20, 30, 255)).save(buf, format = "PNG")+ src = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()+ w, h, _blobs = bk._native_condition_images(+ LAYERED,+ src,+ None,+ None,+ None,+ None,+ full_fidelity = True,+ pad_to_output = False,+ source_sized = True,+ )+ assert (w, h) == _layered_canvas(LAYERED, _snap_to_multiple(Image.new("RGB", size), 16).size)+++def test_layered_needs_a_build_that_carries_layered_support(tmp_path):+ # Layered landed upstream in master-744; an older reused build must not be picked for it.+ old = tmp_path / "sd-cli-old"+ old.write_bytes(b"stable-diffusion.cpp qwen_image_2_1 --ref-image")+ new = tmp_path / "sd-cli-new"+ new.write_bytes(b"stable-diffusion.cpp --qwen-image-layers qwen_image_layers")+ assert not bk.sd_cpp_binary_runs_family(str(old), LAYERED)+ assert bk.sd_cpp_binary_runs_family(str(new), LAYERED)