Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Qwen-Image-Layered on the native sd.cpp engine (#12777)

Daniel Han55c79f19fa63
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 415cfed2db5..eaa890ff5ff 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -401,6 +401,20 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
condition_image_mode = "RGBA",
# same DiT as qwen-image
fp16_incompatible = True,
+ # Native: its own 4-channel VAE (the RGB one cannot decode layers; pixel-identical to the ComfyUI repack) and
+ # qwen-image's encoder, no projector (sd.cpp: enable_vision = version != VERSION_QWEN_IMAGE_LAYERED).
+ sd_cpp_vae = ("Qwen/Qwen-Image-Layered", "vae/diffusion_pytorch_model.safetensors"),
+ sd_cpp_text_encoders = (
+ (
+ "unsloth/Qwen2.5-VL-7B-Instruct-GGUF",
+ "Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf",
+ "qwen2vl",
+ ),
+ ),
+ sd_cpp_sampling_method = "euler",
+ sd_cpp_flow_shift = 1.0,
+ # Layered support landed upstream in master-744 (556f04b); an older reused build has no such literal.
+ sd_cpp_arch_marker = "qwen_image_layers",
),
DiffusionFamily(
name = "qwen-image",
diff --git a/studio/backend/core/inference/sd_cpp_args.py b/studio/backend/core/inference/sd_cpp_args.py
index d5968bf2278..9f226a873ae 100644
--- a/studio/backend/core/inference/sd_cpp_args.py
+++ b/studio/backend/core/inference/sd_cpp_args.py
@@ -32,6 +32,7 @@ _TE_FLAGS_BY_FAMILY: dict[str, tuple[str, ...]] = {
"flux.2-dev": ("--llm",),
"qwen-image": ("--qwen2vl",),
"qwen-image-edit": ("--qwen2vl",),
+ "qwen-image-layered": ("--qwen2vl",),
# 2.1 conditions on Qwen3-VL, which sd-cli takes through --llm. --qwen2vl is an alias of --llm
# that also turns on Qwen2-VL's vision path, so it is the wrong door even though both names
# reach the same field.
@@ -94,6 +95,8 @@ class SdCppGenParams:
ref_images: tuple[str, ...] = ()
lora_dir: Optional[str] = None
lora_apply_mode: Optional[str] = None
+ # Qwen-Image-Layered layer count; sd.cpp returns layers + 1 images (first = input reconstruction). None = its default 3.
+ qwen_image_layers: Optional[int] = None
@dataclass(frozen = True)
@@ -351,6 +354,8 @@ def build_sd_cpp_command(
cmd += ["--mask", params.mask]
for ref in params.ref_images:
cmd += ["--ref-image", ref]
+ if params.qwen_image_layers is not None:
+ cmd += ["--qwen-image-layers", str(int(params.qwen_image_layers))]
# LoRA: the directory to scan; individual LoRAs are <lora:name:w> tags in prompt.
if params.lora_dir:
cmd += ["--lora-model-dir", params.lora_dir]
@@ -609,6 +614,7 @@ def build_img_gen_request(
output_format: str = "png",
lora: Optional[list[dict]] = None,
ref_images: Optional[list[str]] = None,
+ qwen_image_layers: Optional[int] = None,
) -> dict:
"""Build the ``POST /sdcpp/v1/img_gen`` JSON body for one text-to-image request.
@@ -654,9 +660,23 @@ def build_img_gen_request(
# Base64 PNGs in model order; no init_image/strength/mask: this is reference conditioning, not img2img.
if ref_images:
req["ref_images"] = list(ref_images)
+ if qwen_image_layers is not None:
+ req["qwen_image_layers"] = int(qwen_image_layers)
return req
+def sd_cli_output_paths(output_path: str, num_results: int) -> list[str]:
+ """The files one ``sd-cli`` image run writes for ``--output output_path``: the path itself for a
+ single result, else ``<stem>_<i><ext>`` from 0 (``examples/cli/main.cpp`` ``save_results``).
+ A layered generation is the multi-result case: layers + 1 images from one run."""
+ if num_results <= 1:
+ return [output_path]
+ stem, dot, ext = output_path.rpartition(".")
+ if not dot or "/" in ext or "\\" in ext:
+ stem, ext = output_path, "png"
+ return [f"{stem}_{i}.{ext}" for i in range(num_results)]
+
+
def h3_server_eligible(params: "SdCppVideoGenParams") -> bool:
"""False for reference videos / audio: path-only sd-cli flags with no ``vid_gen`` JSON field."""
return not (params.ref_videos or params.ref_video_audios or params.ref_audios)
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index 52820d00749..004102922fc 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -70,6 +70,7 @@ from core.inference.sd_cpp_args import (
device_backend_flags,
is_ggml_unsupported_op_abort,
offload_flags,
+ sd_cli_output_paths,
without_device_backend_flags,
)
from core.inference.sd_cpp_engine import (
@@ -1809,6 +1810,30 @@ def _native_output_image(fam: Any, im: Any) -> Any:
return im.convert("RGB")
+def _layer_count(fam: Any) -> int:
+ return int(getattr(fam, "layer_count", 0) or 0)
+
+
+def _layered_canvas_size(fam: Any, size: tuple[int, int]) -> tuple[int, int]:
+ """Copy of diffusion._layered_canvas (the pipeline's calculate_dimensions); this module never imports torch."""
+ import math
+
+ iw, ih = size
+ area = float(getattr(fam, "layer_resolution", 640) or 640) ** 2
+ ratio = float(iw) / float(max(1, ih))
+ width = math.sqrt(area * ratio)
+ height = width / ratio
+ return max(32, int(round(width / 32)) * 32), max(32, int(round(height / 32)) * 32)
+
+
+def _keep_layers(items: list, layers: int) -> list:
+ """Drop sd.cpp's input reconstruction (first of each layers + 1 group), as the diffusers pipeline does."""
+ if not layers:
+ return list(items)
+ per = layers + 1
+ return [item for i, item in enumerate(items) if i % per]
+
+
def _family_reads_vision(fam: Any) -> bool:
"""Whether ``fam``'s native encoders include a vision projector (``llm_vision``)."""
return any(kind == "llm_vision" for _r, _f, kind in getattr(fam, "sd_cpp_text_encoders", ()))
@@ -1851,7 +1876,14 @@ def _native_condition_images(
)
images = decode_condition_images(fam, init_image, reference_images, localized_edit)
- if source_sized:
+ if source_sized and _layer_count(fam):
+ # Canvas from the 16 px snapped source, as the diffusers engine picks it.
+ sw, sh = images[0].size
+ snapped = (max(16, int(round(sw / 16)) * 16), max(16, int(round(sh / 16)) * 16))
+ width, height = _layered_canvas_size(fam, snapped)
+ if images[0].size != (width, height):
+ images[0] = images[0].resize((width, height), Image.LANCZOS)
+ elif source_sized:
multiple = int(getattr(fam, "dimension_multiple", 16) or 16)
sw, sh = images[0].size
max_side = int(getattr(fam, "max_output_side", 2048) or 2048)
@@ -3845,6 +3877,7 @@ class SdCppDiffusionBackend:
lora_resolved = lora_resolved,
cancel = cancel,
ref_pngs = ref_pngs,
+ layers = _layer_count(state.family) or None,
)
else:
images, seeds = self._generate_oneshot(
@@ -3861,6 +3894,7 @@ class SdCppDiffusionBackend:
lora_resolved = lora_resolved,
cancel = cancel,
ref_pngs = ref_pngs,
+ layers = _layer_count(state.family) or None,
)
except RuntimeError as exc:
# The mid-render hipBLAS death the video path records too; not a cancellation.
@@ -3952,6 +3986,7 @@ class SdCppDiffusionBackend:
lora_resolved: list,
cancel: threading.Event,
ref_pngs: Optional[list[bytes]] = None,
+ layers: Optional[int] = None,
) -> tuple[list, list[int]]:
"""Generate via the resident sd-server (no model reload).
@@ -4024,6 +4059,7 @@ class SdCppDiffusionBackend:
"data:image/png;base64," + base64.b64encode(b).decode("ascii")
for b in ref_pngs or []
],
+ qwen_image_layers = layers,
)
try:
blobs = state.server.img_gen(
@@ -4048,16 +4084,21 @@ class SdCppDiffusionBackend:
cancel_event = cancel,
total_timeout = max(deadline - time.monotonic(), 1.0),
)
- # All-or-nothing per chunk: fail rather than silently drop images from the batch.
- if not cancel.is_set() and len(blobs) != count:
+ # All-or-nothing per chunk; a layered generation decodes layers + 1 images.
+ expected = count * ((layers + 1) if layers else 1)
+ if not cancel.is_set() and len(blobs) != expected:
raise RuntimeError(
- f"sd-server returned {len(blobs)} of {count} requested images in the batch."
+ f"sd-server returned {len(blobs)} of {expected} requested images in the batch."
)
+ kept = _keep_layers(blobs, layers or 0)
images.extend(
- _native_output_image(state.family, Image.open(io.BytesIO(b))) for b in blobs
+ _native_output_image(state.family, Image.open(io.BytesIO(b))) for b in kept
+ )
+ # sd.cpp advances the seed per generation; every layer of one carries that seed.
+ per = len(kept) // count if count else 1
+ seeds.extend(
+ (chunk_seed + i // max(1, per)) & ((1 << 63) - 1) for i in range(len(kept))
)
- # sd.cpp advances the seed per image within a job, so report chunk_seed+i.
- seeds.extend((chunk_seed + i) & ((1 << 63) - 1) for i in range(len(blobs)))
finally:
if lora_stage is not None:
shutil.rmtree(lora_stage, ignore_errors = True)
@@ -4139,6 +4180,7 @@ class SdCppDiffusionBackend:
lora_resolved: list,
cancel: threading.Event,
ref_pngs: Optional[list[bytes]] = None,
+ layers: Optional[int] = None,
) -> tuple[list, list[int]]:
"""Fallback path: re-run one-shot sd-cli per image (reloads the model each time). LoRA on
the one-shot path uses sd-cli's own mechanism: materialize the selected adapters into a
@@ -4197,6 +4239,7 @@ class SdCppDiffusionBackend:
lora_dir = lora_dir,
lora_apply_mode = "auto" if lora_dir else None,
ref_images = tuple(ref_paths),
+ qwen_image_layers = layers,
)
# Each sd-cli run executes out of the managed tree, so hold installs off for its duration (and wait
# here if one is already extracting). getattr: an INJECTED engine is the unit-test seam / escape hatch
@@ -4234,9 +4277,11 @@ class SdCppDiffusionBackend:
on_log = self._on_log,
cancel_event = cancel,
)
- with Image.open(out_path) as im:
- images.append(_native_output_image(state.family, im.copy()))
- seeds.append(seed_i)
+ outputs = sd_cli_output_paths(out_path, (layers + 1) if layers else 1)
+ for path in _keep_layers(outputs, layers or 0):
+ with Image.open(path) as im:
+ images.append(_native_output_image(state.family, im.copy()))
+ seeds.append(seed_i)
return images, seeds
def _on_log(self, line: str) -> None:
diff --git a/studio/backend/core/inference/sd_cpp_engine.py b/studio/backend/core/inference/sd_cpp_engine.py
index aa2a0fd7e57..4d5169412c1 100644
--- a/studio/backend/core/inference/sd_cpp_engine.py
+++ b/studio/backend/core/inference/sd_cpp_engine.py
@@ -44,6 +44,7 @@ from core.inference.sd_cpp_args import (
build_sd_cpp_upscale_command,
build_sd_cpp_video_command,
native_speed_flags,
+ sd_cli_output_paths,
)
logger = logging.getLogger(__name__)
@@ -719,9 +720,13 @@ class SdCppEngine:
verbose = verbose,
extra_args = merged_extra,
)
+ # A layered run writes numbered files, never output_path itself.
+ expected = output_path
+ if params.qwen_image_layers is not None:
+ expected = sd_cli_output_paths(str(output_path), int(params.qwen_image_layers) + 1)[0]
return self._run(
cmd,
- output_path,
+ expected,
timeout = timeout,
env = env,
on_log = on_log,
diff --git a/studio/backend/tests/test_sd_cpp_edit_families.py b/studio/backend/tests/test_sd_cpp_edit_families.py
index 6b75ba5b89e..968fd642b5d 100644
--- a/studio/backend/tests/test_sd_cpp_edit_families.py
+++ b/studio/backend/tests/test_sd_cpp_edit_families.py
@@ -128,8 +128,10 @@ def test_the_new_companions_stay_fetch_only_and_bases_stay_loadable():
assert "black-forest-labs/flux.1-schnell" not in companions
-def test_layered_and_inpaint_variants_are_still_refused():
- assert detect_family("unsloth/Qwen-Image-Layered-GGUF") is None
+def test_inpaint_and_other_layered_variants_are_still_refused():
+ # Qwen-Image-Layered has a family of its own; a layered or inpaint variant of any other family does not.
+ assert detect_family("unsloth/Qwen-Image-Layered-GGUF").name == "qwen-image-layered"
+ assert detect_family("someone/FLUX.1-dev-layered-GGUF") is None
assert detect_family("someone/FLUX.1-dev-inpaint-GGUF") is None
diff --git a/studio/backend/tests/test_sd_cpp_qwen_layered.py b/studio/backend/tests/test_sd_cpp_qwen_layered.py
new file mode 100644
index 00000000000..c85516ff05c
--- /dev/null
+++ b/studio/backend/tests/test_sd_cpp_qwen_layered.py
@@ -0,0 +1,214 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Native (sd.cpp) route for Qwen-Image-Layered: the asset mapping, the layer count on the one-shot
+argv and in the sd-server body, the canvas the input is decomposed at, and the layers + 1 images
+sd.cpp decodes, of which the first (its reconstruction of the input) is dropped as the diffusers
+pipeline drops it."""
+
+from __future__ import annotations
+
+import base64
+import io
+from pathlib import Path
+
+import pytest
+from PIL import Image
+
+from core.inference import sd_cpp_backend as bk
+from core.inference.diffusion_families import (
+ detect_family,
+ detect_family_for_pick,
+ family_sd_cpp_supported,
+)
+from core.inference.sd_cpp_args import (
+ SdCppGenParams,
+ SdCppModelFiles,
+ build_img_gen_request,
+ build_sd_cpp_command,
+ sd_cli_output_paths,
+ text_encoder_flags_for_family,
+)
+from core.inference.sd_cpp_backend import SdCppDiffusionBackend
+
+from .test_sd_cpp_backend import _FakeEngine, _FakeServer
+
+LAYERED = detect_family("unsloth/Qwen-Image-Layered-GGUF")
+FILES = SdCppModelFiles(
+ diffusion_model = "/m/qwen-image-layered-Q4_K_M.gguf",
+ vae = "/m/qwen_image_layered_vae.safetensors",
+ qwen2vl = "/m/Qwen2.5-VL-7B-Instruct-Q4_K_M.gguf",
+)
+
+
+class _LayerServer(_FakeServer):
+ """A server answering with the blobs a test set: sd.cpp's layers + 1 images per generation."""
+
+ next_blobs: list = []
+
+ def img_gen(self, payload, **kw):
+ super().img_gen(payload, **kw)
+ return list(self.next_blobs)
+
+
+def _png(size = (900, 600), color = (200, 10, 10)) -> str:
+ buf = io.BytesIO()
+ Image.new("RGB", size, color).save(buf, format = "PNG")
+ return base64.b64encode(buf.getvalue()).decode()
+
+
+def _blob(color) -> bytes:
+ buf = io.BytesIO()
+ Image.new("RGBA", (4, 4), color).save(buf, format = "PNG")
+ return buf.getvalue()
+
+
+def _state(*, mode = "oneshot", server = None):
+ return bk._SdState(
+ repo_id = "unsloth/Qwen-Image-Layered-GGUF",
+ base_repo = LAYERED.base_repo,
+ family = LAYERED,
+ device = "cpu",
+ files = FILES,
+ sampling_method = LAYERED.sd_cpp_sampling_method,
+ flow_shift = LAYERED.sd_cpp_flow_shift,
+ mode = mode,
+ server = server,
+ )
+
+
+def test_layered_maps_to_its_own_vae_and_the_qwen_encoder():
+ fam = detect_family_for_pick(
+ "unsloth/Qwen-Image-Layered-GGUF", "qwen-image-layered-Q4_K_M.gguf"
+ )
+ assert fam is LAYERED and family_sd_cpp_supported(fam)
+ specs = SdCppDiffusionBackend(engine = _FakeEngine())._asset_specs(
+ "unsloth/Qwen-Image-Layered-GGUF", "qwen-image-layered-Q4_K_M.gguf", fam
+ )
+ # The vendor base's own VAE file (no community repack), the one the diffusers route fetches too.
+ assert specs[1] == ("Qwen/Qwen-Image-Layered", "vae/diffusion_pytorch_model.safetensors", "vae")
+ assert fam.sd_cpp_vae[0] == fam.base_repo
+ # sd.cpp never runs the vision tower for this model, so no projector is fetched.
+ assert [k for _r, _f, k in specs] == ["diffusion_model", "vae", "qwen2vl"]
+ assert text_encoder_flags_for_family(fam.name) == ("--qwen2vl",)
+ # The diffusers route's ComfyUI shift, so both engines sample the same schedule.
+ assert (fam.sd_cpp_sampling_method, fam.sd_cpp_flow_shift) == ("euler", fam.comfy_flow_shift)
+
+
+def test_both_builders_carry_the_layer_count_only_when_set():
+ params = SdCppGenParams(prompt = "p", width = 640, height = 640, ref_images = ("/r.png",))
+ argv = build_sd_cpp_command("sd-cli", FILES, params, output_path = "/o.png")
+ assert "--qwen-image-layers" not in argv
+ layered = SdCppGenParams(
+ prompt = "p", width = 640, height = 640, ref_images = ("/r.png",), qwen_image_layers = 2
+ )
+ argv = build_sd_cpp_command("sd-cli", FILES, layered, output_path = "/o.png")
+ assert argv[argv.index("--qwen-image-layers") + 1] == "2"
+ assert "qwen_image_layers" not in build_img_gen_request(prompt = "p")
+ assert build_img_gen_request(prompt = "p", qwen_image_layers = 2)["qwen_image_layers"] == 2
+
+
+def test_sd_cli_output_paths_follow_its_naming():
+ assert sd_cli_output_paths("/t/img_0.png", 1) == ["/t/img_0.png"]
+ assert sd_cli_output_paths("/t/img_0.png", 3) == [
+ "/t/img_0_0.png",
+ "/t/img_0_1.png",
+ "/t/img_0_2.png",
+ ]
+
+
+def test_canvas_matches_the_diffusers_engine():
+ from core.inference.diffusion import _layered_canvas
+ for size in ((900, 600), (768, 768), (600, 1400), (1024, 1000)):
+ assert bk._layered_canvas_size(LAYERED, size) == _layered_canvas(LAYERED, size), size
+
+
+def test_workflow_is_edit_only_and_ready_without_a_projector():
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state()
+ assert b.status()["workflows"] == ["edit"]
+
+
+def test_server_body_sends_the_layer_count_and_drops_the_reconstruction():
+ server = _LayerServer("sd-server")
+ layers = LAYERED.layer_count
+ colors = [(i * 10, 0, 0, 255) for i in range(layers + 1)]
+ server.next_blobs = [_blob(c) for c in colors]
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state(mode = "server", server = server)
+ out = b.generate(prompt = "p", steps = 20, guidance = 2.5, seed = 5, init_image = _png())
+ body = server.payloads[-1]
+ assert body["qwen_image_layers"] == layers
+ assert (body["width"], body["height"]) == bk._layered_canvas_size(LAYERED, (900, 600))
+ assert body["sample_params"]["guidance"] == {"txt_cfg": 2.5}
+ assert body["sample_params"]["flow_shift"] == LAYERED.sd_cpp_flow_shift
+ ref = Image.open(io.BytesIO(base64.b64decode(body["ref_images"][0].split(",", 1)[1])))
+ assert ref.size == (body["width"], body["height"])
+ assert [im.getpixel((0, 0)) for im in out["images"]] == colors[1:]
+ assert all(im.mode == "RGBA" for im in out["images"])
+ assert out["seeds"] == [5] * layers and out["workflow"] == "edit"
+
+
+def test_server_refuses_a_short_layer_set():
+ server = _LayerServer("sd-server")
+ server.next_blobs = [_blob((1, 1, 1, 255))]
+ b = SdCppDiffusionBackend(engine = _FakeEngine())
+ b._state = _state(mode = "server", server = server)
+ with pytest.raises(RuntimeError, match = r"returned 1 of \d+ requested images"):
+ b.generate(prompt = "p", steps = 2, init_image = _png())
+
+
+def test_oneshot_collects_every_numbered_layer_file(monkeypatch):
+ engine = _FakeEngine()
+ layers = LAYERED.layer_count
+ colors = [(0, i * 10, 0, 255) for i in range(layers + 1)]
+ seen = {}
+
+ def _write(files, params, output_path, **kw):
+ seen["argv"] = build_sd_cpp_command("sd-cli", files, params, output_path = output_path)
+ for path, color in zip(sd_cli_output_paths(output_path, layers + 1), colors):
+ Path(path).write_bytes(_blob(color))
+ return Path(output_path)
+
+ monkeypatch.setattr(engine, "generate", _write)
+ b = SdCppDiffusionBackend(engine = engine)
+ b._state = _state()
+ out = b.generate(prompt = "p", steps = 20, guidance = 2.5, seed = 9, init_image = _png((768, 768)))
+ argv = seen["argv"]
+ assert argv[argv.index("--qwen-image-layers") + 1] == str(layers)
+ assert argv.count("--ref-image") == 1 and argv[argv.index("--width") + 1] == "640"
+ assert [im.getpixel((0, 0)) for im in out["images"]] == colors[1:]
+ assert out["seeds"] == [9] * layers
+
+
[email protected]("size", [(500, 521), (770, 500), (901, 603), (768, 768)])
+def test_off_grid_source_gets_the_same_canvas_as_the_diffusers_engine(size):
+ # The diffusers engine snaps the source to 16 px before the pipeline picks its canvas (500x521 -> 608x672, not
+ # the 640x640 the raw size gives), so the native engine must pick from the snapped size too.
+ from core.inference.diffusion import _layered_canvas, _snap_to_multiple
+
+ buf = io.BytesIO()
+ Image.new("RGBA", size, (10, 20, 30, 255)).save(buf, format = "PNG")
+ src = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+ w, h, _blobs = bk._native_condition_images(
+ LAYERED,
+ src,
+ None,
+ None,
+ None,
+ None,
+ full_fidelity = True,
+ pad_to_output = False,
+ source_sized = True,
+ )
+ assert (w, h) == _layered_canvas(LAYERED, _snap_to_multiple(Image.new("RGB", size), 16).size)
+
+
+def test_layered_needs_a_build_that_carries_layered_support(tmp_path):
+ # Layered landed upstream in master-744; an older reused build must not be picked for it.
+ old = tmp_path / "sd-cli-old"
+ old.write_bytes(b"stable-diffusion.cpp qwen_image_2_1 --ref-image")
+ new = tmp_path / "sd-cli-new"
+ new.write_bytes(b"stable-diffusion.cpp --qwen-image-layers qwen_image_layers")
+ assert not bk.sd_cpp_binary_runs_family(str(old), LAYERED)
+ assert bk.sd_cpp_binary_runs_family(str(new), LAYERED)