Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

12.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Native (sd.cpp) Qwen-Image-2.1 editing: the vision projector asset, the --llm_vision and
reference-image transport on both the one-shot and the server path, the readiness gate, and the
pinned build's alpha and crop workarounds."""

from __future__ import annotations

import base64
import dataclasses
import io
import types

import pytest
from PIL import Image

from core.inference import sd_cpp_backend as bk
from core.inference.diffusion_conditioning import LocalizedEdit
from core.inference.diffusion_families import detect_family, sd_cpp_text_encoders_for
from core.inference.sd_cpp_args import (
    SdCppGenParams,
    SdCppModelFiles,
    build_img_gen_request,
    build_sd_cpp_command,
    build_sd_cpp_server_command,
)
from core.inference.sd_cpp_backend import SdCppDiffusionBackend

from .test_sd_cpp_backend import _FakeEngine, _FakeServer

FAM = detect_family("unsloth/Qwen-Image-2.1-GGUF")
FILES = SdCppModelFiles(
    diffusion_model = "/m/q.gguf",
    vae = "/m/vae.safetensors",
    llm = "/m/llm.gguf",
    llm_vision = "/m/mmproj.gguf",
)


def _png(size = (64, 32), color = (200, 10, 10, 255)) -> str:
    buf = io.BytesIO()
    Image.new("RGBA", size, color).save(buf, format = "PNG")
    return base64.b64encode(buf.getvalue()).decode()


def _decode(blob: bytes) -> Image.Image:
    return Image.open(io.BytesIO(blob))


def test_the_projector_is_a_planned_companion_asset():
    specs = SdCppDiffusionBackend(engine = _FakeEngine())._asset_specs(
        "unsloth/Qwen-Image-2.1-GGUF", "qwen-image-2.1-Q4_K_M.gguf", FAM
    )
    assert ("unsloth/Qwen3-VL-8B-Instruct-GGUF", "mmproj-F16.gguf", "llm_vision") in specs
    assert any(k == "llm_vision" for _r, _f, k in sd_cpp_text_encoders_for(FAM))


def test_both_command_builders_pass_the_projector_and_the_cli_keeps_ref_order():
    cli = build_sd_cpp_command(
        "sd-cli",
        FILES,
        SdCppGenParams(prompt = "p", width = 512, height = 512, ref_images = ("/a.png", "/b.png")),
        output_path = "/o.png",
    )
    assert cli[cli.index("--llm_vision") + 1] == "/m/mmproj.gguf"
    refs = [cli[i + 1] for i, a in enumerate(cli) if a == "--ref-image"]
    assert refs == ["/a.png", "/b.png"]
    assert "--init-img" not in cli and "--mask" not in cli
    server = build_sd_cpp_server_command("sd-server", FILES, host = "127.0.0.1", port = 1)
    assert server[server.index("--llm_vision") + 1] == "/m/mmproj.gguf"


def test_the_server_request_carries_refs_and_nothing_img2img():
    req = build_img_gen_request(prompt = "p", ref_images = ["data:image/png;base64,AA", "B"])
    assert req["ref_images"] == ["data:image/png;base64,AA", "B"]
    assert "init_image" not in req and "strength" not in req and "mask_image" not in req
    assert "ref_images" not in build_img_gen_request(prompt = "p")


def test_condition_images_flatten_alpha_over_white_and_pad_only_for_the_server():
    clear = _png(size = (64, 32), color = (0, 0, 255, 0))
    w, h, blobs = bk._native_condition_images(
        FAM, _png(size = (64, 32)), [clear], None, None, None, full_fidelity = False, pad_to_output = True
    )
    # Image 1's 2:1 aspect ratio at the 1024 area, the same size the diffusers engine picks.
    assert (w, h) == (1440, 736)
    ref = _decode(blobs[1])
    assert ref.mode == "RGB" and ref.getpixel((32, 16)) == (255, 255, 255)
    _w, _h, padded = bk._native_condition_images(
        FAM,
        _png(size = (64, 32)),
        [_png(size = (32, 32))],
        None,
        1024,
        512,
        full_fidelity = False,
        pad_to_output = True,
    )
    assert _decode(padded[1]).size == (64, 32)
    _w, _h, plain = bk._native_condition_images(
        FAM,
        _png(size = (64, 32)),
        [_png(size = (32, 32))],
        None,
        1024,
        512,
        full_fidelity = False,
        pad_to_output = False,
    )
    assert _decode(plain[1]).size == (32, 32)


def test_native_outputs_keep_alpha_for_the_family_only():
    rgba = Image.new("RGBA", (2, 2), (1, 2, 3, 100))
    assert bk._native_output_image(FAM, rgba).mode == "RGBA"
    assert bk._native_output_image(detect_family("z-image"), rgba).mode == "RGB"


def _state(
    files = FILES,
    mode = "oneshot",
    server = None,
):
    return bk._SdState(
        repo_id = "unsloth/Qwen-Image-2.1-GGUF",
        base_repo = FAM.base_repo,
        family = FAM,
        device = "cpu",
        files = files,
        sampling_method = FAM.sd_cpp_sampling_method,
        mode = mode,
        server = server,
    )


def test_edit_is_advertised_only_with_the_projector_and_an_edit_capable_build(tmp_path):
    capable = tmp_path / "sd-new"
    capable.write_bytes(b"xx" + FAM.sd_cpp_edit_marker.encode() + b"yy qwen_image_2_1")
    older = tmp_path / "sd-old"
    older.write_bytes(b"qwen_image_2_1 only")
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    for binary, files, expect in (
        (capable, FILES, True),
        (older, FILES, False),
        (capable, SdCppModelFiles(diffusion_model = "/m/q.gguf", llm = "/m/llm.gguf"), False),
    ):
        b._state = _state(
            files,
            mode = "server",
            server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True),
        )
        status = b.status()
        assert ("edit" in status["workflows"]) is expect, binary
        assert ("reference" in status["workflows"]) is expect
        if expect:
            c = status["conditioning"]
            assert c["reference_resolutions"] == [] and c["alpha"] is False
            assert any("padded to its aspect ratio" in n for n in c["notes"])
            assert any("raise Guidance above 1" in n for n in c["notes"])


def test_an_unreadable_build_advertises_no_edit_and_no_fidelity(tmp_path):
    binary = tmp_path / "sd-server"
    binary.write_bytes(
        FAM.sd_cpp_edit_marker.encode() + b" " + bk._REFERENCE_FIDELITY_MARKER.encode()
    )
    binary.chmod(0o111)
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    b._state = _state(
        mode = "server",
        server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True),
    )
    try:
        assert "edit" not in b.status()["workflows"]
        assert b._native_reference_fidelity(b._state) is False
        assert bk.binary_carries_marker(str(binary), "anything") is True
    finally:
        binary.chmod(0o644)


def test_oneshot_edit_stages_ordered_pngs_and_records_the_workflow(monkeypatch):
    engine = _FakeEngine()
    b = SdCppDiffusionBackend(engine = engine)
    b._state = _state()
    monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: True)
    seen = {}

    def _capture(files, params, output_path, **kw):
        seen["refs"] = [Image.open(p).getpixel((0, 0)) for p in params.ref_images]
        seen["size"] = (params.width, params.height)
        Image.new("RGBA", (1, 1), (1, 2, 3, 4)).save(output_path)

    monkeypatch.setattr(engine, "generate", _capture)
    out = b.generate(
        prompt = "p",
        steps = 2,
        workflow = "edit",
        width = None,
        height = None,
        init_image = _png(color = (10, 0, 0, 255)),
        reference_images = [_png(color = (20, 0, 0, 255)), _png(color = (30, 0, 0, 255))],
        localized_edit = None,
    )
    assert [c[:3] for c in seen["refs"]] == [(10, 0, 0), (20, 0, 0), (30, 0, 0)]
    assert seen["size"][0] / seen["size"][1] == pytest.approx(2.0, rel = 0.05)
    assert out["workflow"] == "edit" and out["images"][0].mode == "RGBA"


def test_server_edit_sends_the_mask_as_image_2(monkeypatch):
    server = _FakeServer("sd-server")
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    b._state = _state(mode = "server", server = server)
    monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: True)
    mask = io.BytesIO()
    Image.new("L", (64, 32), 255).save(mask, format = "PNG")
    b.generate(
        prompt = "p",
        steps = 2,
        workflow = "edit",
        width = 512,
        height = 256,
        init_image = _png(),
        localized_edit = LocalizedEdit("mask", base64.b64encode(mask.getvalue()).decode()),
    )
    refs = server.payloads[-1]["ref_images"]
    assert len(refs) == 2
    second = _decode(base64.b64decode(refs[1].split(",", 1)[1]))
    assert second.getpixel((0, 0)) == (255, 255, 255)


def test_native_refuses_what_it_cannot_honour(monkeypatch):
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    b._state = _state()
    with pytest.raises(ValueError, match = "Reference detail is not adjustable"):
        b.generate(prompt = "p", workflow = "edit", init_image = _png(), reference_resolution = 1024)
    with pytest.raises(ValueError, match = "not yet supported"):
        b.generate(prompt = "p", init_image = _png())
    with pytest.raises(ValueError, match = "localized_edit needs the edit workflow"):
        b.generate(
            prompt = "p",
            workflow = "reference",
            init_image = _png(),
            width = 512,
            height = 256,
            localized_edit = LocalizedEdit("paint", _png()),
        )
    monkeypatch.setattr(SdCppDiffusionBackend, "_native_edit_ready", lambda self, st: False)
    with pytest.raises(ValueError, match = "Image editing is not available"):
        b.generate(prompt = "p", workflow = "edit", init_image = _png(), width = 512, height = 512)


@pytest.mark.parametrize(
    "family, width, height, match",
    [
        ("z-image", 2560, 1024, "at most 2048px"),
        ("qwen-image-2.1", 1040, 1024, "multiples of 32"),
        ("qwen-image-2.1", 2752, 2752, "pixels"),
    ],
)
def test_native_text_to_image_keeps_the_family_size_bounds(family, width, height, match):
    # The request schema admits 2752 for Qwen-Image-2.1's 2K presets; every other bound is the family's.
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    b._state = dataclasses.replace(_state(), family = detect_family(family))
    with pytest.raises(ValueError, match = match):
        b.generate(prompt = "p", steps = 2, width = width, height = height)


def test_a_padded_mask_adds_no_region_and_stays_aligned_with_the_source():
    """Mask padding must be black, by the same amount as the source, not white: white marks the
    added border as part of the edit region."""
    mask = Image.new("L", (64, 32), 0)
    for x in range(8):
        for y in range(8):
            mask.putpixel((x, y), 255)
    buf = io.BytesIO()
    mask.save(buf, format = "PNG")
    _w, _h, blobs = bk._native_condition_images(
        FAM,
        _png(size = (64, 32), color = (9, 9, 9, 255)),
        None,
        LocalizedEdit("mask", base64.b64encode(buf.getvalue()).decode()),
        512,
        512,
        full_fidelity = False,
        pad_to_output = True,
    )
    source, padded = _decode(blobs[0]), _decode(blobs[1]).convert("L")
    assert source.size == padded.size == (64, 64)
    assert padded.getpixel((32, 2)) == 0 and padded.getpixel((32, 60)) == 0
    assert padded.getpixel((2, 18)) == 255 and padded.getpixel((2, 26)) == 0
    assert source.getpixel((32, 2))[:3] == (255, 255, 255)
    assert source.getpixel((2, 18))[:3] == (9, 9, 9)


def test_a_full_fidelity_build_gets_the_images_as_decoded(tmp_path):
    """A full-fidelity build reads alpha and keeps image shapes, so no workaround applies."""
    clear = _png(size = (32, 32), color = (0, 0, 255, 0))
    _w, _h, blobs = bk._native_condition_images(
        FAM, _png(size = (64, 32)), [clear], None, 1024, 512, full_fidelity = True, pad_to_output = True
    )
    ref = _decode(blobs[1])
    assert ref.mode == "RGBA" and ref.size == (32, 32) and ref.getpixel((4, 4)) == (0, 0, 255, 0)

    binary = tmp_path / "sd-server"
    binary.write_bytes(
        b"qwen_image_2_1 "
        + FAM.sd_cpp_edit_marker.encode()
        + b" "
        + bk._REFERENCE_FIDELITY_MARKER.encode()
    )
    b = SdCppDiffusionBackend(engine = _FakeEngine())
    b._state = _state(
        mode = "server", server = types.SimpleNamespace(binary = str(binary), is_alive = lambda: True)
    )
    c = b.status()["conditioning"]
    assert c["alpha"] is True
    assert c["notes"] == [
        "For transparent output on the native engine, raise Guidance above 1 (upstream uses 6)."
    ]