Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Tests for diffusion ControlNet support: discovery/resolve/preprocess/gate helpers, the
request-model validation, the family wiring, and the diffusers ControlNet pipe manager."""
from __future__ import annotations
import sys
import types
import pytest
from core.inference import diffusion_controlnet as dc
def test_sanitize_id():
assert dc.sanitize_id("owner/My ControlNet") == "My_ControlNet"
assert dc.sanitize_id("weird:<>name") == "weird_name"
assert dc.sanitize_id("") == "controlnet"
def test_list_controlnets_family_filter():
flux = {e.id for e in dc.list_controlnets(family = "flux.1")}
qwen = {e.id for e in dc.list_controlnets(family = "qwen-image")}
assert "flux-union-pro" in flux and "qwen-union" not in flux
assert "qwen-union" in qwen and "flux-union-pro" not in qwen
def test_resolve_controlnet_catalog_bare_repo_and_unknown():
r = dc.resolve_controlnet("flux-union-pro", family = "flux.1")
assert r.path == "Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro" and not r.is_local
r2 = dc.resolve_controlnet("owner/some-controlnet")
assert r2.path == "owner/some-controlnet" and not r2.is_local
with pytest.raises(FileNotFoundError):
dc.resolve_controlnet("not-a-known-id")
def test_resolve_controlnet_rejects_filesystem_like_ids():
# Never accept a path-shaped id: from_pretrained would load a local dir outside controlnets_dir().
for bad in ("/tmp/model", "../some/model", "./x/y", "~/x/y", "a/b/c", "C:\\x/y", ".hidden/x"):
with pytest.raises(FileNotFoundError):
dc.resolve_controlnet(bad)
def test_resolve_controlnet_enforces_family_match():
with pytest.raises(ValueError, match = "not the"):
dc.resolve_controlnet("qwen-union", family = "flux.1")
assert dc.resolve_controlnet("qwen-union", family = "qwen-image").path
assert dc.resolve_controlnet("qwen-union").path
def test_resolve_controlnet_repo_id_still_family_gated():
with pytest.raises(ValueError, match = "is for"):
dc.resolve_controlnet("InstantX/Qwen-Image-ControlNet-Union", family = "flux.1")
r = dc.resolve_controlnet("InstantX/Qwen-Image-ControlNet-Union", family = "qwen-image")
assert r.path == "InstantX/Qwen-Image-ControlNet-Union" and not r.is_local
def test_union_control_mode_maps_only_union_entries():
assert dc.union_control_mode("flux-union-pro", "canny") == 0
assert dc.union_control_mode("flux-union-pro", "depth") == 2
assert dc.union_control_mode("flux-union-pro", "pose") == 4
assert dc.union_control_mode("flux-union-pro", "passthrough") == 0
assert dc.union_control_mode("some/bare-repo", "canny") is None
def test_union_control_mode_rejects_unknown_type():
# Non-canny maps pass through preprocess_control unchanged, so a typo must not default to canny.
with pytest.raises(ValueError, match = "Unknown control type"):
dc.union_control_mode("flux-union-pro", "detph")
with pytest.raises(ValueError, match = "Unknown control type"):
dc.union_control_mode("flux-union-pro", "scribble")
assert dc.union_control_mode("flux-union-pro", "") == 0
assert dc.union_control_mode("some/bare-repo", "detph") is None
def test_union_control_mode_matches_curated_repo_id():
# resolve_controlnet accepts curated unions by repo id, so union_control_mode must too.
assert dc.union_control_mode("Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "depth") == 2
assert dc.union_control_mode("Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "pose") == 4
assert dc.union_control_mode("Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "passthrough") == 0
assert dc.union_control_mode("InstantX/Qwen-Image-ControlNet-Union", "canny") == 0
assert dc.union_control_mode("some/other-controlnet", "canny") is None
with pytest.raises(ValueError, match = "Unknown control type"):
dc.union_control_mode("Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro", "detph")
def test_resolve_controlnet_local(tmp_path, monkeypatch):
d = tmp_path / "controlnets"
d.mkdir()
cn = d / "my-cn"
cn.mkdir()
(cn / "config.json").write_text("{}")
(cn / "diffusion_pytorch_model.safetensors").write_bytes(b"x")
monkeypatch.setattr(dc, "controlnets_dir", lambda: d)
entries = {e.id for e in dc.list_controlnets()}
assert "my-cn" in entries
r = dc.resolve_controlnet("my-cn")
assert r.is_local and r.path == str(cn)
def test_scan_local_skips_config_only_folder(tmp_path, monkeypatch):
d = tmp_path / "controlnets"
d.mkdir()
incomplete = d / "incomplete-cn"
incomplete.mkdir()
(incomplete / "config.json").write_text("{}")
monkeypatch.setattr(dc, "controlnets_dir", lambda: d)
assert "incomplete-cn" not in {e.id for e in dc.list_controlnets()}
(incomplete / "diffusion_pytorch_model.safetensors.index.json").write_text("{}")
assert "incomplete-cn" in {e.id for e in dc.list_controlnets()}
def test_preprocess_control_passthrough_and_canny():
from PIL import Image
img = Image.new("RGB", (32, 24), (10, 20, 30))
assert dc.preprocess_control(img, "passthrough") is img
# A flat image has no edges, so the map is all black; passing the source through would condition the ControlNet on raw luminance.
import numpy as np
flat = dc.preprocess_control(img, "canny")
assert flat.mode == "RGB" and flat.size == (32, 24)
assert np.asarray(flat).max() == 0
arr = np.zeros((24, 32, 3), np.uint8)
arr[:, 16:, :] = 255
edged = dc.preprocess_control(Image.fromarray(arr), "canny")
assert edged.mode == "RGB" and edged.size == (32, 24)
assert np.asarray(edged).max() == 255
def test_supports_controlnet_matrix():
ok = dict(engine = "diffusers", family = "flux.1", has_controlnet_pipeline = True)
assert dc.supports_controlnet(**ok, model_kind = "pipeline", transformer_quant = None)
assert dc.supports_controlnet(**ok, model_kind = "single_file", transformer_quant = None)
assert not dc.supports_controlnet(**ok, model_kind = "gguf", transformer_quant = None)
assert not dc.supports_controlnet(**ok, model_kind = "single_file", transformer_quant = "fp8")
assert not dc.supports_controlnet(**ok, model_kind = "single_file", transformer_quant = "int8")
assert not dc.supports_controlnet(
engine = "sd_cpp",
family = "flux.1",
has_controlnet_pipeline = True,
model_kind = "gguf",
transformer_quant = None,
)
assert not dc.supports_controlnet(
engine = "diffusers",
family = "z-image",
has_controlnet_pipeline = False,
model_kind = "pipeline",
transformer_quant = None,
)
def test_controlnet_spec_and_request_validation():
from models.inference import ControlNetSpec, DiffusionGenerateRequest
assert DiffusionGenerateRequest(prompt = "x").controlnet is None
req = DiffusionGenerateRequest(
prompt = "x",
controlnet = {
"id": "flux-union-pro",
"image": "data",
"control_type": "canny",
"strength": 0.6,
},
)
assert req.controlnet.id == "flux-union-pro" and req.controlnet.strength == 0.6
s = ControlNetSpec(id = "a", image = "b")
assert s.control_type == "passthrough" and s.strength == 1.0
assert s.guidance_start == 0.0 and s.guidance_end == 1.0
with pytest.raises(Exception):
ControlNetSpec(id = "a", image = "b", strength = 3.0)
with pytest.raises(Exception):
ControlNetSpec(id = "a", image = "b", guidance_end = 1.5)
def test_families_declare_controlnet_classes():
from core.inference.diffusion_families import _FAMILIES
by_name = {f.name: f for f in _FAMILIES}
assert by_name["flux.1"].controlnet_pipeline_class == "FluxControlNetPipeline"
assert by_name["flux.1"].controlnet_model_class == "FluxControlNetModel"
assert by_name["qwen-image"].controlnet_pipeline_class == "QwenImageControlNetPipeline"
assert by_name["z-image"].controlnet_pipeline_class is None
class _FakeCNModel:
@classmethod
def from_pretrained(
cls,
path,
torch_dtype = None,
token = None,
use_safetensors = None,
# cache_dir (and any future loader kwarg) rides through: the real call pins the live cache root so a load cannot split across two.
**kwargs,
):
m = cls()
m.path = path
m.use_safetensors = use_safetensors
m.cache_dir = kwargs.get("cache_dir")
return m
def to(self, device):
self.device = device
return self
class _FakeCNPipe:
recast_dtype: object = None
def to(self, *args, **kwargs):
_FakeCNPipe.recast_dtype = kwargs.get("dtype")
return self
@classmethod
def from_pipe(
cls,
base,
controlnet = None,
torch_dtype = None,
):
p = cls()
p.base = base
p.controlnet = controlnet
_FakeCNPipe.recast_dtype = None
p.to(dtype = torch_dtype or "float32")
return p
def _fake_diffusers():
mod = types.ModuleType("diffusers")
mod.FluxControlNetModel = _FakeCNModel
mod.FluxControlNetPipeline = _FakeCNPipe
return mod
def _state():
fam = types.SimpleNamespace(
name = "flux.1",
controlnet_pipeline_class = "FluxControlNetPipeline",
controlnet_model_class = "FluxControlNetModel",
)
return types.SimpleNamespace(
family = fam, dtype = "bf16", device = "cpu", hf_token = None, pipe = object()
)
def _allow_cn_security(monkeypatch):
"""Stub the Hub malware preflight to allow the load (hermetic, no network)."""
import utils.security
monkeypatch.setattr(
utils.security,
"evaluate_file_security",
lambda name, hf_token = None, **kw: types.SimpleNamespace(blocked = False, reason = ""),
)
def test_controlnet_pipe_loads_once_and_caches(monkeypatch):
import threading
from core.inference.diffusion import DiffusionBackend
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers())
_allow_cn_security(monkeypatch)
b = DiffusionBackend()
st = _state()
# The pipe cache only commits while ``st`` is the CURRENT load (an unload racing from_pipe must not repopulate it), so mirror the loaded invariant.
b._state = st
resolved = dc.ResolvedControlNet("flux-union-pro", "repo/id", is_local = False)
p1 = b._controlnet_pipe(st, resolved, threading.Event())
assert isinstance(p1, _FakeCNPipe) and isinstance(p1.controlnet, _FakeCNModel)
assert p1.controlnet.path == "repo/id" and p1.controlnet.device == "cpu"
# A remote (non-local) ControlNet must force safetensors so a pickle cannot deserialize even if the Hub scan failed open.
assert p1.controlnet.use_safetensors is True
# The cast never reaches the resident base modules the ControlNet pipe shares.
assert _FakeCNPipe.recast_dtype is None
p2 = b._controlnet_pipe(st, resolved, threading.Event())
assert p2 is p1
assert b._cn_models["flux-union-pro"] is p1.controlnet
def test_controlnet_pipe_blocks_flagged_remote_repo(monkeypatch):
# Bare owner/name ControlNets skip the base trust gate, so the Hub malware preflight must run.
import threading
import utils.security
from core.inference.diffusion import DiffusionBackend
loaded = {"called": False}
class _TrapModel(_FakeCNModel):
@classmethod
def from_pretrained(
cls,
path,
torch_dtype = None,
token = None,
use_safetensors = None,
):
loaded["called"] = True
return super().from_pretrained(
path, torch_dtype = torch_dtype, token = token, use_safetensors = use_safetensors
)
mod = _fake_diffusers()
mod.FluxControlNetModel = _TrapModel
monkeypatch.setitem(sys.modules, "diffusers", mod)
monkeypatch.setattr(
utils.security,
"evaluate_file_security",
lambda name, hf_token = None, **kw: types.SimpleNamespace(
blocked = True, reason = "Hugging Face security scan flagged unsafe files: evil.bin"
),
)
b = DiffusionBackend()
st = _state()
b._state = st
resolved = dc.ResolvedControlNet("evil/cn", "evil/cn", is_local = False)
with pytest.raises(ValueError, match = "security scan flagged"):
b._controlnet_pipe(st, resolved, threading.Event())
assert loaded["called"] is False
def test_controlnet_pipe_skips_scan_for_local_dir(monkeypatch, tmp_path):
import threading
import utils.security
from core.inference.diffusion import DiffusionBackend
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers())
monkeypatch.setattr(
utils.security,
"evaluate_file_security",
lambda name, hf_token = None, **kw: types.SimpleNamespace(blocked = True, reason = "x"),
)
b = DiffusionBackend()
st = _state()
b._state = st
resolved = dc.ResolvedControlNet("my-cn", str(tmp_path), is_local = True)
p = b._controlnet_pipe(st, resolved, threading.Event())
assert isinstance(p, _FakeCNPipe)
assert p.controlnet.use_safetensors is None
def test_controlnet_pipe_rejects_family_without_classes():
import threading
from core.inference.diffusion import DiffusionBackend
b = DiffusionBackend()
st = _state()
st.family.controlnet_pipeline_class = None
with pytest.raises(ValueError, match = "not supported"):
b._controlnet_pipe(st, dc.ResolvedControlNet("x", "y", False), threading.Event())
def test_controlnet_pipe_not_cached_after_unload_race(monkeypatch):
import threading
from core.inference.diffusion import DiffusionBackend
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers())
b = DiffusionBackend()
st = _state()
resolved = dc.ResolvedControlNet("flux-union-pro", "repo/id", is_local = False)
with pytest.raises(RuntimeError, match = "cancelled"):
b._controlnet_pipe(st, resolved, threading.Event())
assert b._cn_pipes == {}
@pytest.mark.parametrize(
"policy, calibrated, streamed",
[("none", False, False), ("none", True, True), ("group", False, True)],
)
def test_controlnet_streams_beside_a_calibrated_resident_tier(
monkeypatch, policy, calibrated, streamed
):
import threading
from core.inference import diffusion as d
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers())
_allow_cn_security(monkeypatch)
calls = []
monkeypatch.setattr(
d, "_offload_controlnet_module", lambda m, device, logger: calls.append(m) or True
)
b = d.DiffusionBackend()
st = _state()
st.offload_policy = policy
st.calibrated_placement = calibrated
b._state = st
p = b._controlnet_pipe(
st, dc.ResolvedControlNet("flux-union-pro", "repo/id", is_local = False), threading.Event()
)
assert bool(calls) is streamed
assert hasattr(p.controlnet, "device") is not streamed
def test_a_calibrated_tier_refuses_a_controlnet_it_cannot_stream(monkeypatch):
import threading
from core.inference import diffusion as d
monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers())
_allow_cn_security(monkeypatch)
monkeypatch.setattr(d, "_offload_controlnet_module", lambda m, device, logger: False)
b = d.DiffusionBackend()
st = _state()
st.offload_policy = "none"
st.calibrated_placement = True
b._state = st
resolved = dc.ResolvedControlNet("flux-union-pro", "repo/id", is_local = False)
with pytest.raises(ValueError, match = "balanced memory mode"):
b._controlnet_pipe(st, resolved, threading.Event())
assert b._cn_models == {} and b._cn_pipes == {}
st.calibrated_placement = False
st.offload_policy = "group"
p = b._controlnet_pipe(st, resolved, threading.Event())
assert hasattr(p.controlnet, "device")
def _fake_group_offload(monkeypatch, with_stream_kwargs = True):
"""Record _offload_controlnet_module's apply_group_offloading kwargs and the denoiser streaming helpers it calls."""
from core.inference import diffusion_memory as dm
calls = {"apply": [], "helpers": []}
if with_stream_kwargs:
def apply_group_offloading(
module,
onload_device,
offload_device = None,
offload_type = "block_level",
num_blocks_per_group = None,
non_blocking = False,
use_stream = False,
record_stream = False,
low_cpu_mem_usage = False,
):
calls["apply"].append(
dict(
onload_device = onload_device,
offload_type = offload_type,
num_blocks_per_group = num_blocks_per_group,
non_blocking = non_blocking,
use_stream = use_stream,
record_stream = record_stream,
)
)
else:
def apply_group_offloading(
module,
onload_device,
offload_device = None,
offload_type = "block_level",
num_blocks_per_group = None,
use_stream = False,
):
calls["apply"].append(dict(onload_device = onload_device, use_stream = use_stream))
hooks = types.ModuleType("diffusers.hooks")
hooks.apply_group_offloading = apply_group_offloading
monkeypatch.setitem(sys.modules, "diffusers.hooks", hooks)
for name, ret in (
("install_group_offload_buffer_restore", True),
("install_group_offload_hooks_eager", True),
("_pin_top_level_group", True),
("_skip_top_level_copy_back", True),
("install_group_prefetch", 3),
):
monkeypatch.setattr(
dm,
name,
lambda *a, _n = name, _r = ret, **k: calls["helpers"].append(_n) or _r,
)
return calls
def test_streamed_controlnet_uses_the_event_fenced_prefetcher_on_cuda(monkeypatch):
from core.inference import diffusion as d
calls = _fake_group_offload(monkeypatch)
module = object()
assert d._offload_controlnet_module(module, "cuda", None) is True
(kwargs,) = calls["apply"]
assert kwargs["use_stream"] and kwargs["record_stream"] and kwargs["non_blocking"]
assert kwargs["offload_type"] == "block_level" and kwargs["num_blocks_per_group"] == 1
assert calls["helpers"] == [
"install_group_offload_buffer_restore",
"install_group_offload_hooks_eager",
"_pin_top_level_group",
"install_group_prefetch",
]
def test_streamed_controlnet_falls_back_to_no_copy_back_when_the_top_group_cannot_pin(monkeypatch):
from core.inference import diffusion as d
from core.inference import diffusion_memory as dm
calls = _fake_group_offload(monkeypatch)
monkeypatch.setattr(
dm, "_pin_top_level_group", lambda *a, **k: calls["helpers"].append("pin") and False
)
assert d._offload_controlnet_module(object(), "cuda", None) is True
assert calls["helpers"][-3:] == ["pin", "_skip_top_level_copy_back", "install_group_prefetch"]
@pytest.mark.parametrize("device, with_stream_kwargs", [("cpu", True), ("cuda", False)])
def test_controlnet_offload_unchanged_without_a_cuda_stream_path(
monkeypatch, device, with_stream_kwargs
):
from core.inference import diffusion as d
calls = _fake_group_offload(monkeypatch, with_stream_kwargs)
assert d._offload_controlnet_module(object(), device, None) is True
(kwargs,) = calls["apply"]
assert kwargs["use_stream"] is (device == "cuda")
assert not kwargs.get("record_stream") and not kwargs.get("non_blocking")
assert calls["helpers"] == []