Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let FLUX.3 Klein edit through the Edit workflow (#15263)

Daniel Han277bd03018fe
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index 73878f9eebf..e22ec3eaa76 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -14043,7 +14043,9 @@ class DiffusionBackend:
init_image, mode = getattr(fam, "condition_image_mode", "RGB") or "RGB"
)
elif requested_workflow == "edit":
- if not getattr(fam, "unified_edit", False):
+ if not (
+ getattr(fam, "unified_edit", False) or getattr(fam, "reference_edit", False)
+ ):
raise ValueError(
f"Instruction editing is not supported for the '{fam.name}' "
"model family."
@@ -15612,8 +15614,8 @@ def _family_workflows(fam: DiffusionFamily) -> list[str]:
# Reference families (FLUX.2) add reference conditioning via their pipeline's image arg.
if getattr(fam, "reference", False):
workflows.append("reference")
- # Unified families (Qwen-Image-2.1) also follow edit instructions through that same pipeline.
- if getattr(fam, "unified_edit", False):
+ # Unified (Qwen-Image-2.1) and reference-edit (FLUX.3) families also edit through that same pipeline.
+ if getattr(fam, "unified_edit", False) or getattr(fam, "reference_edit", False):
workflows.append("edit")
if getattr(fam, "img2img_pipeline_class", None):
# Upscale runs on the img2img pipeline, so available exactly when img2img is.
diff --git a/studio/backend/core/inference/diffusion_conditioning.py b/studio/backend/core/inference/diffusion_conditioning.py
index d833f9ee081..76242d67105 100644
--- a/studio/backend/core/inference/diffusion_conditioning.py
+++ b/studio/backend/core/inference/diffusion_conditioning.py
@@ -247,7 +247,9 @@ def edit_takes_extra_images(fam: Any) -> bool:
def conditioning_capabilities(fam: Any, workflows: Sequence[str]) -> dict[str, Any]:
"""What the UI may offer for ``fam`` given the ``workflows`` the active engine runs for it.
Localized editing is advertised only where the unified edit workflow itself is."""
- unified = bool(getattr(fam, "unified_edit", False)) and "edit" in workflows
+ localized = bool(getattr(fam, "unified_edit", False)) and "edit" in workflows
+ # The page's unified Edit form (Image 1 + extras, output sized from Image 1) fits a reference-edit family too.
+ unified = localized or (bool(getattr(fam, "reference_edit", False)) and "edit" in workflows)
return {
"max_condition_images": int(getattr(fam, "max_condition_images", 4) or 4),
"alpha": (getattr(fam, "condition_image_mode", "RGB") or "RGB") == "RGBA",
@@ -258,5 +260,5 @@ def conditioning_capabilities(fam: Any, workflows: Sequence[str]) -> dict[str, A
"unified_edit": unified,
# Edit-only families whose edit takes the extra images too (the UI shows them on the Edit tab).
"multi_image_edit": edit_takes_extra_images(fam) and "edit" in workflows,
- "localized_edit_modes": list(LOCALIZED_EDIT_MODES) if unified else [],
+ "localized_edit_modes": list(LOCALIZED_EDIT_MODES) if localized else [],
}
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py
index 1823698834b..d9c3eef0953 100644
--- a/studio/backend/core/inference/diffusion_families.py
+++ b/studio/backend/core/inference/diffusion_families.py
@@ -123,6 +123,8 @@ class DiffusionFamily:
# True when the text-to-image pipeline also follows edit instructions over ``image`` (Qwen-Image-2.1), at the
# requested size. Never inferred from ``reference``: a reference family is not necessarily trained to edit.
unified_edit: bool = False
+ # Trained to edit through its reference call (FLUX.3 Klein; BFL's i2i / mi2i): Edit, without localized modes or alpha.
+ reference_edit: bool = False
# Qwen-Image-Layered: RGBA layers per input (each an output image); 0 = not layered.
layer_count: int = 0
# The pipeline's ``resolution`` area bucket (640 or 1024).
@@ -604,6 +606,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
base_repo = "black-forest-labs/flux-3-image-klein-base",
aliases = ("flux-3-image-klein-base", "flux3-image-klein-base", "flux3-klein-base"),
reference = True,
+ reference_edit = True,
max_condition_images = 10,
# Two DiT patch grids (VAE /8, patch 2): sizes snap to 16 like FLUX.2.
dimension_multiple = 16,
@@ -657,6 +660,7 @@ _FAMILIES: tuple[DiffusionFamily, ...] = (
base_repo = "black-forest-labs/flux-3-image-klein",
aliases = ("flux-3-image-klein", "flux3-image-klein", "flux3-klein"),
reference = True,
+ reference_edit = True,
max_condition_images = 10,
dimension_multiple = 16,
condition_pixel_weight = 0.11,
diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py
index ddc661581af..69134854660 100644
--- a/studio/backend/core/inference/sd_cpp_backend.py
+++ b/studio/backend/core/inference/sd_cpp_backend.py
@@ -4646,12 +4646,12 @@ class SdCppDiffusionBackend:
def _native_reference_ready(self, state: Optional[_SdState], workflow: Optional[str]) -> bool:
"""Projector-free references (FLUX.3): sd.cpp feeds them through the VAE, so the arch gate that admitted the
- load is all a reference call needs. Not an instruction edit."""
- return (
- state is not None
- and bool(getattr(state.family, "sd_cpp_reference", False))
- and workflow in (None, "reference")
- )
+ load is all a reference call needs. A reference-edit family's Edit is that same call."""
+ if state is None or not getattr(state.family, "sd_cpp_reference", False):
+ return False
+ if workflow == "edit":
+ return bool(getattr(state.family, "reference_edit", False))
+ return workflow in (None, "reference")
@holds_inference
def generate(
@@ -5706,6 +5706,8 @@ class SdCppDiffusionBackend:
workflows += ["reference", "edit"]
elif self._native_reference_ready(state, "reference"):
workflows.append("reference")
+ if self._native_reference_ready(state, "edit"):
+ workflows.append("edit")
conditioning = conditioning_capabilities(state.family, workflows)
# No reference detail natively: sd.cpp sizes inputs to the output area.
conditioning["reference_resolutions"] = []
@@ -5713,7 +5715,7 @@ class SdCppDiffusionBackend:
conditioning["alpha"] = full_fidelity
notes: list[str] = []
# The alpha / padding notes are about the unified family's RGBA path; edit-only families are RGB, source-sized.
- if "edit" in workflows and not getattr(state.family, "edit", False):
+ if "edit" in workflows and getattr(state.family, "unified_edit", False):
if not full_fidelity:
notes.append(
"Transparent parts of input images are filled with white on this native build."
diff --git a/studio/backend/tests/test_diffusion_flux3.py b/studio/backend/tests/test_diffusion_flux3.py
index 9f158923156..ea238d456b3 100644
--- a/studio/backend/tests/test_diffusion_flux3.py
+++ b/studio/backend/tests/test_diffusion_flux3.py
@@ -85,7 +85,8 @@ def test_family_records():
for fam in (distilled, base):
# One text-to-image call takes up to 10 reference images (the BFL reference's MAX_REFERENCES).
assert fam.reference and fam.max_condition_images == 10
- assert not fam.edit and not fam.unified_edit
+ # Edit is that same call (BFL's i2i / mi2i), not Qwen-Image-2.1's localized unified edit.
+ assert fam.reference_edit and not fam.edit and not fam.unified_edit
assert fam.dimension_multiple == 16
# A single file / GGUF denoiser assembles around the base's modular companions.
assert not fam.pipeline_only
diff --git a/studio/backend/tests/test_diffusion_unified_edit.py b/studio/backend/tests/test_diffusion_unified_edit.py
index f6ed826b720..9ad1875342d 100644
--- a/studio/backend/tests/test_diffusion_unified_edit.py
+++ b/studio/backend/tests/test_diffusion_unified_edit.py
@@ -130,7 +130,52 @@ def test_edit_only_and_reference_families_keep_their_contracts():
from core.inference.diffusion import _family_workflows
assert _family_workflows(kontext) == ["edit"]
- assert "edit" not in _family_workflows(flux2)
+ assert "edit" not in _family_workflows(flux2) and not flux2.reference_edit
+ for repo in (
+ "black-forest-labs/flux-3-image-klein",
+ "black-forest-labs/flux-3-image-klein-base",
+ ):
+ flux3 = detect_family(repo)
+ assert _family_workflows(flux3) == ["txt2img", "reference", "edit"]
+ c = cond.conditioning_capabilities(flux3, _family_workflows(flux3))
+ assert c["unified_edit"] is True and c["localized_edit_modes"] == []
+ assert c["multi_image_edit"] is False and c["max_condition_images"] == 10
+
+
+def test_reference_edit_runs_the_reference_call_at_the_requested_size(fake_runtime, tmp_path):
+ import dataclasses
+
+ backend = _flux2(fake_runtime, tmp_path)
+ backend._state = dataclasses.replace(
+ backend._state, family = dataclasses.replace(backend._state.family, reference_edit = True)
+ )
+ assert backend.status()["workflows"] == ["txt2img", "reference", "edit", "inpaint"]
+ calls = {}
+ for workflow in ("reference", "edit"):
+ out = backend.generate(
+ prompt = "make it night",
+ steps = 2,
+ seed = 1,
+ width = 768,
+ height = 512,
+ workflow = workflow,
+ init_image = _png(size = (96, 64), mode = "RGB"),
+ reference_images = [_png(size = (64, 64), mode = "RGB")],
+ )
+ assert out["workflow"] == workflow
+ call = backend._state.pipe.last_kwargs
+ calls[workflow] = ([im.size for im in _images(call)], call["width"], call["height"])
+ assert calls["edit"] == calls["reference"] and calls["edit"][1:] == (768, 512)
+ with pytest.raises(ValueError, match = "localized_edit"):
+ backend.generate(
+ prompt = "p",
+ steps = 2,
+ width = 512,
+ height = 512,
+ workflow = "edit",
+ init_image = _png(mode = "RGB"),
+ localized_edit = cond.LocalizedEdit("paint", _png(mode = "L", color = (255,))),
+ )
def test_sampling_defaults_resolve_for_every_21_artifact_name():
diff --git a/studio/backend/tests/test_sd_cpp_flux3.py b/studio/backend/tests/test_sd_cpp_flux3.py
index 99a38d571b7..a77ce24d185 100644
--- a/studio/backend/tests/test_sd_cpp_flux3.py
+++ b/studio/backend/tests/test_sd_cpp_flux3.py
@@ -177,14 +177,17 @@ def _ref_png(color):
@pytest.mark.parametrize("name", ["flux.3-klein", "flux.3-klein-base"])
def test_flux3_references_run_natively_without_a_projector(monkeypatch, name):
- # sd.cpp feeds FLUX.3 references through the VAE: no vision projector, no edit marker.
+ # sd.cpp feeds FLUX.3 references through the VAE: no vision projector, no edit marker. Edit is that same call.
from PIL import Image
engine = _FakeEngine()
b = SdCppDiffusionBackend(engine = engine)
b._state = _flux3_native_state(name)
status = b.status()
- assert "reference" in status["workflows"] and "edit" not in status["workflows"]
+ assert status["workflows"] == ["txt2img", "reference", "edit"]
+ assert status["conditioning"]["unified_edit"] is True
+ assert status["conditioning"]["localized_edit_modes"] == []
+ assert status["conditioning"]["notes"] == []
seen = {}
def _capture(files, params, output_path, **kw):
@@ -202,10 +205,27 @@ def test_flux3_references_run_natively_without_a_projector(monkeypatch, name):
reference_images = [_ref_png((20, 0, 0))],
)
assert seen["refs"] == 2 and out["workflow"] == "reference"
- # An instruction edit is still not something these families do natively.
- with pytest.raises(ValueError, match = "Image editing is not available"):
+ seen.clear()
+ out = b.generate(
+ prompt = "p",
+ steps = 2,
+ workflow = "edit",
+ width = 768,
+ height = 512,
+ init_image = _ref_png((1, 1, 1)),
+ reference_images = [_ref_png((2, 2, 2))],
+ )
+ assert seen["refs"] == 2 and out["workflow"] == "edit"
+ from core.inference.diffusion_conditioning import LocalizedEdit
+
+ with pytest.raises(ValueError, match = "localized_edit"):
b.generate(
- prompt = "p", workflow = "edit", init_image = _ref_png((1, 1, 1)), width = 512, height = 512
+ prompt = "p",
+ workflow = "edit",
+ init_image = _ref_png((1, 1, 1)),
+ width = 512,
+ height = 512,
+ localized_edit = LocalizedEdit("paint", _ref_png((255, 255, 255))),
)
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index d455005f175..1511d546906 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -1820,7 +1820,7 @@ export function ImagesPage({
const [referenceResolution, setReferenceResolution] = useState<number | null>(null);
const [editSizing, setEditSizing] = useState<EditSizing>("source");
const [matchResolution, setMatchResolution] = useState(1024);
- const [localizedMode, setLocalizedMode] = useState<LocalizedEditMode | null>(null);
+ const [pickedLocalizedMode, setLocalizedMode] = useState<LocalizedEditMode | null>(null);
const [localizedLayer, setLocalizedLayer] = useState<string | null>(null);
const [localizedColor, setLocalizedColor] = useState(ANNOTATION_COLORS[0].value);
const [localizedColors, setLocalizedColors] = useState<string[]>([]);
@@ -1999,6 +1999,11 @@ export function ImagesPage({
const conditioning = !external && status?.loaded ? (status.conditioning ?? null) : null;
const sizeLimits = useMemo(() => sizeLimitsFrom(conditioning), [conditioning]);
const unifiedEdit = Boolean(conditioning?.unified_edit);
+ // Localized modes and the transparent-background prompt are Qwen-Image-2.1's; FLUX.3 shares the Edit form without them.
+ const localizedEdit = Boolean(conditioning?.localized_edit_modes?.length);
+ // A mode picked on a model that has it would be refused by one that does not.
+ const localizedMode =
+ pickedLocalizedMode && conditioning?.localized_edit_modes?.includes(pickedLocalizedMode) ? pickedLocalizedMode : null;
// Edit-only model whose Edit takes more images after the source (Qwen-Image-Edit-Plus).
const multiImageEdit = !unifiedEdit && Boolean(conditioning?.multi_image_edit);
const referenceResolutions = conditioning?.reference_resolutions ?? [];
@@ -4046,7 +4051,7 @@ export function ImagesPage({
}, []);
const applyEditSource = useCallback(
(dataUrl: string) => {
- // FLUX.2 klein and FLUX.3 edit through Reference; without this fallback the source lands hidden on Create.
+ // FLUX.2 klein edits through Reference; without this fallback the source lands hidden on Create.
const target: WorkflowId | null = isWorkflowEnabled("edit", supported)
? "edit"
: isWorkflowEnabled("transform", supported)
@@ -6876,7 +6881,7 @@ export function ImagesPage({
{editSizing === "custom" && sizeControls}
{referenceDetailControl}
{engineNotes}
- {unifiedEdit && (
+ {localizedEdit && (
<Button
type="button"
variant="secondary"