Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: edit images through an external connection's /images/edits endpoint (#17448)

oobabooga63ef0fa63040
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index d05932aac41..f8cf9672161 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -7370,6 +7370,32 @@ class ExternalProviderClient:
response.raise_for_status()
return [item["b64_json"] for item in response.json()["data"]]
+ async def create_image_edits(
+ self, prompt: str, model: str, n: int, images: list[bytes]
+ ) -> list[str]:
+ """POST PNGs to /images/edits. No ``size`` is sent: the server picks the output size."""
+ field = "image" if len(images) == 1 else "image[]"
+ headers = self._auth_headers()
+ headers.pop("Content-Type", None)
+ response = await _client().post(
+ _append_provider_path(self.base_url, "/images/edits"),
+ headers = headers,
+ # llama-swap routes on the `model` form field.
+ data = {
+ "model": model,
+ "prompt": prompt,
+ "n": str(n),
+ "response_format": "b64_json",
+ },
+ files = [
+ (field, (f"image-{index}.png", image, "image/png"))
+ for index, image in enumerate(images, start = 1)
+ ],
+ timeout = self._timeout,
+ )
+ response.raise_for_status()
+ return [item["b64_json"] for item in response.json()["data"]]
+
async def create_transcription(
self,
audio: bytes,
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index abdf1d07a9c..6a2ea1178e2 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -5582,7 +5582,10 @@ class DiffusionGenerateRequest(BaseModel):
provider_id: Optional[str] = Field(
None,
description = "[x-unsloth] Saved connection ID. When set, the prompt goes to that "
- "connection's /images/generations as `model` and the results are saved to the gallery.",
+ "connection's /images/generations as `model` and the results are saved to the gallery. "
+ "With `workflow` edit, init_image (then reference_images) and the prompt go to its "
+ "/images/edits instead. No size is sent with an edit, so width and height are not used and "
+ "the server picks the output size. No other image-input workflow runs on a connection.",
)
model: Optional[str] = Field(
None, description = "Model name on the connection named by `provider_id`."
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a9bcf5d3895..8017c485da4 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -49799,6 +49799,41 @@ def _decode_external_images(encoded: list) -> list:
return images
+# decode_b64_image's per-side limit, restated so a connection run does not import the diffusion stack.
+_EXTERNAL_EDIT_MAX_SIDE = 4096
+
+
+def _external_edit_uploads(sources: list[str]) -> list[bytes]:
+ """Each base64 or data-URL source as an upright, flattened PNG (sd-server ignores EXIF orientation)."""
+ from PIL import Image
+
+ uploads = []
+ for source in sources:
+ raw = source.strip()
+ if raw.startswith("data:"):
+ raw = raw.partition(",")[2]
+ try:
+ image = Image.open(io.BytesIO(base64.b64decode(raw)))
+ width, height = image.size
+ # Checked from the header so a huge image is never loaded.
+ oversized = max(width, height) > _EXTERNAL_EDIT_MAX_SIDE
+ if not oversized:
+ image.load()
+ image = _mcp_flattened_rgb(exif_upright(image))
+ except Exception as exc: # noqa: BLE001 - surfaced as a 400
+ # Not str(exc): Pillow's message includes the buffer's repr.
+ raise ValueError("Could not decode the image to edit.") from exc
+ if oversized:
+ raise ValueError(
+ f"Image is too large ({width}x{height}); maximum is "
+ f"{_EXTERNAL_EDIT_MAX_SIDE}px per side."
+ )
+ buffer = io.BytesIO()
+ image.save(buffer, format = "PNG", compress_level = 1)
+ uploads.append(buffer.getvalue())
+ return uploads
+
+
async def _external_image_generation(
body: DiffusionGenerateRequest, request: Request
) -> DiffusionGenerateResponse:
@@ -49817,9 +49852,10 @@ async def _external_image_generation(
status_code = 400,
detail = "model is required when using an external image connection.",
)
+ editing = body.workflow == "edit"
if (
- body.init_image
- or body.reference_images
+ (not editing and (body.init_image or body.reference_images or body.workflow))
+ or body.reference_resolution is not None
or body.loras
or body.controlnet
or body.prompts
@@ -49828,7 +49864,6 @@ async def _external_image_generation(
or body.mask_image
or body.strength is not None
or body.upscale is not None
- or body.workflow
or body.localized_edit
or body.sampler
or body.schedule_type
@@ -49844,7 +49879,12 @@ async def _external_image_generation(
):
raise HTTPException(
status_code = 400,
- detail = "An external image connection only supports text to image.",
+ detail = "An external image connection only supports text to image and image edits.",
+ )
+ if editing and not body.init_image:
+ raise HTTPException(
+ status_code = 400,
+ detail = "init_image is required to edit an image through an external connection.",
)
config = await asyncio.to_thread(providers_db.get_provider, provider_id)
@@ -49900,10 +49940,25 @@ async def _external_image_generation(
**_saved_extra_headers_kwarg(current, request),
timeout = 600.0,
)
+
+ async def _edit() -> list[str]:
+ try:
+ uploads = await asyncio.to_thread(
+ _external_edit_uploads, [body.init_image, *(body.reference_images or [])]
+ )
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc)) from None
+ return await client.create_image_edits(
+ prompt = body.prompt, model = external_model, n = body.batch_size, images = uploads
+ )
+
owner = current_account_id()
generation_started = time.perf_counter()
+ # Source normalization runs inside the tracked task, so a Stop during it cancels the edit.
task = asyncio.create_task(
- client.create_images(
+ _edit()
+ if editing
+ else client.create_images(
prompt = body.prompt,
model = external_model,
n = body.batch_size,
@@ -49929,7 +49984,8 @@ async def _external_image_generation(
raise HTTPException(
status_code = 502,
detail = (
- f"Image endpoint returned HTTP {exc.response.status_code}: {exc.response.text[:300]}"
+ f"Image {'edit ' if editing else ''}endpoint returned HTTP "
+ f"{exc.response.status_code}: {exc.response.text[:300]}"
),
) from exc
except httpx.HTTPError as exc:
@@ -49967,6 +50023,16 @@ async def _external_image_generation(
"model": external_model,
"model_kind": "connection",
"created_at": created_at,
+ # Recorded like a local edit, so Restore reopens Edit.
+ **(
+ {
+ "workflow": "edit",
+ "reference_image_count": len(body.reference_images or [])
+ or None,
+ }
+ if editing
+ else {}
+ ),
},
)
for index, image in enumerate(images)
diff --git a/studio/backend/tests/test_diffusion_inpaint_mask.py b/studio/backend/tests/test_diffusion_inpaint_mask.py
index c609f6e5042..94f4c763dec 100644
--- a/studio/backend/tests/test_diffusion_inpaint_mask.py
+++ b/studio/backend/tests/test_diffusion_inpaint_mask.py
@@ -296,7 +296,10 @@ def test_external_connection_refuses_inpaint_mask_fields(client, monkeypatch, fi
json = {"prompt": "p", "provider_id": "openai-1", "model": "gpt-image-1", **field},
)
assert response.status_code == 400, response.text
- assert response.json()["detail"] == "An external image connection only supports text to image."
+ assert (
+ response.json()["detail"]
+ == "An external image connection only supports text to image and image edits."
+ )
assert looked_up == []
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 014f2797e35..13742ce4b6a 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -3327,6 +3327,241 @@ def test_external_connection_refuses_options_it_cannot_forward(client, monkeypat
assert sent == []
+def _png_data_url(
+ size,
+ color = (30, 120, 200),
+ fmt = "PNG",
+ **save,
+):
+ import base64
+ import io
+
+ from PIL import Image
+
+ buf = io.BytesIO()
+ Image.new("RGB", size, color).save(buf, format = fmt, **save)
+ return f"data:image/{fmt.lower()};base64,{base64.b64encode(buf.getvalue()).decode()}"
+
+
+def _edit_endpoint(sent, result_size = (96, 64)):
+ """A connection whose /images/edits records each multipart request it receives."""
+ import base64
+ import email
+ import io
+
+ import httpx
+ from PIL import Image
+
+ def handler(req):
+ message = email.message_from_bytes(
+ b"Content-Type: " + req.headers["content-type"].encode() + b"\r\n\r\n" + req.content
+ )
+ fields, files = {}, []
+ for part in message.get_payload():
+ name = part.get_param("name", header = "content-disposition")
+ payload = part.get_payload(decode = True)
+ if part.get_filename() is None:
+ fields[name] = payload.decode()
+ else:
+ image = Image.open(io.BytesIO(payload))
+ files.append((name, part.get_content_type(), image.format, image.size))
+ sent.append((str(req.url), req.headers.get("authorization"), fields, files))
+ buf = io.BytesIO()
+ Image.new("RGB", result_size, (10, 200, 10)).save(buf, format = "PNG")
+ png = base64.b64encode(buf.getvalue()).decode()
+ return httpx.Response(
+ 200, json = {"created": 0, "data": [{"b64_json": png}] * int(fields.get("n", 1))}
+ )
+
+ return handler
+
+
+def test_external_connection_edits_an_image_through_images_edits(client, monkeypatch):
+ sent = []
+ _external_image_connection(monkeypatch, _edit_endpoint(sent))
+ resp = _post_generate(
+ client,
+ prompt = "make it night",
+ workflow = "edit",
+ init_image = _png_data_url((96, 64)),
+ batch_size = 2,
+ provider_id = "conn-1",
+ model = "flux2-klein",
+ )
+ assert resp.status_code == 200, resp.text
+ assert sent == [
+ (
+ "http://127.0.0.1:8000/v1/images/edits",
+ "Bearer sk-test",
+ {
+ "model": "flux2-klein",
+ "prompt": "make it night",
+ "n": "2",
+ "response_format": "b64_json",
+ },
+ [("image", "image/png", "PNG", (96, 64))],
+ )
+ ]
+ images = resp.json()["images"]
+ assert [
+ (i["prompt"], i["model"], i["model_kind"], i["workflow"], i["width"], i["height"])
+ for i in images
+ ] == [("make it night", "flux2-klein", "connection", "edit", 96, 64)] * 2
+ assert [i["reference_image_count"] for i in images] == [None, None]
+ assert len(client.get("/api/inference/images/gallery").json()["images"]) == 2
+
+
+def test_external_connection_edit_sends_every_input_image_in_order(client, monkeypatch):
+ sent = []
+ _external_image_connection(monkeypatch, _edit_endpoint(sent))
+ resp = _post_generate(
+ client,
+ prompt = "put the hat from image 2 on the cat",
+ workflow = "edit",
+ init_image = _png_data_url((96, 64)),
+ reference_images = [_png_data_url((48, 48)), _png_data_url((32, 80))],
+ provider_id = "conn-1",
+ model = "flux2-klein",
+ )
+ assert resp.status_code == 200, resp.text
+ assert [f[0] for f in sent[0][3]] == ["image[]"] * 3
+ assert [f[3] for f in sent[0][3]] == [(96, 64), (48, 48), (32, 80)]
+ assert resp.json()["images"][0]["reference_image_count"] == 2
+
+
+def test_external_connection_edit_uploads_a_rotated_photo_upright(client, monkeypatch):
+ from PIL import Image
+
+ exif = Image.Exif()
+ exif[0x0112] = 6 # stored sideways, displayed 64x96
+ sent = []
+ _external_image_connection(monkeypatch, _edit_endpoint(sent))
+ resp = _post_generate(
+ client,
+ prompt = "make it night",
+ workflow = "edit",
+ init_image = _png_data_url((96, 64), fmt = "JPEG", exif = exif),
+ provider_id = "conn-1",
+ model = "flux2-klein",
+ )
+ assert resp.status_code == 200, resp.text
+ assert sent[0][3] == [("image", "image/png", "PNG", (64, 96))]
+
+
+def test_external_connection_refuses_the_other_image_input_workflows(client, monkeypatch):
+ import httpx
+
+ sent = []
+ _external_image_connection(monkeypatch, lambda req: sent.append(req) or httpx.Response(500))
+ source = _png_data_url((64, 64))
+ refused = "An external image connection only supports text to image and image edits."
+ for extra, detail in (
+ ({"init_image": source}, refused),
+ ({"init_image": source, "workflow": "reference"}, refused),
+ ({"reference_images": [source]}, refused),
+ ({"init_image": source, "workflow": "edit", "mask_image": source}, refused),
+ ({"init_image": source, "workflow": "edit", "strength": 0.5}, refused),
+ ({"init_image": source, "workflow": "edit", "reference_resolution": 1024}, refused),
+ (
+ {"workflow": "edit"},
+ "init_image is required to edit an image through an external connection.",
+ ),
+ ):
+ resp = _post_generate(
+ client, prompt = "a sloth", provider_id = "conn-1", model = "qwen-image", **extra
+ )
+ assert resp.status_code == 400, (extra, resp.text)
+ assert resp.json()["detail"] == detail, extra
+ assert sent == []
+
+
+def test_external_connection_edit_refuses_an_unreadable_or_oversized_source(client, monkeypatch):
+ import httpx
+
+ sent = []
+ _external_image_connection(monkeypatch, lambda req: sent.append(req) or httpx.Response(500))
+ for source, reason in (
+ ("data:image/png;base64,bm90IGFuIGltYWdl", "Could not decode the image to edit."),
+ (_png_data_url((4100, 16)), "Image is too large (4100x16); maximum is 4096px per side."),
+ ):
+ resp = _post_generate(
+ client,
+ prompt = "make it night",
+ workflow = "edit",
+ init_image = source,
+ provider_id = "conn-1",
+ model = "flux2-klein",
+ )
+ assert resp.status_code == 400, resp.text
+ assert resp.json()["detail"] == reason
+ assert sent == []
+
+
+def test_stop_during_source_normalization_cancels_a_connection_edit(client, monkeypatch):
+ import asyncio
+ import threading
+
+ import httpx
+
+ sent = []
+ _external_image_connection(monkeypatch, lambda req: sent.append(req) or httpx.Response(500))
+ entered, release = threading.Event(), threading.Event()
+ real = inference_routes._external_edit_uploads
+
+ def slow_uploads(sources):
+ entered.set()
+ release.wait(5)
+ return real(sources)
+
+ monkeypatch.setattr(inference_routes, "_external_edit_uploads", slow_uploads)
+
+ async def run():
+ transport = httpx.ASGITransport(app = client.app)
+ async with httpx.AsyncClient(transport = transport, base_url = "http://t") as ac:
+ gen = asyncio.create_task(
+ ac.post(
+ "/api/inference/images/generate",
+ json = {
+ "prompt": "make it night",
+ "workflow": "edit",
+ "init_image": _png_data_url((64, 64)),
+ "provider_id": "conn-1",
+ "model": "flux2-klein",
+ },
+ )
+ )
+ assert await asyncio.to_thread(entered.wait, 5)
+ progress = (await ac.get("/api/inference/images/generate-progress")).json()
+ stop = (await ac.post("/api/inference/images/generate/cancel?target=connection")).json()
+ release.set()
+ resp = await asyncio.wait_for(gen, 5)
+ return progress, stop, resp
+
+ progress, stop, resp = asyncio.run(run())
+ assert progress["target"] == "connection"
+ assert stop == {"cancelled": True}
+ assert resp.status_code == 409
+ assert sent == []
+ assert client.get("/api/inference/images/gallery").json()["images"] == []
+
+
+def test_external_connection_without_an_edit_endpoint_is_a_502_naming_it(client, monkeypatch):
+ import httpx
+
+ _external_image_connection(monkeypatch, lambda req: httpx.Response(404, text = "Not Found"))
+ resp = _post_generate(
+ client,
+ prompt = "make it night",
+ workflow = "edit",
+ init_image = _png_data_url((64, 64)),
+ provider_id = "conn-1",
+ model = "flux2-klein",
+ )
+ assert resp.status_code == 502
+ assert resp.json()["detail"] == "Image edit endpoint returned HTTP 404: Not Found"
+ assert client.get("/api/inference/images/gallery").json()["images"] == []
+
+
def test_connection_images_are_marked_in_the_gallery(client, monkeypatch):
import base64
import io
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 7c412e2b6ce..ba1d38394d7 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -164,6 +164,7 @@ import { usePersistedToggle } from "@/hooks/use-persisted-toggle";
import { useImageEditHandoffStore } from "@/stores/image-edit-handoff";
import { isWorkflowEnabled, useImageWorkflowStore } from "./stores/image-workflow-store";
import {
+ CONNECTION_WORKFLOWS,
WORKFLOW_EXAMPLE_PROMPTS,
WORKFLOW_TABS,
recipeWorkflowLabel,
@@ -5907,7 +5908,7 @@ export function ImagesPage({
useEffect(() => {
if (external) {
- setSupported(["create"]);
+ setSupported([...CONNECTION_WORKFLOWS]);
return;
}
if (!status?.loaded) {
@@ -7079,7 +7080,8 @@ export function ImagesPage({
</div>
</Field>
)}
- {!unifiedEditActive && sizeControls}
+ {/* A connection edit sends no size. */}
+ {!unifiedEditActive && !(external && workflow === "edit") && sizeControls}
{/* First of the one-line sliders, so it takes a bigger break than the gap gives. */}
{!external && (
diff --git a/studio/frontend/src/features/images/workflows.ts b/studio/frontend/src/features/images/workflows.ts
index f374b8d53ef..79fba322e01 100644
--- a/studio/frontend/src/features/images/workflows.ts
+++ b/studio/frontend/src/features/images/workflows.ts
@@ -84,6 +84,9 @@ export const WORKFLOW_TABS: Array<{
},
];
+/** Create and Edit only: the others need a strength or mask, which servers' /images/edits do not take uniformly. */
+export const CONNECTION_WORKFLOWS: readonly WorkflowId[] = ["create", "edit"];
+
export const BACKEND_WORKFLOW_RECIPE_LABELS: Record<string, string> = {
txt2img: "Create",
img2img: "Transform",
diff --git a/studio/frontend/tests/image-connection-workflows.test.ts b/studio/frontend/tests/image-connection-workflows.test.ts
new file mode 100644
index 00000000000..d969a45448d
--- /dev/null
+++ b/studio/frontend/tests/image-connection-workflows.test.ts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { CONNECTION_WORKFLOWS, WORKFLOW_TABS } from "../src/features/images/workflows.ts";
+import { readSrc } from "./helpers/kit.ts";
+
+const PAGE = readSrc("features/images/images-page.tsx");
+
+test("a connection offers Create and Edit, and no other image workflow", () => {
+ assert.deepEqual([...CONNECTION_WORKFLOWS], ["create", "edit"]);
+ const off = WORKFLOW_TABS.map((tab) => tab.id).filter((id) => !CONNECTION_WORKFLOWS.includes(id));
+ assert.deepEqual(off, ["transform", "inpaint", "extend", "upscale", "reference"]);
+});
+
+// The page is one component, so its wiring is read from source.
+test("the Images page publishes that list while a connection is selected", () => {
+ assert.match(PAGE, /if \(external\) \{\s+setSupported\(\[\.\.\.CONNECTION_WORKFLOWS\]\);\s+return;/);
+ assert.match(PAGE, /!unifiedEditActive && !\(external && workflow === "edit"\) && sizeControls/);
+});