Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: encode a batch's gallery PNGs concurrently (#15909)

Daniel Han66ec3fcab4da
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index 71faee6c049..1ba69b1fddc 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -12,8 +12,9 @@ import json
import os
import re
import sys
+import threading
import uuid
-from collections.abc import Callable
+from collections.abc import Callable, Iterable, Iterator
from pathlib import Path
from typing import Any, Optional
@@ -154,19 +155,78 @@ def _engine() -> Optional[str]:
return None
-def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:
- """Persist a PIL image with its recipe embedded; return the gallery record."""
- image_id = uuid.uuid4().hex
- meta = {
+def _recipe(meta: dict[str, Any]) -> dict[str, Any]:
+ return {
# A Connection run never touched the local engine.
"engine": None if meta.get("model_kind") == "connection" else _engine(),
"generator": _generator(),
**meta,
"schema_version": RECIPE_SCHEMA_VERSION,
}
+
+
+def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:
+ """Persist a PIL image with its recipe embedded; return the gallery record."""
+ image_id = uuid.uuid4().hex
+ meta = _recipe(meta)
# Encoded before the folder is looked up: a Settings move during the encode would otherwise
# finish first, and the image land in the folder it left.
- data = _png_bytes(image, meta)
+ data = _encoded_ahead(image, meta)
+ if data is None:
+ data = _png_bytes(image, meta)
+ return _publish(image_id, meta, data)
+
+
+# Pillow releases the GIL while it filters and deflates, so a batch's PNGs encode side by side.
+_MAX_ENCODE_WORKERS = 8
+_ahead = threading.local()
+
+
+def _encoded_ahead(image: Any, meta: dict[str, Any]) -> Optional[bytes]:
+ """The PNG ``save_many`` already encoded from this image and this exact recipe, else None."""
+ pending = getattr(_ahead, "pending", None)
+ entry = pending.pop(id(image), None) if pending else None
+ if entry is None or entry[0] is not image or list(entry[1].items()) != list(meta.items()):
+ return None
+ return entry[2].result()
+
+
+def save_many(items: Iterable[tuple[Any, dict[str, Any]]]) -> Iterator[dict[str, Any]]:
+ """``save`` each (image, meta) in order, with every PNG encoded concurrently up front.
+
+ Each ``save`` still writes its file and builds its record in turn, so the bytes, the gallery
+ order and what a failure leaves behind are those of a plain loop of ``save``."""
+ items = list(items)
+ cpus = getattr(os, "process_cpu_count", os.cpu_count)() or 1
+ workers = min(len(items), cpus, _MAX_ENCODE_WORKERS)
+ # One image object twice would be loaded by two threads at once.
+ if workers < 2 or len({id(image) for image, _ in items}) < len(items):
+ for image, meta in items:
+ yield save(image, meta)
+ return
+ from concurrent.futures import ThreadPoolExecutor
+
+ with ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "gallery-png") as pool:
+ pending = {}
+ for image, meta in items:
+ recipe = _recipe(meta)
+ pending[id(image)] = (image, recipe, pool.submit(_png_bytes, image, recipe))
+ futures = [entry[2] for entry in pending.values()]
+ _ahead.pending = pending
+ # The gallery sorts by mtime and back-to-back writes can tie (Windows ~15.6 ms tick): keep each strictly newer.
+ _ahead.floor_ns = None
+ try:
+ for image, meta in items:
+ yield save(image, meta)
+ finally:
+ if getattr(_ahead, "pending", None) is pending:
+ _ahead.pending = None
+ _ahead.floor_ns = None
+ for future in futures:
+ future.cancel()
+
+
+def _publish(image_id: str, meta: dict[str, Any], data: bytes) -> dict[str, Any]:
directory = gallery_dir()
final_path = directory / f"{image_id}.png"
# Dotted temp (skipped by the *.png glob) then atomic rename: no truncated PNG in the listing.
@@ -180,9 +240,25 @@ def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:
except OSError:
pass
raise
+ if getattr(_ahead, "pending", None) is not None:
+ _keep_after_previous(final_path)
return _record(image_id, meta, directory = directory)
+def _keep_after_previous(path: Path) -> None:
+ """In a ``save_many`` batch, move ``path``'s mtime 1 us past the previous image's when it is not already later."""
+ try:
+ stat = path.stat()
+ mtime_ns = stat.st_mtime_ns
+ floor = getattr(_ahead, "floor_ns", None)
+ if floor is not None and mtime_ns <= floor:
+ mtime_ns = floor + 1000
+ os.utime(path, ns = (stat.st_atime_ns, mtime_ns))
+ _ahead.floor_ns = mtime_ns
+ except OSError:
+ pass
+
+
def _record(
image_id: str,
meta: dict[str, Any],
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 2e049285b03..f1e390d0dc8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -48267,25 +48267,27 @@ async def _external_image_generation(
created_at = time.time()
def _persist() -> list[dict]:
- return [
- image_gallery.save(
- image,
- {
- "prompt": body.prompt,
- "width": image.width,
- "height": image.height,
- "steps": body.steps,
- "guidance": body.guidance,
- "seed": body.seed or 0,
- "batch_index": index,
- "batch_size": body.batch_size,
- "model": external_model,
- "model_kind": "connection",
- "created_at": created_at,
- },
+ return list(
+ image_gallery.save_many(
+ (
+ image,
+ {
+ "prompt": body.prompt,
+ "width": image.width,
+ "height": image.height,
+ "steps": body.steps,
+ "guidance": body.guidance,
+ "seed": body.seed or 0,
+ "batch_index": index,
+ "batch_size": body.batch_size,
+ "model": external_model,
+ "model_kind": "connection",
+ "created_at": created_at,
+ },
+ )
+ for index, image in enumerate(images)
)
- for index, image in enumerate(images)
- ]
+ )
try:
records = await asyncio.to_thread(_persist)
@@ -48567,7 +48569,7 @@ async def _generate_diffusion_image_run(
list_driven = bool(request.prompts or request.seeds)
def _persist() -> list[dict]:
- records = []
+ items = []
for index, image in enumerate(result["images"]):
seed = (
per_image_seeds[index]
@@ -48580,8 +48582,8 @@ async def _generate_diffusion_image_run(
if request.prompts and index < len(request.prompts)
else request.prompt
)
- records.append(
- image_gallery.save(
+ items.append(
+ (
image,
{
"prompt": typed_prompt,
@@ -48681,7 +48683,7 @@ async def _generate_diffusion_image_run(
},
)
)
- return records
+ return list(image_gallery.save_many(items))
# Hold generate-progress "active" across the persist so a reload mount probe cannot refresh the gallery before these records exist.
_diffusion_persist_active += 1
@@ -50552,6 +50554,7 @@ async def _generate_openai_images(
def _persist() -> list[ImageGenerationData]:
items: list[ImageGenerationData] = []
+ saves = []
for index, image in enumerate(result["images"]):
seed = (
per_image_seeds[index]
@@ -50559,10 +50562,13 @@ async def _generate_openai_images(
else result["seed"]
)
# batch_seed is the base the native engine derives per-image seeds from, so restore does not double-advance.
- record = image_gallery.save(
- image,
- {**recipe, "batch_index": index, "seed": seed, "batch_seed": result["seed"]},
+ saves.append(
+ (
+ image,
+ {**recipe, "batch_index": index, "seed": seed, "batch_seed": result["seed"]},
+ )
)
+ for record in image_gallery.save_many(saves):
if want_b64:
encoded = image_gallery.image_b64(record["id"])
if encoded is None: # vanished between write and read — fail the call
diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py
index 2d505c250c6..c743148b686 100644
--- a/studio/backend/tests/test_diffusion_routes.py
+++ b/studio/backend/tests/test_diffusion_routes.py
@@ -508,6 +508,23 @@ def test_generate_batch_size_persists_each_image(client):
assert len(client.get("/api/inference/images/gallery").json()["images"]) == 3
+def test_generate_batch_persists_through_one_save_many(client, monkeypatch):
+ batches = []
+ real = gallery_module.save_many
+
+ def _spy(items):
+ items = list(items)
+ batches.append([meta["batch_index"] for _, meta in items])
+ return real(items)
+
+ monkeypatch.setattr(gallery_module, "save_many", _spy)
+ _post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")
+ resp = _post_generate(client, prompt = "p", batch_size = 3, seed = 5)
+ assert resp.status_code == 200
+ assert batches == [[0, 1, 2]]
+ assert [i["batch_index"] for i in resp.json()["images"]] == [0, 1, 2]
+
+
def test_generate_seed_list_records_replay_from_each_own_seed(client):
# a seeds list gives each image its own seed, so restore must prefer batch_seed
_post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")
diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.py
index d6ec5bead19..c3c2896d44f 100644
--- a/studio/backend/tests/test_image_gallery.py
+++ b/studio/backend/tests/test_image_gallery.py
@@ -595,18 +595,33 @@ def _recipe_keys_written() -> set[str]:
for t in node.targets
if isinstance(t, ast.Name)
}
- for call in ast.walk(fn):
- if not (
- isinstance(call, ast.Call)
- and isinstance(call.func, ast.Attribute)
- and call.func.attr == "save"
- and isinstance(call.func.value, ast.Name)
- and call.func.value.id == "image_gallery"
- and len(call.args) > 1
- and isinstance(call.args[1], ast.Dict)
- ):
- continue
- pending = [call.args[1]]
+ calls = [
+ node
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr in ("save", "save_many")
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id == "image_gallery"
+ ]
+ recipes = [
+ call.args[1]
+ for call in calls
+ if call.func.attr == "save"
+ and len(call.args) > 1
+ and isinstance(call.args[1], ast.Dict)
+ ]
+ if any(call.func.attr == "save_many" for call in calls):
+ # save_many takes (image, recipe) pairs, built in the same function.
+ recipes += [
+ node.elts[1]
+ for node in ast.walk(fn)
+ if isinstance(node, ast.Tuple)
+ and len(node.elts) == 2
+ and isinstance(node.elts[1], ast.Dict)
+ ]
+ for recipe in recipes:
+ pending = [recipe]
while pending:
d = pending.pop()
for k, v in zip(d.keys, d.values):
@@ -683,3 +698,110 @@ def test_parameters_chunk_carries_the_encoded_caption():
text = gallery._params_text(meta)
assert text.splitlines()[0] == "A tabby cat on a sunlit sill."
assert gallery._params_text({**_meta(), "prompt": "a cat"}).splitlines()[0] == "a cat"
+
+
+def _batch(n = 4):
+ return [
+ Image.frombytes("RGB", (32, 24), bytes((i * 7 + j) % 256 for j in range(32 * 24 * 3)))
+ for i in range(n)
+ ]
+
+
+def _fixed_ids(monkeypatch):
+ import uuid as _uuid
+ ids = iter(_uuid.UUID(int = i) for i in range(1, 100))
+ monkeypatch.setattr(gallery.uuid, "uuid4", lambda: next(ids))
+
+
+def _many_cpus(monkeypatch):
+ monkeypatch.setattr(gallery.os, "cpu_count", lambda: 8)
+ if hasattr(gallery.os, "process_cpu_count"):
+ monkeypatch.setattr(gallery.os, "process_cpu_count", lambda: 8)
+
+
+def test_save_many_writes_what_a_save_loop_writes(monkeypatch, tmp_path):
+ _many_cpus(monkeypatch)
+ images = _batch()
+ metas = [_meta(seed = i, batch_index = i) for i in range(len(images))]
+ out = {}
+ for name, run in (
+ ("loop", lambda: [gallery.save(im, m) for im, m in zip(images, metas)]),
+ ("many", lambda: list(gallery.save_many(zip(images, metas)))),
+ ):
+ folder = tmp_path / name
+ folder.mkdir()
+ monkeypatch.setattr(gallery, "gallery_dir", lambda folder = folder: folder)
+ _fixed_ids(monkeypatch)
+ records = run()
+ out[name] = (
+ [{k: v for k, v in r.items() if k != "order_at"} for r in records],
+ [(folder / f"{r['id']}.png").read_bytes() for r in records],
+ [r["id"] for r in gallery.list_images()],
+ )
+ assert out["many"] == out["loop"]
+ assert len(set(out["many"][1])) == len(images)
+
+
+def test_save_many_encodes_the_batch_concurrently(monkeypatch):
+ import threading
+
+ _many_cpus(monkeypatch)
+ barrier = threading.Barrier(4, timeout = 10)
+ encode = gallery._png_bytes
+
+ def _encode(image, meta):
+ barrier.wait() # breaks (and fails the save) unless all four encodes run at once
+ return encode(image, meta)
+
+ monkeypatch.setattr(gallery, "_png_bytes", _encode)
+ records = list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch())))
+ assert [r["batch_index"] for r in records] == [0, 1, 2, 3]
+
+
+def test_save_many_keeps_the_images_before_a_failed_encode(monkeypatch):
+ _many_cpus(monkeypatch)
+ encode = gallery._png_bytes
+
+ def _encode(image, meta):
+ if meta["batch_index"] == 2:
+ raise OSError("disk full")
+ return encode(image, meta)
+
+ monkeypatch.setattr(gallery, "_png_bytes", _encode)
+ saved = []
+ with pytest.raises(OSError, match = "disk full"):
+ for record in gallery.save_many(
+ (im, _meta(batch_index = i)) for i, im in enumerate(_batch())
+ ):
+ saved.append(record["batch_index"])
+ assert saved == [0, 1]
+ assert sorted(r["batch_index"] for r in gallery.list_images()) == [0, 1]
+
+
+def test_save_many_still_saves_through_save(monkeypatch):
+ _many_cpus(monkeypatch)
+ seen = []
+ real_save = gallery.save
+
+ def _save(image, meta):
+ seen.append(meta["batch_index"])
+ return real_save(image, meta)
+
+ monkeypatch.setattr(gallery, "save", _save)
+ list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch(3))))
+ assert seen == [0, 1, 2]
+
+
+def test_save_many_keeps_batch_order_when_writes_share_an_mtime(monkeypatch):
+ _many_cpus(monkeypatch)
+ real_replace = os.replace
+
+ def _replace(src, dst):
+ real_replace(src, dst)
+ os.utime(dst, ns = (1_700_000_000_000_000_000, 1_700_000_000_000_000_000))
+
+ monkeypatch.setattr(gallery.os, "replace", _replace)
+ records = list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch())))
+ ranks = [r["order_at"] for r in records]
+ assert ranks == sorted(set(ranks)), ranks
+ assert [r["batch_index"] for r in gallery.list_images()] == [3, 2, 1, 0]