Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: encode a batch's gallery PNGs concurrently (#15909)
Daniel Han66ec3fcab4da
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.pyindex 71faee6c049..1ba69b1fddc 100644--- a/studio/backend/core/inference/image_gallery.py+++ b/studio/backend/core/inference/image_gallery.py@@ -12,8 +12,9 @@ import jsonimport osimport reimport sys+import threadingimport uuid-from collections.abc import Callable+from collections.abc import Callable, Iterable, Iteratorfrom pathlib import Pathfrom typing import Any, Optional@@ -154,19 +155,78 @@ def _engine() -> Optional[str]:return None-def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:- """Persist a PIL image with its recipe embedded; return the gallery record."""- image_id = uuid.uuid4().hex- meta = {+def _recipe(meta: dict[str, Any]) -> dict[str, Any]:+ return {# A Connection run never touched the local engine."engine": None if meta.get("model_kind") == "connection" else _engine(),"generator": _generator(),**meta,"schema_version": RECIPE_SCHEMA_VERSION,}+++def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:+ """Persist a PIL image with its recipe embedded; return the gallery record."""+ image_id = uuid.uuid4().hex+ meta = _recipe(meta)# Encoded before the folder is looked up: a Settings move during the encode would otherwise# finish first, and the image land in the folder it left.- data = _png_bytes(image, meta)+ data = _encoded_ahead(image, meta)+ if data is None:+ data = _png_bytes(image, meta)+ return _publish(image_id, meta, data)+++# Pillow releases the GIL while it filters and deflates, so a batch's PNGs encode side by side.+_MAX_ENCODE_WORKERS = 8+_ahead = threading.local()+++def _encoded_ahead(image: Any, meta: dict[str, Any]) -> Optional[bytes]:+ """The PNG ``save_many`` already encoded from this image and this exact recipe, else None."""+ pending = getattr(_ahead, "pending", None)+ entry = pending.pop(id(image), None) if pending else None+ if entry is None or entry[0] is not image or list(entry[1].items()) != list(meta.items()):+ return None+ return entry[2].result()+++def save_many(items: Iterable[tuple[Any, dict[str, Any]]]) -> Iterator[dict[str, Any]]:+ """``save`` each (image, meta) in order, with every PNG encoded concurrently up front.++ Each ``save`` still writes its file and builds its record in turn, so the bytes, the gallery+ order and what a failure leaves behind are those of a plain loop of ``save``."""+ items = list(items)+ cpus = getattr(os, "process_cpu_count", os.cpu_count)() or 1+ workers = min(len(items), cpus, _MAX_ENCODE_WORKERS)+ # One image object twice would be loaded by two threads at once.+ if workers < 2 or len({id(image) for image, _ in items}) < len(items):+ for image, meta in items:+ yield save(image, meta)+ return+ from concurrent.futures import ThreadPoolExecutor++ with ThreadPoolExecutor(max_workers = workers, thread_name_prefix = "gallery-png") as pool:+ pending = {}+ for image, meta in items:+ recipe = _recipe(meta)+ pending[id(image)] = (image, recipe, pool.submit(_png_bytes, image, recipe))+ futures = [entry[2] for entry in pending.values()]+ _ahead.pending = pending+ # The gallery sorts by mtime and back-to-back writes can tie (Windows ~15.6 ms tick): keep each strictly newer.+ _ahead.floor_ns = None+ try:+ for image, meta in items:+ yield save(image, meta)+ finally:+ if getattr(_ahead, "pending", None) is pending:+ _ahead.pending = None+ _ahead.floor_ns = None+ for future in futures:+ future.cancel()+++def _publish(image_id: str, meta: dict[str, Any], data: bytes) -> dict[str, Any]:directory = gallery_dir()final_path = directory / f"{image_id}.png"# Dotted temp (skipped by the *.png glob) then atomic rename: no truncated PNG in the listing.@@ -180,9 +240,25 @@ def save(image: Any, meta: dict[str, Any]) -> dict[str, Any]:except OSError:passraise+ if getattr(_ahead, "pending", None) is not None:+ _keep_after_previous(final_path)return _record(image_id, meta, directory = directory)+def _keep_after_previous(path: Path) -> None:+ """In a ``save_many`` batch, move ``path``'s mtime 1 us past the previous image's when it is not already later."""+ try:+ stat = path.stat()+ mtime_ns = stat.st_mtime_ns+ floor = getattr(_ahead, "floor_ns", None)+ if floor is not None and mtime_ns <= floor:+ mtime_ns = floor + 1000+ os.utime(path, ns = (stat.st_atime_ns, mtime_ns))+ _ahead.floor_ns = mtime_ns+ except OSError:+ pass++def _record(image_id: str,meta: dict[str, Any],diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 2e049285b03..f1e390d0dc8 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -48267,25 +48267,27 @@ async def _external_image_generation(created_at = time.time()def _persist() -> list[dict]:- return [- image_gallery.save(- image,- {- "prompt": body.prompt,- "width": image.width,- "height": image.height,- "steps": body.steps,- "guidance": body.guidance,- "seed": body.seed or 0,- "batch_index": index,- "batch_size": body.batch_size,- "model": external_model,- "model_kind": "connection",- "created_at": created_at,- },+ return list(+ image_gallery.save_many(+ (+ image,+ {+ "prompt": body.prompt,+ "width": image.width,+ "height": image.height,+ "steps": body.steps,+ "guidance": body.guidance,+ "seed": body.seed or 0,+ "batch_index": index,+ "batch_size": body.batch_size,+ "model": external_model,+ "model_kind": "connection",+ "created_at": created_at,+ },+ )+ for index, image in enumerate(images))- for index, image in enumerate(images)- ]+ )try:records = await asyncio.to_thread(_persist)@@ -48567,7 +48569,7 @@ async def _generate_diffusion_image_run(list_driven = bool(request.prompts or request.seeds)def _persist() -> list[dict]:- records = []+ items = []for index, image in enumerate(result["images"]):seed = (per_image_seeds[index]@@ -48580,8 +48582,8 @@ async def _generate_diffusion_image_run(if request.prompts and index < len(request.prompts)else request.prompt)- records.append(- image_gallery.save(+ items.append(+ (image,{"prompt": typed_prompt,@@ -48681,7 +48683,7 @@ async def _generate_diffusion_image_run(},))- return records+ return list(image_gallery.save_many(items))# Hold generate-progress "active" across the persist so a reload mount probe cannot refresh the gallery before these records exist._diffusion_persist_active += 1@@ -50552,6 +50554,7 @@ async def _generate_openai_images(def _persist() -> list[ImageGenerationData]:items: list[ImageGenerationData] = []+ saves = []for index, image in enumerate(result["images"]):seed = (per_image_seeds[index]@@ -50559,10 +50562,13 @@ async def _generate_openai_images(else result["seed"])# batch_seed is the base the native engine derives per-image seeds from, so restore does not double-advance.- record = image_gallery.save(- image,- {**recipe, "batch_index": index, "seed": seed, "batch_seed": result["seed"]},+ saves.append(+ (+ image,+ {**recipe, "batch_index": index, "seed": seed, "batch_seed": result["seed"]},+ ))+ for record in image_gallery.save_many(saves):if want_b64:encoded = image_gallery.image_b64(record["id"])if encoded is None: # vanished between write and read — fail the calldiff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.pyindex 2d505c250c6..c743148b686 100644--- a/studio/backend/tests/test_diffusion_routes.py+++ b/studio/backend/tests/test_diffusion_routes.py@@ -508,6 +508,23 @@ def test_generate_batch_size_persists_each_image(client):assert len(client.get("/api/inference/images/gallery").json()["images"]) == 3+def test_generate_batch_persists_through_one_save_many(client, monkeypatch):+ batches = []+ real = gallery_module.save_many++ def _spy(items):+ items = list(items)+ batches.append([meta["batch_index"] for _, meta in items])+ return real(items)++ monkeypatch.setattr(gallery_module, "save_many", _spy)+ _post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")+ resp = _post_generate(client, prompt = "p", batch_size = 3, seed = 5)+ assert resp.status_code == 200+ assert batches == [[0, 1, 2]]+ assert [i["batch_index"] for i in resp.json()["images"]] == [0, 1, 2]++def test_generate_seed_list_records_replay_from_each_own_seed(client):# a seeds list gives each image its own seed, so restore must prefer batch_seed_post_load(client, model_path = "x/z-image", gguf_filename = "q.gguf")diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.pyindex d6ec5bead19..c3c2896d44f 100644--- a/studio/backend/tests/test_image_gallery.py+++ b/studio/backend/tests/test_image_gallery.py@@ -595,18 +595,33 @@ def _recipe_keys_written() -> set[str]:for t in node.targetsif isinstance(t, ast.Name)}- for call in ast.walk(fn):- if not (- isinstance(call, ast.Call)- and isinstance(call.func, ast.Attribute)- and call.func.attr == "save"- and isinstance(call.func.value, ast.Name)- and call.func.value.id == "image_gallery"- and len(call.args) > 1- and isinstance(call.args[1], ast.Dict)- ):- continue- pending = [call.args[1]]+ calls = [+ node+ for node in ast.walk(fn)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr in ("save", "save_many")+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id == "image_gallery"+ ]+ recipes = [+ call.args[1]+ for call in calls+ if call.func.attr == "save"+ and len(call.args) > 1+ and isinstance(call.args[1], ast.Dict)+ ]+ if any(call.func.attr == "save_many" for call in calls):+ # save_many takes (image, recipe) pairs, built in the same function.+ recipes += [+ node.elts[1]+ for node in ast.walk(fn)+ if isinstance(node, ast.Tuple)+ and len(node.elts) == 2+ and isinstance(node.elts[1], ast.Dict)+ ]+ for recipe in recipes:+ pending = [recipe]while pending:d = pending.pop()for k, v in zip(d.keys, d.values):@@ -683,3 +698,110 @@ def test_parameters_chunk_carries_the_encoded_caption():text = gallery._params_text(meta)assert text.splitlines()[0] == "A tabby cat on a sunlit sill."assert gallery._params_text({**_meta(), "prompt": "a cat"}).splitlines()[0] == "a cat"+++def _batch(n = 4):+ return [+ Image.frombytes("RGB", (32, 24), bytes((i * 7 + j) % 256 for j in range(32 * 24 * 3)))+ for i in range(n)+ ]+++def _fixed_ids(monkeypatch):+ import uuid as _uuid+ ids = iter(_uuid.UUID(int = i) for i in range(1, 100))+ monkeypatch.setattr(gallery.uuid, "uuid4", lambda: next(ids))+++def _many_cpus(monkeypatch):+ monkeypatch.setattr(gallery.os, "cpu_count", lambda: 8)+ if hasattr(gallery.os, "process_cpu_count"):+ monkeypatch.setattr(gallery.os, "process_cpu_count", lambda: 8)+++def test_save_many_writes_what_a_save_loop_writes(monkeypatch, tmp_path):+ _many_cpus(monkeypatch)+ images = _batch()+ metas = [_meta(seed = i, batch_index = i) for i in range(len(images))]+ out = {}+ for name, run in (+ ("loop", lambda: [gallery.save(im, m) for im, m in zip(images, metas)]),+ ("many", lambda: list(gallery.save_many(zip(images, metas)))),+ ):+ folder = tmp_path / name+ folder.mkdir()+ monkeypatch.setattr(gallery, "gallery_dir", lambda folder = folder: folder)+ _fixed_ids(monkeypatch)+ records = run()+ out[name] = (+ [{k: v for k, v in r.items() if k != "order_at"} for r in records],+ [(folder / f"{r['id']}.png").read_bytes() for r in records],+ [r["id"] for r in gallery.list_images()],+ )+ assert out["many"] == out["loop"]+ assert len(set(out["many"][1])) == len(images)+++def test_save_many_encodes_the_batch_concurrently(monkeypatch):+ import threading++ _many_cpus(monkeypatch)+ barrier = threading.Barrier(4, timeout = 10)+ encode = gallery._png_bytes++ def _encode(image, meta):+ barrier.wait() # breaks (and fails the save) unless all four encodes run at once+ return encode(image, meta)++ monkeypatch.setattr(gallery, "_png_bytes", _encode)+ records = list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch())))+ assert [r["batch_index"] for r in records] == [0, 1, 2, 3]+++def test_save_many_keeps_the_images_before_a_failed_encode(monkeypatch):+ _many_cpus(monkeypatch)+ encode = gallery._png_bytes++ def _encode(image, meta):+ if meta["batch_index"] == 2:+ raise OSError("disk full")+ return encode(image, meta)++ monkeypatch.setattr(gallery, "_png_bytes", _encode)+ saved = []+ with pytest.raises(OSError, match = "disk full"):+ for record in gallery.save_many(+ (im, _meta(batch_index = i)) for i, im in enumerate(_batch())+ ):+ saved.append(record["batch_index"])+ assert saved == [0, 1]+ assert sorted(r["batch_index"] for r in gallery.list_images()) == [0, 1]+++def test_save_many_still_saves_through_save(monkeypatch):+ _many_cpus(monkeypatch)+ seen = []+ real_save = gallery.save++ def _save(image, meta):+ seen.append(meta["batch_index"])+ return real_save(image, meta)++ monkeypatch.setattr(gallery, "save", _save)+ list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch(3))))+ assert seen == [0, 1, 2]+++def test_save_many_keeps_batch_order_when_writes_share_an_mtime(monkeypatch):+ _many_cpus(monkeypatch)+ real_replace = os.replace++ def _replace(src, dst):+ real_replace(src, dst)+ os.utime(dst, ns = (1_700_000_000_000_000_000, 1_700_000_000_000_000_000))++ monkeypatch.setattr(gallery.os, "replace", _replace)+ records = list(gallery.save_many((im, _meta(batch_index = i)) for i, im in enumerate(_batch())))+ ranks = [r["order_at"] for r in records]+ assert ranks == sorted(set(ranks)), ranks+ assert [r["batch_index"] for r in gallery.list_images()] == [3, 2, 1, 0]