Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: start the FLUX.3 prompt upsampler while Enhance prompt is on (#14208)

Daniel Hana8cf93a05c1e
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py
index 49cfe982218..dba408a5898 100644
--- a/studio/backend/core/inference/diffusion_engine_router.py
+++ b/studio/backend/core/inference/diffusion_engine_router.py
@@ -553,6 +553,13 @@ def _activate(name: str, reason: Optional[str]) -> Any:
if check is not None:
check()
if engine_to_unload is not None:
+ # A launching FLUX.3 upsampler warm start holds the old engine's generation slot the unload waits on.
+ _upsampler = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if _upsampler is not None:
+ try:
+ _upsampler.cancel_warm_start()
+ except Exception as exc: # noqa: BLE001 - the start times out on its own
+ logger.warning("could not cancel a prompt upsampler warm start: %s", exc)
# Publish the new engine only AFTER the old one unloads: the evictor unloads
# get_active_diffusion_engine(), so flipping the name first would let a concurrent acquire_for evict the
# new (empty) engine while the old model frees VRAM.
diff --git a/studio/backend/core/inference/flux3_prompt_upsampler.py b/studio/backend/core/inference/flux3_prompt_upsampler.py
index 6d0129dd4e8..65b5760932c 100644
--- a/studio/backend/core/inference/flux3_prompt_upsampler.py
+++ b/studio/backend/core/inference/flux3_prompt_upsampler.py
@@ -710,6 +710,8 @@ class PromptUpsamplerSidecar:
self._stop_epoch = 0
self._stopping = 0
self._gpu_ordinal: Optional[int] = None
+ # A warm start owns the load in progress: a teardown, an unenhanced render or a Stop cancels it.
+ self._warming = False
@property
def loaded(self) -> bool:
@@ -960,7 +962,7 @@ class PromptUpsamplerSidecar:
if self._update_in_progress:
raise UpsamplerUnavailableError("llama.cpp is being updated.")
epoch = self._stop_epoch
- with self._request_lock:
+ with self._request_slot(cancel_event):
if cancel_event is not None and cancel_event.is_set():
raise UpsamplerCancelledError("Cancelled.")
with self._lock:
@@ -999,6 +1001,24 @@ class PromptUpsamplerSidecar:
self._active_requests -= 1
self._schedule_idle_locked()
+ @contextmanager
+ def _request_slot(self, cancel_event: Optional[threading.Event]) -> Iterator[None]:
+ """The request lock, waited for cancellably: a Stop on a request queued behind a warm start cancels that start
+ instead of waiting it out."""
+ while not self._request_lock.acquire(timeout = 0.2):
+ if cancel_event is not None and cancel_event.is_set():
+ if self._warming:
+ self.cancel_pending_load()
+ raise UpsamplerCancelledError("Cancelled.")
+ try:
+ yield
+ finally:
+ self._request_lock.release()
+
+ def cancel_warm_start(self) -> bool:
+ """Cancel a warm start still launching, without waiting. True when one was."""
+ return self._warming and self.cancel_pending_load()
+
def release_if_tight(self) -> bool:
"""Unload now when the server holds VRAM and the accelerator is short, so the DiT and text encoder do not
denoise next to it. True when it unloaded."""
@@ -1015,6 +1035,204 @@ class PromptUpsamplerSidecar:
self.unload(wait = False)
return True
+ @property
+ def stop_epoch(self) -> int:
+ """Bumped by every blocking unload (a GPU handover, a training start). A warm start scheduled under an older
+ epoch never starts."""
+ return self._stop_epoch
+
+ def serves(self, binary: str, model: str, mmproj: Optional[str]) -> bool:
+ """Up for these files (a vision server also serves a text-only ask)."""
+ with self._lock:
+ current = self._key if self.loaded else None
+ return (
+ current is not None
+ and current[:2] == (binary, model)
+ and (mmproj is None or current[2] == mmproj)
+ )
+
+ def warm(
+ self,
+ *,
+ binary: str,
+ model: str,
+ mmproj: Optional[str],
+ expected_epoch: Optional[int] = None,
+ ) -> str:
+ """Start the server ahead of the first enhanced run, so that run skips the ~20 s start. Never waits on a
+ request: one in flight already starts or holds the server. Declines where a call would not keep it resident
+ (training, another GPU owner, unknown or tight memory once it is up) and after any blocking unload since
+ ``expected_epoch``. Returns why it did or did not start."""
+ if self._update_in_progress:
+ return "updating"
+ epoch = self._stop_epoch if expected_epoch is None else expected_epoch
+ if not self._request_lock.acquire(blocking = False):
+ return "busy"
+ try:
+ with self._lock:
+ current = self._key if self.loaded else None
+ if (
+ current is not None
+ and current[:2] == (binary, model)
+ and (mmproj is None or current[2] == mmproj)
+ ):
+ # Already up for these files: the toggle counts as use, so the keep-alive restarts.
+ self._schedule_idle_locked()
+ return "loaded"
+ if self._stopping or epoch != self._stop_epoch:
+ return "stopped"
+ self._cancel_idle_locked()
+ # Counted as a request while it starts, as generate does: a non-blocking unload leaves it alone.
+ self._active_requests += 1
+ try:
+ return self._warm_start(binary, model, mmproj, epoch, replace = current is not None)
+ finally:
+ with self._lock:
+ self._active_requests -= 1
+ self._schedule_idle_locked()
+ except Exception as exc: # noqa: BLE001 -- a warm start is an optimisation; the run starts it itself
+ logger.warning("flux3.prompt_upsampler: warm start failed (%s)", exc)
+ return "failed"
+ finally:
+ self._request_lock.release()
+
+ def _warm_start(
+ self, binary: str, model: str, mmproj: Optional[str], epoch: int, *, replace: bool
+ ) -> str:
+ from core.inference.stt_sidecar import _training_active
+
+ # Published before the training and owner probes, as _launch does: a training start that passes them finds a
+ # load to cancel (free_prompt_upsampler_for_training reads is_loading).
+ self._load_cancel.clear()
+ with self._lock:
+ self._loading = True
+ self._warming = True
+ slot_held = None
+ try:
+ # The image model must hold the GPU: with nobody on it the server would start CPU-only and stay outside
+ # every eviction.
+ if _gpu_unowned() or not _image_model_owns_gpu() or _training_active():
+ return "gpu"
+ # The engine's generation slot, held until the server is up: a render or model transition that arrives
+ # meanwhile queues behind the start instead of overlapping it (a load cancels it first through
+ # stop_for_handover). Not taken while one runs or a load is in flight.
+ slot, why = _take_render_slot()
+ if slot is None:
+ return why
+ slot_held = slot
+ ordinal = _resident_cuda_ordinal()
+ # Unpinnable on a multi-GPU host (the native engine records its card elsewhere): the server would split
+ # over every visible card and the probe below would read the wrong one. Its first enhanced run starts it.
+ if ordinal is None and _cuda_device_count() > 1:
+ return "placement"
+ free_mib, total_mib, device = _accelerator_memory_mib(ordinal)
+ if free_mib is None or device not in ("cuda", "mps"):
+ return "memory"
+ need_mib = _footprint_mib(model, mmproj)
+ # A replaced server's memory comes back before the new one loads: free memory already excludes it.
+ reclaim_mib = 0.0
+ if replace and self._gpu_layers and self._key is not None:
+ reclaim_mib = _footprint_mib(self._key[1], self._key[2], self._gpu_layers)
+ # The same floor release_if_tight applies after a call: a server that floor would unload is not started.
+ if vram_is_tight(int(free_mib + reclaim_mib - need_mib), total_mib):
+ return "tight"
+ with self._lock:
+ if self._load_cancel.is_set() or self._stopping or epoch != self._stop_epoch:
+ return "stopped"
+ if replace:
+ # Up for other files (text-only, and an edit needs the projector): replaced only once starting.
+ self._release_locked()
+ self._spawn_server(binary, model, mmproj)
+ # Revalidate once up: a load published or a handover made while the server came up means it now serves a
+ # model that is leaving.
+ with self._lock:
+ stale = self._load_cancel.is_set() or self._stopping or epoch != self._stop_epoch
+ if stale or _active_engine_loading():
+ with self._lock:
+ self._release_locked()
+ return "stale"
+ return "started"
+ except UpsamplerCancelledError:
+ return "cancelled"
+ finally:
+ if slot_held is not None:
+ slot_held.release()
+ with self._lock:
+ self._loading = False
+ self._warming = False
+ self._starting_process = None
+
+
+def _footprint_mib(
+ model: str,
+ mmproj: Optional[str],
+ gpu_layers: Optional[int] = None,
+) -> float:
+ """What a server holds on the accelerator: its share of the GGUF (all of it by default, ``gpu_layers`` of
+ _NUM_LAYERS for a partial offload), the projector (offloaded with any GPU layer) and the KV / compute overhead."""
+
+ def _size(path: Optional[str]) -> int:
+ if not path:
+ return 0
+ with suppress(OSError):
+ return os.path.getsize(path)
+ return 0
+
+ share = 1.0 if gpu_layers is None else min(max(gpu_layers, 0), _NUM_LAYERS) / _NUM_LAYERS
+ return (_size(model) * share + _size(mmproj)) / (1024 * 1024) + _GPU_OVERHEAD_MIB
+
+
+def _cuda_device_count() -> int:
+ try:
+ import torch
+ return torch.cuda.device_count() if torch.cuda.is_available() else 0
+ except Exception: # noqa: BLE001
+ return 0
+
+
+def cancel_warm_start() -> bool:
+ """Cancel a warm start still launching (it holds the image engine's generation slot until its server is up), for
+ a teardown, a model switch or an unenhanced render that would otherwise wait it out. Never creates the sidecar,
+ never blocks."""
+ sidecar = _sidecar
+ return sidecar is not None and sidecar.cancel_warm_start()
+
+
+def _take_render_slot() -> tuple[Any, str]:
+ """(the active image engine's generation lock, acquired) or (None, why not): ``generating`` while a render or model
+ transition holds it, ``loading`` while a load is in flight (the old model still reads as loaded), ``engine``
+ when the engine cannot be read. Never blocks."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ engine = get_active_diffusion_engine()
+ lock = getattr(engine, "_generate_lock", None)
+ except Exception: # noqa: BLE001 -- fail closed
+ return None, "engine"
+ if lock is None:
+ return None, "engine"
+ if _engine_loading(engine):
+ return None, "loading"
+ if not lock.acquire(blocking = False):
+ return None, "generating"
+ # A load that began while this waited for nothing: re-read under the slot.
+ if _engine_loading(engine):
+ lock.release()
+ return None, "loading"
+ return lock, ""
+
+
+def _active_engine_loading() -> bool:
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ return _engine_loading(get_active_diffusion_engine())
+ except Exception: # noqa: BLE001 -- unreadable: treat as a load in flight
+ return True
+
+
+def _engine_loading(engine: Any) -> bool:
+ loading = getattr(engine, "_loading", None)
+ return loading is not None and not getattr(loading, "error", None)
+
def _gpu_unowned() -> bool:
try:
@@ -1142,6 +1360,56 @@ def stop_for_handover() -> None:
unload_sidecar(wait = True)
+def _resident_identity() -> Optional[tuple]:
+ """(repo_id, family) of the resident image model, else None."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ status = get_active_diffusion_engine().status()
+ except Exception: # noqa: BLE001 -- no engine reads as nothing loaded
+ return None
+ return (status.get("repo_id"), status.get("family")) if status.get("loaded") else None
+
+
+def warm_for_family(
+ family_name: Optional[str],
+ *,
+ expected_epoch: Optional[int] = None,
+ resident: Optional[tuple] = None,
+) -> str:
+ """Start the server for the resident ``family_name`` when Enhance prompt is on, in the background. The vision
+ projector comes up too when downloaded, so a later edit does not relaunch it. ``expected_epoch`` (the sidecar's
+ stop epoch read before the resident was) and ``resident`` (its (repo_id, family)) bind the start to that model: a
+ handover or model swap before the worker runs cancels it. Returns ``scheduled`` or why not."""
+ if not supports_family(family_name):
+ return "unsupported"
+ binary = find_llama_server_binary()
+ model = model_path()
+ if binary is None:
+ return "unavailable"
+ if model is None:
+ return "not_downloaded"
+ sidecar = get_sidecar()
+ mmproj = mmproj_path()
+ if sidecar.is_loading() or sidecar.serves(binary, model, mmproj):
+ # Starting, or up for these files (the call refreshes its keep-alive): nothing to wait for on a thread. A
+ # server for other files is replaced on the thread below, so the route never waits for a start.
+ return sidecar.warm(
+ binary = binary, model = model, mmproj = mmproj, expected_epoch = expected_epoch
+ )
+
+ def _run() -> None:
+ if resident is not None and _resident_identity() != resident:
+ outcome = "stale"
+ else:
+ outcome = sidecar.warm(
+ binary = binary, model = model, mmproj = mmproj, expected_epoch = expected_epoch
+ )
+ logger.info("flux3.prompt_upsampler: warm start %s", outcome)
+
+ threading.Thread(target = _run, name = "flux3-upsampler-warm", daemon = True).start()
+ return "scheduled"
+
+
def unload_sidecar(wait: bool = False) -> None:
"""Stop the server if one was ever started (the FLUX.3 model it serves was unloaded). Never creates one.
``wait`` also stops a busy one, for a GPU handover."""
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index de363cdcb56..1ad3fba8293 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -186,10 +186,22 @@ def _commit_swap_kv(unloaded) -> None:
def _evict_diffusion() -> None:
# Unload whichever engine the router has active (diffusers or native sd.cpp).
from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ # A launching upsampler warm start holds the engine's generation slot the unload waits on.
+ _cancel_prompt_upsampler_warm_start()
get_active_diffusion_engine().unload()
_stop_prompt_upsampler()
+def _cancel_prompt_upsampler_warm_start() -> None:
+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if module is None:
+ return
+ try:
+ module.cancel_warm_start()
+ except Exception as exc: # noqa: BLE001 - never block an eviction on this
+ logger.warning("gpu_arbiter: could not cancel a prompt upsampler warm start: %s", exc)
+
+
def _stop_prompt_upsampler() -> None:
"""The FLUX.3 prompt upsampler serves the image model, so it leaves with every DIFFUSION claim: an evicted, a
released or a replaced one. Only if Images ever started it."""
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 42784f7ea05..477d1fdda79 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -51240,7 +51240,7 @@ async def load_diffusion_model_gated(
stop_for_handover()
# Kicks the slow load onto a background thread and returns at once (the client polls images/load-progress).
- return engine.begin_load(
+ started = engine.begin_load(
request.model_path,
display_repo_id = request.display_repo_id,
# a load nobody asked for may not reach the hub: the switch verified locality
@@ -51268,6 +51268,10 @@ async def load_diffusion_model_gated(
gpu_ordinal = gpu_ordinal,
**_component_file_kwargs(request),
)
+ # A warm start that took the generation slot between the stop above and begin_load publishing the load would
+ # bring the old model's upsampler up beside this one: cancel it now that the load is visible.
+ _cancel_upsampler_warm_start()
+ return started
def _begin_load():
# Under the router transition lock: begin_load on a deactivated engine leaves a resident model nothing can reach.
@@ -51710,6 +51714,10 @@ async def _generate_diffusion_image_core(
submitted = request
# FLUX.3 prompt upsampling runs ahead of the run, so the sidecar can leave the VRAM before the DiT denoises.
enhancement = await _enhance_generate_prompts(backend, request)
+ if enhancement.status != "applied":
+ # A warm start still launching holds the engine's generation slot: a run that is not using the upsampler does
+ # not wait it out.
+ _cancel_upsampler_warm_start()
result = None
for attempt in range(2):
request = submitted
@@ -52697,6 +52705,8 @@ async def unload_diffusion_model(
unload = get_active_diffusion_engine().unload
if account_access.account_scope() is not None:
unload = functools.partial(unload, expected_account = current_account_id())
+ # Before the teardown waits on the generation slot a launching warm start holds.
+ _cancel_upsampler_warm_start()
status_dict = await asyncio.to_thread(unload)
# The FLUX.3 prompt upsampler serves the image model: it leaves with it instead of holding VRAM to its keep-alive.
from core.inference import flux3_prompt_upsampler as pup
@@ -52930,6 +52940,64 @@ async def prompt_upsampler_cancel(current_subject: str = Depends(get_current_sub
)
+def _cancel_upsampler_warm_start() -> None:
+ """Cancel a FLUX.3 upsampler warm start before a teardown or an unenhanced run waits on the generation slot it
+ holds. Only if Images ever started the sidecar; never blocks."""
+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if module is not None:
+ try:
+ module.cancel_warm_start()
+ except Exception as exc: # noqa: BLE001 -- the start times out on its own
+ logger.warning("flux3.prompt_upsampler: could not cancel a warm start: %s", exc)
+
+
+@studio_router.post("/images/prompt-upsampler/release")
+async def prompt_upsampler_release(current_subject: str = Depends(get_current_subject)):
+ """Enhance prompt was turned off: cancel a warm start still launching and stop an idle server instead of holding it
+ for the keep-alive. A request using it is left alone. Never an error."""
+ from core.inference import flux3_prompt_upsampler as pup
+ from auth import policy
+
+ if policy.installation_has_managed_accounts():
+ return {"released": False}
+ cancelled = await asyncio.to_thread(pup.cancel_warm_start)
+ await asyncio.to_thread(pup.unload_sidecar, False)
+ return {"released": True, "cancelled_warm": bool(cancelled)}
+
+
+@studio_router.post("/images/prompt-upsampler/warm")
+async def prompt_upsampler_warm(current_subject: str = Depends(get_current_subject)):
+ """Start the upsampler's llama-server in the background for the resident FLUX.3 model while Enhance prompt is on,
+ so the first enhanced run does not pay the start. Best effort: ``{"warm": <outcome>}``, never an error. Single-user
+ installations only (``shared`` with managed accounts)."""
+ from core.inference import flux3_prompt_upsampler as pup
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+
+ from auth import policy
+
+ # Not with managed accounts: another account's render could begin while the server starts, and a warm start
+ # holding a cross-account reservation for that long would block it instead. Their first enhanced run starts it.
+ if policy.installation_has_managed_accounts():
+ return {"warm": "shared"}
+ # Read before the resident: a handover after this point bumps it, and the start is dropped.
+ epoch = pup.get_sidecar().stop_epoch
+ try:
+ status = await asyncio.to_thread(get_active_diffusion_engine().status)
+ except Exception: # noqa: BLE001 -- no engine reads as nothing loaded
+ status = {}
+ if not status.get("loaded"):
+ return {"warm": "not_loaded"}
+ outcome = await asyncio.to_thread(
+ functools.partial(
+ pup.warm_for_family,
+ status.get("family"),
+ expected_epoch = epoch,
+ resident = (status.get("repo_id"), status.get("family")),
+ )
+ )
+ return {"warm": outcome}
+
+
@studio_router.post(
"/images/prompt-upsampler/preview", response_model = PromptUpsamplerPreviewResponse
)
diff --git a/studio/backend/tests/test_flux3_prompt_upsampler.py b/studio/backend/tests/test_flux3_prompt_upsampler.py
index 9af8b6bd6b4..177f50a1fb3 100644
--- a/studio/backend/tests/test_flux3_prompt_upsampler.py
+++ b/studio/backend/tests/test_flux3_prompt_upsampler.py
@@ -1206,3 +1206,509 @@ def test_a_managed_account_is_authorized_before_the_upsampler_runs(routes, runna
monkeypatch.setattr(account_access, "require_media_generation_access", deny)
r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
assert r.status_code == 404 and runnable.calls == []
+
+
+# ── warm start ──────────────────────────────────────────────────────────────────────────────────────────────────────
+def warmable(monkeypatch, tmp_path):
+ """A sidecar that can start: the image model holds the GPU, no training, 160 of 180 GiB free."""
+ from core.inference import gpu_arbiter
+ from core.inference import stt_sidecar
+
+ model = tmp_path / "pup.gguf"
+ model.write_bytes(b"\0" * 1024)
+ mmproj = tmp_path / "mmproj.gguf"
+ mmproj.write_bytes(b"\0" * 1024)
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)
+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: False)
+ slot = threading.Lock()
+
+ def _take_slot():
+ return (slot, "") if slot.acquire(blocking = False) else (None, "generating")
+
+ monkeypatch.setattr(pup, "_take_render_slot", _take_slot)
+ monkeypatch.setattr(pup, "_active_engine_loading", lambda: False)
+ memory = {"free": 160 * 1024, "total": 180 * 1024, "device": "cuda"}
+ monkeypatch.setattr(
+ pup,
+ "_accelerator_memory_mib",
+ lambda ordinal = None: (memory["free"], memory["total"], memory["device"]),
+ )
+ sidecar = pup.PromptUpsamplerSidecar()
+ launched = []
+
+ def _spawn(binary, model, mmproj):
+ launched.append((binary, model, mmproj))
+ sidecar._process = type("P", (), {"poll": lambda self: None})()
+ sidecar._key = (binary, model, mmproj)
+
+ monkeypatch.setattr(sidecar, "_spawn_server", _spawn)
+ return sidecar, launched, memory, str(model), str(mmproj)
+
+
+def test_warm_starts_the_server_and_arms_the_keep_alive(warmable):
+ sidecar, launched, _memory, model, mmproj = warmable
+ assert sidecar.warm(binary = "b", model = model, mmproj = mmproj) == "started"
+ assert launched == [("b", model, mmproj)] and sidecar.loaded
+ assert sidecar._active_requests == 0 and sidecar._idle_timer is not None
+ # Already up for these files (or a text-only ask of a vision server): no second start, keep-alive refreshed.
+ timer = sidecar._idle_timer
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "loaded"
+ assert len(launched) == 1 and sidecar._idle_timer is not timer
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_relaunches_for_other_files(warmable, monkeypatch):
+ sidecar, launched, _memory, model, mmproj = warmable
+ released = []
+ monkeypatch.setattr(sidecar, "_release_locked", lambda: released.append(True))
+ sidecar.warm(binary = "b", model = model, mmproj = None)
+ # An edit needs the vision projector the text-only server lacks.
+ assert sidecar.warm(binary = "b", model = model, mmproj = mmproj) == "started"
+ assert released == [True] and launched[-1] == ("b", model, mmproj)
+ sidecar._cancel_idle_locked()
+
+
+ "free_mib, total_mib, device, expected",
+ [
+ (
+ 12 * 1024,
+ 24 * 1024,
+ "cuda",
+ "tight",
+ ), # 12 GiB free leaves ~3 GiB once it is up: under the 6 GiB floor
+ (160 * 1024, 180 * 1024, "cuda", "started"),
+ (None, None, "cuda", "memory"),
+ (None, None, "cpu", "memory"),
+ ],
+)
+def test_warm_declines_where_a_call_would_not_keep_it(
+ warmable, monkeypatch, free_mib, total_mib, device, expected
+):
+ sidecar, launched, memory, model, _mmproj = warmable
+ memory.update(free = free_mib, total = total_mib, device = device)
+ # A 6.5 GB model: the size the shipped UD-Q4_K_XL file has.
+ real = pup.os.path.getsize
+ monkeypatch.setattr(
+ pup.os.path, "getsize", lambda path: 6_500_000_000 if path == model else real(path)
+ )
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == expected
+ assert bool(launched) == (expected == "started")
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_declines_without_the_gpu(warmable, monkeypatch):
+ from core.inference import gpu_arbiter
+ from core.inference import stt_sidecar
+
+ sidecar, launched, _memory, model, _mmproj = warmable
+ for owner in (None, gpu_arbiter.CHAT):
+ monkeypatch.setattr(gpu_arbiter, "_owner", owner)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "gpu"
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)
+ monkeypatch.setattr(stt_sidecar, "_training_active", lambda: True)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "gpu"
+ assert launched == []
+
+
+def test_warm_never_waits_on_a_request_or_starts_during_a_stop(warmable):
+ sidecar, launched, _memory, model, _mmproj = warmable
+ sidecar._request_lock.acquire()
+ try:
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "busy"
+ finally:
+ sidecar._request_lock.release()
+ sidecar._stopping = 1
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "stopped"
+ sidecar._stopping = 0
+ sidecar._update_in_progress = True
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "updating"
+ assert launched == []
+
+
+def test_warm_reports_a_cancelled_or_failed_start(warmable, monkeypatch):
+ sidecar, _launched, _memory, model, _mmproj = warmable
+
+ def _cancelled(*_a):
+ raise pup.UpsamplerCancelledError("Cancelled.")
+
+ monkeypatch.setattr(sidecar, "_spawn_server", _cancelled)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "cancelled"
+ monkeypatch.setattr(sidecar, "_spawn_server", lambda *_a: 1 / 0)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "failed"
+ assert sidecar._active_requests == 0 and not sidecar._request_lock.locked()
+ assert not sidecar.is_loading()
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_is_visible_to_training_before_its_probe(warmable, monkeypatch):
+ """A training start between the eligibility probe and the spawn sees a load and cancels it (and its unload bumps
+ the epoch), so exactly one of training and the warm start proceeds."""
+ from core.inference import stt_sidecar
+
+ sidecar, launched, _memory, model, _mmproj = warmable
+ seen = []
+
+ def _training_probe():
+ seen.append(sidecar.is_loading())
+ # Training admission runs now: free_prompt_upsampler_for_training finds the load and cancels it.
+ sidecar.cancel_pending_load()
+ return False
+
+ monkeypatch.setattr(stt_sidecar, "_training_active", _training_probe)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "stopped"
+ assert seen == [True] and launched == [] and not sidecar.is_loading()
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_is_bound_to_the_epoch_it_was_scheduled_under(warmable):
+ sidecar, launched, _memory, model, _mmproj = warmable
+ epoch = sidecar.stop_epoch
+ sidecar.unload(wait = True) # a GPU handover between the route and the worker
+ assert sidecar.warm(binary = "b", model = model, mmproj = None, expected_epoch = epoch) == "stopped"
+ assert launched == []
+ assert (
+ sidecar.warm(binary = "b", model = model, mmproj = None, expected_epoch = sidecar.stop_epoch)
+ == "started"
+ )
+ sidecar._cancel_idle_locked()
+
+
+def test_a_declined_warm_keeps_the_running_server(warmable, monkeypatch):
+ sidecar, launched, memory, model, mmproj = warmable
+ released = []
+ sidecar.warm(binary = "b", model = model, mmproj = None)
+ monkeypatch.setattr(sidecar, "_release_locked", lambda: released.append(True))
+ memory.update(free = None)
+ assert sidecar.warm(binary = "b", model = model, mmproj = mmproj) == "memory"
+ assert released == [] and len(launched) == 1
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_for_family_runs_in_the_background(monkeypatch):
+ calls = []
+
+ class _Sidecar:
+ loaded = False
+
+ def is_loading(self):
+ return False
+
+ def serves(self, binary, model, mmproj):
+ return False
+
+ def warm(self, **kwargs):
+ calls.append(kwargs)
+ return "started"
+
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: None)
+ monkeypatch.setattr(pup, "mmproj_path", lambda: "/m/mmproj.gguf")
+ monkeypatch.setattr(pup, "get_sidecar", lambda: _Sidecar())
+ resident = {"value": ("unsloth/flux", "flux.3-klein-base")}
+ monkeypatch.setattr(pup, "_resident_identity", lambda: resident["value"])
+ assert pup.warm_for_family("flux.2-klein") == "unsupported"
+ assert pup.warm_for_family("flux.3-klein") == "not_downloaded"
+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")
+ assert pup.warm_for_family("flux.3-klein-base") == "scheduled"
+ deadline = time.monotonic() + 5
+ while not calls and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert calls == [
+ {
+ "binary": "/bin/llama-server",
+ "model": "/m/pup.gguf",
+ "mmproj": "/m/mmproj.gguf",
+ "expected_epoch": None,
+ }
+ ]
+ # The model swapped before the worker ran: it never warms for the old one.
+ resident["value"] = ("unsloth/other", "z-image")
+ assert (
+ pup.warm_for_family("flux.3-klein-base", resident = ("unsloth/flux", "flux.3-klein-base"))
+ == "scheduled"
+ )
+ time.sleep(0.3)
+ assert len(calls) == 1
+
+
+def test_warm_route_warms_the_resident_family(routes, monkeypatch):
+ client, backend, _saved = routes
+ seen = []
+ monkeypatch.setattr(
+ pup, "warm_for_family", lambda family, **kw: seen.append((family, kw)) or "scheduled"
+ )
+ r = client.post("/api/inference/images/prompt-upsampler/warm")
+ assert r.status_code == 200 and r.json() == {"warm": "scheduled"}
+ assert seen == [
+ (
+ "flux.3-klein",
+ {
+ "expected_epoch": pup.get_sidecar().stop_epoch,
+ "resident": ("unsloth/flux", "flux.3-klein"),
+ },
+ )
+ ]
+ monkeypatch.setattr(backend, "status", lambda: {"loaded": False})
+ assert client.post("/api/inference/images/prompt-upsampler/warm").json() == {
+ "warm": "not_loaded"
+ }
+ assert len(seen) == 1
+
+
+def test_warm_route_skips_installations_with_managed_accounts(routes, monkeypatch):
+ from auth import policy
+
+ client, _backend, _saved = routes
+ seen = []
+ monkeypatch.setattr(
+ pup, "warm_for_family", lambda family, **kw: seen.append(family) or "scheduled"
+ )
+ monkeypatch.setattr(policy, "installation_has_managed_accounts", lambda: True)
+ assert client.post("/api/inference/images/prompt-upsampler/warm").json() == {"warm": "shared"}
+ assert seen == []
+
+
+def test_a_replacement_warm_credits_the_server_it_replaces(warmable, monkeypatch):
+ """Free memory already excludes the text-only server a vision warm replaces, so its footprint counts back."""
+ sidecar, launched, memory, model, mmproj = warmable
+ real = pup.os.path.getsize
+ monkeypatch.setattr(
+ pup.os.path, "getsize", lambda path: 6_500_000_000 if path == model else real(path)
+ )
+
+ def _release():
+ sidecar._process = sidecar._key = sidecar._gpu_layers = None
+
+ monkeypatch.setattr(sidecar, "_release_locked", _release)
+ sidecar.warm(binary = "b", model = model, mmproj = None)
+ sidecar._gpu_layers = 99
+ # 24 GiB card, 9 GiB free with the text-only server resident: ~17 GiB free once it goes, ~8.5 once replaced.
+ memory.update(free = 9 * 1024, total = 24 * 1024)
+ assert sidecar.warm(binary = "b", model = model, mmproj = mmproj) == "started"
+ assert launched[-1] == ("b", model, mmproj)
+ # Without a resident to reclaim, the same free memory is too tight.
+ sidecar._release_locked()
+ assert sidecar.warm(binary = "b", model = model, mmproj = mmproj) == "tight"
+ sidecar._cancel_idle_locked()
+
+
+def test_warm_declines_beside_a_render_and_holds_the_slot_while_it_starts(warmable, monkeypatch):
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(pup, "_take_render_slot", lambda: (None, "generating"))
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "generating"
+ assert launched == [] and not sidecar.is_loading()
+ # While the server starts, the engine's generation slot is held: a render queues behind it.
+ slot = threading.Lock()
+ held = []
+ monkeypatch.setattr(pup, "_take_render_slot", lambda: (slot.acquire(blocking = False) and slot, ""))
+ real_spawn = sidecar._spawn_server
+ monkeypatch.setattr(
+ sidecar, "_spawn_server", lambda *a: held.append(slot.locked()) or real_spawn(*a)
+ )
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"
+ assert held == [True] and not slot.locked()
+ sidecar._cancel_idle_locked()
+
+
+def test_render_slot_is_never_taken_beside_a_render_or_a_load(monkeypatch):
+ import core.inference.diffusion_engine_router as engine_router
+
+ class _Engine:
+ def __init__(self):
+ self._generate_lock = threading.Lock()
+ self._loading = None
+
+ engine = _Engine()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: engine)
+ slot, why = pup._take_render_slot()
+ assert slot is engine._generate_lock and why == "" and slot.locked()
+ # Held (a render, or a model transition): declined without waiting.
+ assert pup._take_render_slot() == (None, "generating")
+ slot.release()
+ # A replacement load in flight: the old model still reads as loaded, so the load itself is what counts.
+ engine._loading = type("L", (), {"error": None})()
+ assert pup._take_render_slot() == (None, "loading") and not engine._generate_lock.locked()
+ engine._loading = type("L", (), {"error": "boom"})()
+ slot, _ = pup._take_render_slot()
+ assert slot is not None
+ slot.release()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: 1 / 0)
+ assert pup._take_render_slot() == (None, "engine")
+
+
+def test_a_replacement_warm_runs_on_the_background_thread(monkeypatch):
+ """A server up for other files is replaced off the route: only a compatible one is refreshed inline."""
+ calls, threads = [], []
+
+ class _Sidecar:
+ compatible = False
+
+ def is_loading(self):
+ return False
+
+ def serves(self, binary, model, mmproj):
+ return self.compatible
+
+ def warm(self, **kwargs):
+ calls.append(threading.current_thread().name)
+ return "loaded" if self.compatible else "started"
+
+ sidecar = _Sidecar()
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")
+ monkeypatch.setattr(pup, "mmproj_path", lambda: "/m/mmproj.gguf")
+ monkeypatch.setattr(pup, "get_sidecar", lambda: sidecar)
+ monkeypatch.setattr(pup, "_resident_identity", lambda: None)
+ assert pup.warm_for_family("flux.3-klein") == "scheduled"
+ deadline = time.monotonic() + 5
+ while not calls and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert calls == ["flux3-upsampler-warm"]
+ sidecar.compatible = True
+ assert pup.warm_for_family("flux.3-klein") == "loaded"
+ assert calls[-1] == threading.current_thread().name
+
+
+def test_a_partial_offload_credits_only_its_resident_share(tmp_path):
+ model = tmp_path / "m.gguf"
+ model.write_bytes(b"\0" * (32 * 1024 * 1024)) # 32 MiB: 1 MiB per layer
+ full = pup._footprint_mib(str(model), None)
+ assert full == 32 + pup._GPU_OVERHEAD_MIB
+ assert pup._footprint_mib(str(model), None, 99) == full
+ assert pup._footprint_mib(str(model), None, 8) == 8 + pup._GPU_OVERHEAD_MIB
+
+
+
+def test_warm_declines_where_it_cannot_be_pinned(warmable, monkeypatch):
+ sidecar, launched, _memory, model, _mmproj = warmable
+ monkeypatch.setattr(pup, "_resident_cuda_ordinal", lambda: None)
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 2)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "placement"
+ # One visible card needs no pin.
+ monkeypatch.setattr(pup, "_cuda_device_count", lambda: 1)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "started"
+ assert len(launched) == 1
+ sidecar._cancel_idle_locked()
+
+
+def test_a_stop_on_a_request_queued_behind_a_warm_start_cancels_it(warmable, monkeypatch):
+ sidecar, _launched, _memory, model, _mmproj = warmable
+ started, release = threading.Event(), threading.Event()
+
+ def _slow_spawn(binary, model, mmproj):
+ started.set()
+ # A real start polls the cancel event while llama-server comes up.
+ while not sidecar._load_cancel.is_set() and not release.is_set():
+ time.sleep(0.01)
+ if sidecar._load_cancel.is_set():
+ raise pup.UpsamplerCancelledError("cancelled")
+
+ monkeypatch.setattr(sidecar, "_spawn_server", _slow_spawn)
+ outcomes = []
+ warm = threading.Thread(
+ target = lambda: outcomes.append(sidecar.warm(binary = "b", model = model, mmproj = None))
+ )
+ warm.start()
+ assert started.wait(5) and sidecar._warming
+ cancel = threading.Event()
+ errors = []
+
+ def _queued():
+ try:
+ sidecar.generate({}, binary = "b", model = model, mmproj = None, cancel_event = cancel)
+ except pup.UpsamplerCancelledError as exc:
+ errors.append(exc)
+
+ queued = threading.Thread(target = _queued)
+ queued.start()
+ time.sleep(0.3)
+ cancel.set() # Stop pressed while the request waits on the warm start
+ queued.join(5)
+ warm.join(5)
+ release.set()
+ assert len(errors) == 1 and outcomes == ["cancelled"]
+ assert not sidecar._warming and not sidecar._request_lock.locked()
+ sidecar._cancel_idle_locked()
+
+
+def test_cancel_warm_start_only_touches_a_warm_start(monkeypatch):
+ sidecar = pup.PromptUpsamplerSidecar()
+ monkeypatch.setattr(pup, "_sidecar", sidecar)
+ assert pup.cancel_warm_start() is False
+ # A request's own start is not a warm start: a teardown stops it through stop_for_handover as before.
+ sidecar._loading = True
+ assert pup.cancel_warm_start() is False and not sidecar._load_cancel.is_set()
+ sidecar._warming = True
+ assert pup.cancel_warm_start() is True and sidecar._load_cancel.is_set()
+ monkeypatch.setattr(pup, "_sidecar", None)
+ assert pup.cancel_warm_start() is False
+
+
+def test_teardowns_cancel_a_warm_start_before_they_unload(routes, monkeypatch):
+ import sys as _sys
+
+ from core.inference import gpu_arbiter
+
+ client, backend, _saved = routes
+ order = []
+ monkeypatch.setattr(pup, "cancel_warm_start", lambda: order.append("cancel") or True)
+ monkeypatch.setattr(pup, "stop_for_handover", lambda: order.append("stop"))
+ monkeypatch.setitem(_sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ monkeypatch.setattr(
+ backend, "unload", lambda **_: order.append("unload") or {"loaded": False}, raising = False
+ )
+ assert client.post("/api/inference/images/unload", json = {}).status_code == 200
+ assert order[:2] == ["cancel", "unload"]
+ order.clear()
+ gpu_arbiter._evict_diffusion()
+ assert order == ["cancel", "unload", "stop"]
+
+
+def test_an_unenhanced_run_does_not_wait_out_a_warm_start(routes, runnable, monkeypatch):
+ import sys as _sys
+
+ client, backend, _saved = routes
+ cancels = []
+ monkeypatch.setattr(pup, "cancel_warm_start", lambda: cancels.append(True) or False)
+ monkeypatch.setitem(_sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ r = client.post(
+ "/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4, "enhance_prompt": False}
+ )
+ assert r.status_code == 200, r.text
+ assert cancels == [True]
+ # An enhanced run used the server the warm start brought up: nothing to cancel.
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a dog", "steps": 4})
+ assert r.status_code == 200, r.text
+ assert cancels == [True]
+
+
+def test_a_load_published_while_the_server_came_up_drops_it(warmable, monkeypatch):
+ """The window between a load's stop_for_handover and begin_load publishing it: the started server is released."""
+ sidecar, launched, _memory, model, _mmproj = warmable
+ released = []
+ real_release = sidecar._release_locked
+
+ def _release():
+ released.append(True)
+ sidecar._process = sidecar._key = None
+
+ monkeypatch.setattr(sidecar, "_release_locked", _release)
+ monkeypatch.setattr(pup, "_active_engine_loading", lambda: True)
+ assert sidecar.warm(binary = "b", model = model, mmproj = None) == "stale"
+ assert launched and released == [True] and not sidecar.loaded
+ monkeypatch.setattr(sidecar, "_release_locked", real_release)
+ sidecar._cancel_idle_locked()
+
+
+def test_release_route_cancels_a_warm_start_and_stops_an_idle_server(routes, monkeypatch):
+ client, _backend, _saved = routes
+ calls = []
+ monkeypatch.setattr(pup, "cancel_warm_start", lambda: calls.append("cancel") or True)
+ monkeypatch.setattr(pup, "unload_sidecar", lambda wait = False: calls.append(("unload", wait)))
+ r = client.post("/api/inference/images/prompt-upsampler/release")
+ assert r.status_code == 200 and r.json() == {"released": True, "cancelled_warm": True}
+ # Non-blocking: a request still using the server keeps it.
+ assert calls == ["cancel", ("unload", False)]
+
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index a837d734aef..1152f63e38e 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -1703,6 +1703,14 @@ export function ImagesPage({
const seededResident = useRef<string | null>(null);
const [busy, setBusy] = useState<Busy>(null);
+ // Bumped when a load ends: a reload of the same repo and file (other precision, memory or LoRA settings) stops the
+ // upsampler on the backend, so Enhance prompt warms it again.
+ const [loadEpoch, setLoadEpoch] = useState(0);
+ const wasLoading = useRef(false);
+ useEffect(() => {
+ if (wasLoading.current && busy !== "loading") setLoadEpoch((n) => n + 1);
+ wasLoading.current = busy === "loading";
+ }, [busy]);
const [genDone, setGenDone] = useState<number | null>(null);
const [stopping, setStopping] = useState(false);
const [genStep, setGenStep] = useState<DiffusionGenerateProgress | null>(null);
@@ -5863,6 +5871,7 @@ export function ImagesPage({
<EnhancePromptControls
enabled={enhanceOn}
onEnabledChange={changeEnhancePrompt}
+ modelKey={`${status?.repo_id ?? ""}|${status?.gguf_filename ?? ""}|${loadEpoch}`}
prompt={prompt}
previewRequest={() => {
// Edit and Reference condition on their images, so the caption describes them too.
diff --git a/studio/frontend/src/features/images/lib/prompt-enhance.ts b/studio/frontend/src/features/images/lib/prompt-enhance.ts
index cd5c819ac7a..28a37609396 100644
--- a/studio/frontend/src/features/images/lib/prompt-enhance.ts
+++ b/studio/frontend/src/features/images/lib/prompt-enhance.ts
@@ -10,6 +10,8 @@ export const PROMPT_UPSAMPLER_ROUTES = {
download: "/api/inference/images/prompt-upsampler/download",
cancel: "/api/inference/images/prompt-upsampler/cancel",
preview: "/api/inference/images/prompt-upsampler/preview",
+ warm: "/api/inference/images/prompt-upsampler/warm",
+ release: "/api/inference/images/prompt-upsampler/release",
} as const;
export const ENHANCE_PROMPT_LABEL = "Enhance prompt";
@@ -136,6 +138,28 @@ export function promptUpsamplerCapability(
return { supported, defaultOn: supported && status?.prompt_upsampler_default === true };
}
+/** Whether to start the upsampler ahead of the first enhanced run: Enhance on, its files downloaded, no download
+ * running. The backend decides the rest (GPU owner, free memory) and answers idempotently. */
+export function shouldWarmUpsampler(
+ enabled: boolean,
+ status: Pick<PromptUpsamplerStatus, "supported" | "downloaded" | "download"> | null,
+): boolean {
+ return enabled && status !== null && status.supported && status.downloaded && !status.download.downloading;
+}
+
+/** Warm outcomes that clear on their own (a render, a load, training or a GPU handover in progress, a request
+ * holding the server): the panel asks again a little later. Memory, placement and setup outcomes do not. */
+export const WARM_RETRY_OUTCOMES: ReadonlySet<string> = new Set([
+ "generating",
+ "loading",
+ "gpu",
+ "busy",
+ "stopped",
+ "cancelled",
+]);
+export const WARM_RETRY_DELAY_MS = 15_000;
+export const WARM_RETRY_LIMIT = 8;
+
/** The toggle's value: a one-off restore override, else the stored per-family choice, else the family default. */
export function resolveEnhancePrompt(args: {
capability: PromptUpsamplerCapability;
diff --git a/studio/frontend/src/features/images/prompt-enhance-panel.tsx b/studio/frontend/src/features/images/prompt-enhance-panel.tsx
index 51c812a3a62..3addfb0053e 100644
--- a/studio/frontend/src/features/images/prompt-enhance-panel.tsx
+++ b/studio/frontend/src/features/images/prompt-enhance-panel.tsx
@@ -21,7 +21,9 @@ import {
cancelPromptUpsamplerDownload,
getPromptUpsamplerStatus,
previewEnhancedPrompt,
+ releasePromptUpsampler,
startPromptUpsamplerDownload,
+ warmPromptUpsampler,
} from "./prompt-upsampler-api";
import {
ENHANCE_PROMPT_HINT,
@@ -33,6 +35,10 @@ import {
clearPreviewDigests,
previewSignature,
upsamplerDownloadFraction,
+ shouldWarmUpsampler,
+ WARM_RETRY_DELAY_MS,
+ WARM_RETRY_LIMIT,
+ WARM_RETRY_OUTCOMES,
} from "./lib/prompt-enhance";
/** Upsampler status, polled every second while a download runs and once otherwise. */
@@ -178,11 +184,14 @@ function UpsamplerDownloadNotice({
export function EnhancePromptControls({
enabled,
onEnabledChange,
+ modelKey,
prompt,
previewRequest,
onUseCaption,
}: {
enabled: boolean;
+ /** The resident model and its load; a switch or reload warms the upsampler for it. */
+ modelKey: string;
onEnabledChange: (on: boolean) => void;
prompt: string;
/** The preview body for the current form (prompt, reference images, variant). */
@@ -199,6 +208,37 @@ export function EnhancePromptControls({
wasDownloading.current = now;
}, [status?.download.downloading, refresh]);
+ // Start the upsampler while Enhance is on, so the first enhanced run does not wait ~20 s for it. A decline that
+ // clears on its own (a render, a load) is asked again a few times.
+ const warm = shouldWarmUpsampler(enabled, status);
+ useEffect(() => {
+ if (!warm) return;
+ const controller = new AbortController();
+ let timer: number | undefined;
+ let attempts = 0;
+ const attempt = () => {
+ warmPromptUpsampler(controller.signal)
+ .then((outcome) => {
+ if (controller.signal.aborted || !WARM_RETRY_OUTCOMES.has(outcome)) return;
+ if (++attempts >= WARM_RETRY_LIMIT) return;
+ timer = window.setTimeout(attempt, WARM_RETRY_DELAY_MS);
+ })
+ .catch(() => {});
+ };
+ attempt();
+ return () => {
+ controller.abort();
+ if (timer !== undefined) window.clearTimeout(timer);
+ };
+ }, [warm, modelKey]);
+
+ // Turning Enhance off stops what the warm start brought up instead of holding it for the keep-alive.
+ const wasEnabled = useRef(enabled);
+ useEffect(() => {
+ if (wasEnabled.current && !enabled) releasePromptUpsampler().catch(() => {});
+ wasEnabled.current = enabled;
+ }, [enabled]);
+
const [preview, setPreview] = useState<{ signature: string; caption: string } | null>(null);
// The stale check stops running without a preview, so its cache would otherwise keep the last uploads alive.
useEffect(() => {
diff --git a/studio/frontend/src/features/images/prompt-upsampler-api.ts b/studio/frontend/src/features/images/prompt-upsampler-api.ts
index 4ec9f6bb09a..1f5b57ba8dd 100644
--- a/studio/frontend/src/features/images/prompt-upsampler-api.ts
+++ b/studio/frontend/src/features/images/prompt-upsampler-api.ts
@@ -57,3 +57,26 @@ export async function previewEnhancedPrompt(
}),
);
}
+
+/** Start the upsampler's server in the background so the first enhanced run skips its start. Best effort: the
+ * backend's outcome ("scheduled", "loaded", or why it declined). */
+export async function warmPromptUpsampler(signal?: AbortSignal): Promise<string> {
+ // A release sent before this (Enhance off, then quickly on) is answered first, so it cannot stop what this starts.
+ await pendingRelease;
+ signal?.throwIfAborted();
+ const body = await parseJson<{ warm?: unknown }>(
+ await authFetch(PROMPT_UPSAMPLER_ROUTES.warm, { method: "POST", signal }),
+ );
+ return typeof body.warm === "string" ? body.warm : "";
+}
+
+let pendingRelease: Promise<void> = Promise.resolve();
+
+/** Enhance prompt turned off: cancel a warm start still launching and stop an idle server. Best effort. */
+export function releasePromptUpsampler(): Promise<void> {
+ const release = (async () => {
+ await parseJson<unknown>(await authFetch(PROMPT_UPSAMPLER_ROUTES.release, { method: "POST" }));
+ })();
+ pendingRelease = release.catch(() => {});
+ return release;
+}
diff --git a/studio/frontend/tests/image-prompt-enhance.test.ts b/studio/frontend/tests/image-prompt-enhance.test.ts
index a36abeb8f69..d594f9ac678 100644
--- a/studio/frontend/tests/image-prompt-enhance.test.ts
+++ b/studio/frontend/tests/image-prompt-enhance.test.ts
@@ -19,7 +19,11 @@ import {
promptUpsamplerCapability,
readEnhanceChoices,
resolveEnhancePrompt,
+ shouldWarmUpsampler,
upsamplerDownloadBytes,
+ WARM_RETRY_DELAY_MS,
+ WARM_RETRY_LIMIT,
+ WARM_RETRY_OUTCOMES,
upsamplerDownloadFraction,
upsamplerNeedsDownload,
upsamplerVariantFor,
@@ -242,3 +246,35 @@ test("a family change consumes the one-off override instead of leaving it dorman
assert.equal(overrideAfterFamilyChange(unloaded, ENHANCE_UNSUPPORTED_FAMILY), unloaded);
});
+test("the upsampler warms only while Enhance is on and its files are ready", () => {
+ const ready = parsePromptUpsamplerStatus({ supported: true, downloaded: true, mmproj_downloaded: true });
+ assert.equal(shouldWarmUpsampler(true, ready), true);
+ assert.equal(shouldWarmUpsampler(false, ready), false);
+ assert.equal(shouldWarmUpsampler(true, null), false);
+ assert.equal(shouldWarmUpsampler(true, { ...ready, downloaded: false }), false);
+ assert.equal(shouldWarmUpsampler(true, { ...ready, supported: false }), false);
+ assert.equal(shouldWarmUpsampler(true, { ...ready, download: { ...ready.download, downloading: true } }), false);
+ assert.equal(PROMPT_UPSAMPLER_ROUTES.warm, "/api/inference/images/prompt-upsampler/warm");
+ // The panel warms on the toggle, the download finishing and a model switch.
+ const panel = readSrc("features/images/prompt-enhance-panel.tsx");
+ assert.match(panel, /warmPromptUpsampler\(controller\.signal\)[\s\S]{0,600}\}, \[warm, modelKey\]\)/);
+ // Transient declines are asked again, a bounded number of times; memory and placement ones are not.
+ for (const outcome of ["generating", "loading", "gpu", "busy"]) assert.ok(WARM_RETRY_OUTCOMES.has(outcome));
+ for (const outcome of ["tight", "memory", "placement", "shared", "loaded", "scheduled"]) {
+ assert.ok(!WARM_RETRY_OUTCOMES.has(outcome), outcome);
+ }
+ assert.ok(WARM_RETRY_LIMIT * WARM_RETRY_DELAY_MS <= 5 * 60_000);
+ assert.match(panel, /WARM_RETRY_OUTCOMES\.has\(outcome\)/);
+ // Enhance turned off releases the server the warm start brought up.
+ assert.equal(PROMPT_UPSAMPLER_ROUTES.release, "/api/inference/images/prompt-upsampler/release");
+ assert.match(panel, /if \(wasEnabled\.current && !enabled\) releasePromptUpsampler\(\)/);
+ // A warm start waits for a release still in flight, so a quick off / on cannot stop the server it asked for.
+ const api = readSrc("features/images/prompt-upsampler-api.ts");
+ assert.match(api, /warmPromptUpsampler[\s\S]{0,200}await pendingRelease;[\s\S]{0,120}authFetch\(PROMPT_UPSAMPLER_ROUTES\.warm/);
+ assert.match(api, /pendingRelease = release\.catch/);
+ // A reload of the same repo and file stops the upsampler, so the key carries the load too.
+ const page = readSrc("features/images/images-page.tsx");
+ assert.match(page, /modelKey=\{`\$\{status\?\.repo_id \?\? ""\}\|\$\{status\?\.gguf_filename \?\? ""\}\|\$\{loadEpoch\}`\}/);
+ assert.match(page, /if \(wasLoading\.current && busy !== "loading"\) setLoadEpoch/);
+});
+