Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: FLUX.3 prompt upsampling (Enhance prompt) through a llama.cpp sidecar (#14137)

Daniel Handef3451e4e17
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py
index 7a75d4d4684..06b64905740 100644
--- a/studio/backend/core/inference/diffusion_engine_router.py
+++ b/studio/backend/core/inference/diffusion_engine_router.py
@@ -933,6 +933,10 @@ def annotate_status(status: dict[str, Any]) -> dict[str, Any]:
out["supported_families"] = list(_supported_family_capabilities())
# The family override picker classes an opaque folder as a modular pipeline only for these (FLUX.3).
out["modular_families"] = list(modular_image_family_names(_supported_family_capabilities()))
+ # FLUX.3 prompt upsampling, by the loaded family (pure: no binary or cache probe on a status poll).
+ from .flux3_prompt_upsampler import family_capabilities
+
+ out.update(family_capabilities(out.get("family") if out.get("loaded") else None))
return out
diff --git a/studio/backend/core/inference/flux3_prompt_upsampler.py b/studio/backend/core/inference/flux3_prompt_upsampler.py
new file mode 100644
index 00000000000..08d875bc2c6
--- /dev/null
+++ b/studio/backend/core/inference/flux3_prompt_upsampler.py
@@ -0,0 +1,1351 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 Image Klein prompt upsampling through a llama.cpp sidecar.
+
+FLUX.3 was trained on ~250-word captions, and a short request ("a gymnast doing a backflip") renders broken
+anatomy. BFL ships a prompt upsampler (a Qwen3.5-9B fine-tune) that rewrites the request into such a caption;
+ComfyUI's FLUX.3 templates run it by default. Studio runs a GGUF of it in one ``llama-server`` on an ephemeral
+127.0.0.1 port, loaded on demand and unloaded after the dictation keep-alive (or right after the call when VRAM is
+tight, so the DiT and text encoder never denoise next to it). The diffusers graph's own ``prompt_upsampler`` step
+stays dropped (``diffusion_flux3.flux3_blocks``); this replaces it.
+
+Prompts and sampling follow ComfyUI's FLUX.3 templates. The system prompts are BFL's ``system_prompts.json``
+(black-forest-labs/flux-3-image-klein-pup at commit 8b069751), byte-identical to ``SYSTEM_PROMPTS`` in diffusers'
+``modular_pipelines/flux3/prompt_upsampler.py`` at d42ebf9c6e (huggingface/diffusers#15058). The reference
+preparation is adapted from that file's ``get_reference_size`` / ``prepare_reference`` (Apache-2.0).
+
+Weights only ever come from the unsloth GGUF repo below or a local override; nothing is fetched implicitly by a
+generate call, the UI drives the download. Any failure leaves the prompt as typed.
+"""
+
+from __future__ import annotations
+
+import collections
+import hashlib
+import http.client
+import io
+import json
+import math
+import os
+import re
+import secrets
+import subprocess
+import threading
+import time
+from contextlib import contextmanager, suppress
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Iterator, Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+# ── weights: one place for repo, files and sizes ────────────────────────────────────────────────────────────────────
+UPSAMPLER_ID = "flux-3-image-klein-pup"
+UPSAMPLER_REPO = "unsloth/flux-3-image-klein-pup-GGUF"
+UPSAMPLER_MODEL_FILES = {
+ "Q4_K_M": "flux-3-image-klein-pup-Q4_K_M.gguf",
+ "Q8_0": "flux-3-image-klein-pup-Q8_0.gguf",
+}
+UPSAMPLER_DEFAULT_QUANT = "Q4_K_M"
+UPSAMPLER_MODEL_FILE = UPSAMPLER_MODEL_FILES[UPSAMPLER_DEFAULT_QUANT]
+# Vision projector, needed only for edits (reference images).
+UPSAMPLER_MMPROJ_FILE = "mmproj-F16.gguf"
+# Expected sizes of the files as built, for the UI before the download starts (the download reads the real ones).
+UPSAMPLER_FILE_SIZES = {
+ "flux-3-image-klein-pup-Q4_K_M.gguf": 5_629_109_248,
+ "flux-3-image-klein-pup-Q8_0.gguf": 9_527_501_824,
+ "mmproj-F16.gguf": 918_165_984,
+}
+# Local files instead of the repo, for testing a build before upload.
+ENV_MODEL_OVERRIDE = "UNSLOTH_FLUX3_UPSAMPLER_GGUF"
+ENV_MMPROJ_OVERRIDE = "UNSLOTH_FLUX3_UPSAMPLER_MMPROJ"
+# Append llama-server's output to this file (debugging only).
+ENV_SERVER_LOG = "UNSLOTH_FLUX3_UPSAMPLER_SERVER_LOG"
+
+FLUX3_UPSAMPLER_FAMILIES = {"flux.3-klein": "distilled", "flux.3-klein-base": "base"}
+VARIANTS = ("distilled", "base")
+
+# ── prompts (ComfyUI FLUX.3 templates; BFL system_prompts.json, verbatim) ───────────────────────────────────────────
+SYSTEM_PROMPTS = {
+ "t2i": (
+ "You are the prompt upsampler of a text-to-image model. Expand the user's short request into ONE detailed, "
+ "self-contained caption: the exact text the image model receives. Output only the caption, with no greetings, "
+ "explanations, refusals, questions, Markdown, headings, JSON, coordinates, ids or angle-bracket tokens, and no "
+ "quotes around it. If the input is not clearly an image request, caption a visualization of its concept. The "
+ 'request may begin with a "TARGET CANVAS: W:H" line: compose for that canvas (wide spreads content sideways, '
+ "tall stacks it) and never mention the ratio or the canvas.\n"
+ "\n"
+ "Fidelity and planning: keep every explicit subject, attribute, count, spatial constraint, style cue and piece "
+ "of wording; never contradict, substitute, drop or narrow the request, and keep an abstract concept abstract. "
+ "Decide everything left open as ONE concrete, consistent plan; never offer alternatives or hedge. Never "
+ "re-express a stated constraint through another mechanism; add camera optics (depth of field, bokeh, blur) only "
+ "when asked for or implied. Without a stated style, plan an actual photograph and open with the medium "
+ 'convention ("Full-color outdoor photograph, ...", "Full-color indoor photograph, ...", "Studio '
+ 'photograph, ...", "Close-up photograph, ..."); keep "Photorealistic" for renders, and turn '
+ '"photorealistic" or "realistic" requests for photos into the photograph convention. Give a photograph a '
+ "few ordinary, non-ideal specifics drawn from the scene's own objects instead of the stock-photo ideal. Do not "
+ "invent names, brands, logos or celebrities; keep user-named ones. A long request keeps its content but not "
+ "its wording: drop appraisal and sales vocabulary, turn moods into observable physical facts, and add only new "
+ "concrete content with positions. Time of day is lighting, not rendered text.\n"
+ "\n"
+ 'Caption format: one continuous prose block in the generative voice, never "this image shows", "we can '
+ 'see" or "is visible". Lead with medium and style, then scene, lighting and camera, then the subjects with '
+ "concrete colors, materials and state. Every person in focus gets an observable profile: ethnicity, gender, "
+ 'age range, skin tone ("fair skin", "brown skin"), hair and clothing. Make the layout reconstructable from '
+ "words: frame-relative positions (frame-left, frame-right, upper, lower, foreground, midground, background), "
+ 'sentence-initial anchors ("On the left, ...", "In the background, ..."), relations (holds, faces, '
+ "behind) and a consistent order; describe up to about ten instances one by one, beyond that give the exact "
+ "count and arrangement. Out-of-focus or distant planes are shapes, colors and silhouettes, never identities or "
+ "readable text, and the background takes about a sixth of the caption. Be definite: no hedging adverbs, no "
+ "quality or emotion judgments, no closing sentence about mood; state negations as what is present. Rendered "
+ "text: transcribe every wording verbatim inside double quotes in its original language and script, keep the "
+ "line breaks of multi-line text, and give its typography, color and placement; author concrete, domain-correct "
+ "wording when the request needs text it does not supply, and name canonical structures (calendar, keyboard, "
+ "periodic table) exactly with their full layout. All prose is English; only quoted text keeps its language. "
+ "Typical length is 100 to 350 words, more for complex scenes.\n"
+ "\n"
+ "Output only the caption."
+ ),
+ "i2i": (
+ "You are the prompt upsampler of an image-editing model. You see the reference image(s), attached in order "
+ "(the first image, the second image, ...), and the user's short request. Expand it into ONE detailed, "
+ "self-contained instruction: the caption the editing model receives together with the same images. Output only "
+ "the instruction as one continuous English prose block, with no greetings, explanations, Markdown, JSON, "
+ "coordinates, ids or angle-bracket tokens. Read a vague or conversational request as the most plausible edit "
+ "of the supplied images.\n"
+ "\n"
+ "Pick the voice by task shape. When one reference stays and gets bounded changes, write an imperative editing "
+ 'instruction on it ("Remove the ...", "Change the ... to ...", "Zoom in on ..."); a whole-image '
+ 'transformation opens with its scope ("Convert the entire image into ...", "Apply a ... filter over the '
+ 'entire scene", "Horizontally flip the entire image so that ...", "Shift the camera viewpoint to ...", '
+ "then where the major elements should now be). When the references are drawn into a newly composed frame "
+ '(combining subjects, a new scene, a garment, object or style donor), write a generation request ("Create a '
+ 'photograph of ...", "Create an anime-style illustration of ...", "Generate a new image featuring ...") '
+ "that says what to take from each image, by ordinal, and what is newly generated.\n"
+ "\n"
+ "Keep every explicit edit, constraint, count, position, style cue and piece of wording, and decide what is left "
+ "open as one concrete plan. Ground everything in what is actually visible: identify each referent "
+ "unambiguously by its real appearance, never mention content that is not there, and use every image the user "
+ "attached. State every end state concretely (which color, how large, which pose, which material) and give new "
+ "content enough appearance to be generated. Cover the knock-on effects of the change (shadows, reflections, "
+ "occlusion, perspective, scale, lighting) and say what fills a removed region. A zoom-in names what stays in "
+ "frame and which neighbors are cropped out; an outpainting continues the scene in the same medium. Describe "
+ "the change, not the whole scene, and close an editing instruction with one compact clause naming what stays "
+ "unchanged; transferred people keep their identity, face and accessories unless asked otherwise. Transcribe "
+ "on-image text verbatim in its original language inside double quotes, with line breaks as \\n, and match the "
+ "original font, size and color when replacing it. Name recognizable public figures, brands and landmarks you "
+ "can see. Directions are frame-relative; use no evaluative or quality words. Typical length is 50 to 150 "
+ "words.\n"
+ "\n"
+ "Output only the instruction."
+ ),
+}
+# The user turn of an edit: the reference images, then this text.
+I2I_SCAFFOLD = (
+ "Reference images follow in order (the first image, the second image, ...).\n"
+ "User request: {instruction}\n"
+ "Output the caption."
+)
+
+# ── sampling (ComfyUI FLUX.3 templates) ─────────────────────────────────────────────────────────────────────────────
+MAX_NEW_TOKENS = 3584
+_BASE_SAMPLING = {
+ "temperature": 1.0,
+ "top_k": 1,
+ "top_p": 0.95,
+ "min_p": 0.0,
+ "seed": 0,
+}
+# (mode, variant) -> (repeat_penalty, presence_penalty)
+PENALTIES = {
+ ("t2i", "distilled"): (1.05, 1.5),
+ ("t2i", "base"): (1.0, 1.5),
+ ("i2i", "distilled"): (1.05, 0.0),
+ ("i2i", "base"): (1.0, 0.0),
+}
+# ComfyUI and BFL's reference penalize every generated token and never the prompt. llama-server feeds the prompt into
+# the penalty history unless it has `penalize_prompt` (unslothai/llama.cpp#283); with it the window spans the whole
+# caption. Older servers keep their default 64-token window: a wider one there would mostly penalize the system prompt.
+GENERATED_ONLY_PENALTIES = {"penalize_prompt": False, "repeat_last_n": 4096}
+
+# ── reference preparation (diffusers prepare_reference) ─────────────────────────────────────────────────────────────
+MAX_REFERENCES = 10
+REFERENCE_MULTIPLE = 32
+MAX_ASPECT_RATIO = 200
+MAX_PIXELS_BY_REFS = {
+ 1: 1024 * 1024,
+ 2: 724 * 724,
+ **dict.fromkeys(range(3, MAX_REFERENCES + 1), 512 * 512),
+}
+JPEG_QUALITY = 92
+# Header guard on decoded uploads, diffusion.decode_b64_image's default: a larger reference fails the generate anyway.
+_MAX_UPLOAD_SIDE = 4096
+
+# ── server ──────────────────────────────────────────────────────────────────────────────────────────────────────────
+# Holds the system prompt (~1k tokens), up to ten references (256-1024 image tokens each) and the 3584-token caption.
+# The model's own 262k default would allocate a KV cache many times the weights.
+_CONTEXT_TOKENS = 16384
+_SERVER_START_TIMEOUT_SECONDS = 180.0
+_REQUEST_TIMEOUT_SECONDS = 600.0
+_NUM_LAYERS = 32
+# KV cache (8 full-attention layers at 16k), recurrent state and compute buffers on top of the weights.
+_GPU_OVERHEAD_MIB = 2560
+# Fewer layers than this on the GPU is not worth the split.
+_MIN_GPU_LAYERS = 4
+# After a call, keep the server resident only when at least this much accelerator memory stays free; else unload it
+# before the DiT denoises.
+_TIGHT_FREE_MIB = 6144
+_TIGHT_FREE_FRACTION = 0.20
+_CACHE_ENTRIES = 128
+
+STATUS_APPLIED = "applied"
+STATUS_OFF = "off"
+STATUS_UNAVAILABLE = "unavailable"
+STATUS_FAILED = "failed"
+
+
+class UpsamplerUnavailableError(RuntimeError):
+ """The upsampler cannot run here: no llama-server, or the GGUF is not downloaded."""
+
+
+class UpsamplerNotDownloadedError(UpsamplerUnavailableError):
+ pass
+
+
+class UpsamplerCancelledError(RuntimeError):
+ pass
+
+
+# ── family defaults ─────────────────────────────────────────────────────────────────────────────────────────────────
+def supports_family(family_name: Optional[str]) -> bool:
+ return family_name in FLUX3_UPSAMPLER_FAMILIES
+
+
+def variant_for_family(family_name: Optional[str]) -> str:
+ """ "base" for a base family, else "distilled" (the frontend applies the same rule)."""
+ return "base" if "base" in (family_name or "") else "distilled"
+
+
+def resolve_enhance(family_name: Optional[str], enhance_prompt: Optional[bool]) -> bool:
+ """Whether a generate call upsamples: None is the family default (on for every FLUX.3 Klein family), and every
+ other family ignores the flag."""
+ if not supports_family(family_name):
+ return False
+ return True if enhance_prompt is None else bool(enhance_prompt)
+
+
+def family_capabilities(family_name: Optional[str]) -> dict:
+ """The two fields the loaded-model status carries."""
+ supported = supports_family(family_name)
+ return {"supports_prompt_upsampler": supported, "prompt_upsampler_default": supported}
+
+
+# ── request building ────────────────────────────────────────────────────────────────────────────────────────────────
+def get_reference_size(width: int, height: int, num_references: int) -> tuple[int, int]:
+ """Sides rounded to a multiple of 32, under the pixel budget for this many references."""
+ if max(width, height) / max(1, min(width, height)) > MAX_ASPECT_RATIO:
+ raise ValueError(
+ f"The aspect ratio of a {width}x{height} reference image exceeds {MAX_ASPECT_RATIO}:1."
+ )
+
+ def floor(value):
+ return max(
+ REFERENCE_MULTIPLE, int(math.floor(value / REFERENCE_MULTIPLE)) * REFERENCE_MULTIPLE
+ )
+
+ budget = MAX_PIXELS_BY_REFS[min(max(1, num_references), MAX_REFERENCES)]
+ w = max(REFERENCE_MULTIPLE, round(width / REFERENCE_MULTIPLE) * REFERENCE_MULTIPLE)
+ h = max(REFERENCE_MULTIPLE, round(height / REFERENCE_MULTIPLE) * REFERENCE_MULTIPLE)
+ if w * h > budget:
+ scale = math.sqrt(width * height / budget)
+ w, h = floor(width / scale), floor(height / scale)
+ if w * h > budget:
+ if w == REFERENCE_MULTIPLE:
+ h = floor(budget / w)
+ else:
+ w = floor(budget / h)
+ return w, h
+
+
+def prepare_reference(image: Any, num_references: int) -> bytes:
+ """One reference as the upsampler saw them in training, as JPEG bytes: EXIF-upright, RGB over white, LANCZOS to
+ the size budget, JPEG quality 92."""
+ from PIL import Image, ImageOps
+
+ image = ImageOps.exif_transpose(image)
+ if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info):
+ rgba = image.convert("RGBA")
+ image = Image.alpha_composite(Image.new("RGBA", rgba.size, (255, 255, 255, 255)), rgba)
+ image = image.convert("RGB")
+ size = get_reference_size(image.width, image.height, num_references)
+ if image.size != size:
+ image = image.resize(size, Image.Resampling.LANCZOS)
+ buffer = io.BytesIO()
+ image.save(buffer, format = "JPEG", quality = JPEG_QUALITY)
+ return buffer.getvalue()
+
+
+def decode_reference(data: str) -> Any:
+ """A base64 / data-URL upload as a PIL image, untouched (prepare_reference does the rest)."""
+ import base64
+ import binascii
+
+ from PIL import Image
+
+ raw = (data or "").strip()
+ if raw.startswith("data:"):
+ _, _, raw = raw.partition(",")
+ try:
+ blob = base64.b64decode(raw, validate = False)
+ except (binascii.Error, ValueError) as exc:
+ raise ValueError(f"Invalid base64 image data: {exc}") from exc
+ try:
+ image = Image.open(io.BytesIO(blob))
+ w, h = image.size
+ if w > _MAX_UPLOAD_SIDE or h > _MAX_UPLOAD_SIDE:
+ raise ValueError(f"Image is too large ({w}x{h}).")
+ image.load()
+ except ValueError:
+ raise
+ except Exception as exc: # noqa: BLE001 -- surfaced as a 400 / an unenhanced generate
+ raise ValueError(f"Could not decode image: {exc}") from exc
+ return image
+
+
+# Decoded uploads are large (an 8192x8192 RGBA is 256 MB): one at a time per request, two requests at once.
+_DECODE_SLOTS = threading.BoundedSemaphore(2)
+
+
+def prepare_references(images: Optional[list]) -> list[bytes]:
+ """Data URLs or PIL images as the upsampler's JPEG references (at most ten, in order)."""
+ items = list(images or [])[:MAX_REFERENCES]
+ prepared = []
+ with _DECODE_SLOTS:
+ for item in items:
+ image = decode_reference(item) if isinstance(item, str) else item
+ prepared.append(prepare_reference(image, len(items)))
+ del image
+ return prepared
+
+
+def references_hash(references: list[bytes]) -> str:
+ digest = hashlib.sha256()
+ for ref in references:
+ digest.update(hashlib.sha256(ref).digest())
+ return digest.hexdigest()
+
+
+def build_chat_payload(
+ prompt: str,
+ variant: str = "distilled",
+ references: Optional[list[bytes]] = None,
+ *,
+ edit: Optional[bool] = None,
+) -> dict:
+ """The /v1/chat/completions body. ``references`` are prepared JPEGs; ``edit`` forces the edit prompt without
+ images (a missing mmproj runs an edit text-only)."""
+ if variant not in VARIANTS:
+ raise ValueError(f"variant must be one of {', '.join(VARIANTS)}")
+ references = list(references or [])
+ edit = bool(references) if edit is None else bool(edit)
+ mode = "i2i" if edit else "t2i"
+ if edit:
+ import base64
+ content: list[dict] = [
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "data:image/jpeg;base64," + base64.b64encode(ref).decode("ascii")
+ },
+ }
+ for ref in references
+ ]
+ content.append({"type": "text", "text": I2I_SCAFFOLD.format(instruction = prompt)})
+ else:
+ content = [{"type": "text", "text": prompt}]
+ repeat_penalty, presence_penalty = PENALTIES[(mode, variant)]
+ return {
+ "messages": [
+ {"role": "system", "content": SYSTEM_PROMPTS[mode]},
+ {"role": "user", "content": content},
+ ],
+ **_BASE_SAMPLING,
+ "repeat_penalty": repeat_penalty,
+ "presence_penalty": presence_penalty,
+ "max_tokens": MAX_NEW_TOKENS,
+ "chat_template_kwargs": {"enable_thinking": False},
+ "stream": False,
+ }
+
+
+_THINK_BLOCK = re.compile(r"<think>.*?</think>", re.DOTALL)
+
+
+def clean_caption(text: Optional[str]) -> str:
+ """The caption without a stray (empty) think block or surrounding quotes and whitespace."""
+ text = _THINK_BLOCK.sub("", text or "").strip()
+ if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'":
+ text = text[1:-1].strip()
+ return text
+
+
+# ── weights on disk ─────────────────────────────────────────────────────────────────────────────────────────────────
+@dataclass(frozen = True)
+class _UpsamplerSpec:
+ repo: str
+ model_file: str
+ mmproj_file: str
+ label: str
+
+
+UPSAMPLER_SPEC = _UpsamplerSpec(
+ repo = UPSAMPLER_REPO,
+ model_file = UPSAMPLER_MODEL_FILE,
+ mmproj_file = UPSAMPLER_MMPROJ_FILE,
+ label = "FLUX.3 prompt upsampler",
+)
+
+
+def _env_path(name: str) -> Optional[str]:
+ value = (os.environ.get(name) or "").strip()
+ if not value:
+ return None
+ path = Path(value).expanduser()
+ return str(path) if path.is_file() else None
+
+
+def _cached_file(
+ filename: str,
+ *,
+ hub_cache: Optional[Path] = None,
+ revision: Optional[str] = None,
+) -> Optional[str]:
+ try:
+ from huggingface_hub import hf_hub_download
+
+ from core.inference.stt_sidecar import _active_hf_hub_cache
+
+ root = hub_cache if hub_cache is not None else _active_hf_hub_cache()
+ return hf_hub_download(
+ repo_id = UPSAMPLER_REPO,
+ filename = filename,
+ revision = revision,
+ local_files_only = True,
+ cache_dir = str(root),
+ )
+ except Exception:
+ return None
+
+
+def _candidate_revisions(hub_cache: Optional[Path] = None) -> list:
+ from core.inference.stt_sidecar import _fallback_revisions, _read_revision_record
+ recorded = _read_revision_record(UPSAMPLER_REPO)
+ return ([recorded] if recorded else []) + [
+ rev for rev in _fallback_revisions(UPSAMPLER_REPO, hub_cache = hub_cache) if rev != recorded
+ ]
+
+
+def _cached_at_any_revision(filename: str, hub_cache: Optional[Path] = None) -> Optional[str]:
+ for revision in _candidate_revisions(hub_cache):
+ path = _cached_file(filename, hub_cache = hub_cache, revision = revision)
+ if path is not None:
+ return path
+ return None
+
+
+def _cached_pair(hub_cache: Optional[Path] = None) -> tuple[Optional[str], Optional[str]]:
+ """The model and projector from ONE revision (the recorded one first), as the downloader pins them; a model
+ with no projector at its revision when no revision holds both."""
+ model_only = None
+ for revision in _candidate_revisions(hub_cache):
+ model = _cached_file(UPSAMPLER_MODEL_FILE, hub_cache = hub_cache, revision = revision)
+ if model is None:
+ continue
+ mmproj = _cached_file(UPSAMPLER_MMPROJ_FILE, hub_cache = hub_cache, revision = revision)
+ if mmproj is not None:
+ return model, mmproj
+ model_only = model_only or model
+ return model_only, None
+
+
+def _cached_download_paths(
+ model_id: str,
+ *,
+ hub_cache: Optional[Path] = None,
+ revision: Optional[str] = None,
+) -> Optional[tuple[str, str]]:
+ """Both repo files at one revision, or None (the download state's completion check)."""
+ if revision is None:
+ model, mmproj = _cached_pair(hub_cache)
+ else:
+ model = _cached_file(UPSAMPLER_MODEL_FILE, hub_cache = hub_cache, revision = revision)
+ mmproj = _cached_file(UPSAMPLER_MMPROJ_FILE, hub_cache = hub_cache, revision = revision)
+ if model is None or mmproj is None:
+ return None
+ return model, mmproj
+
+
+def model_path() -> Optional[str]:
+ """The upsampler GGUF: the env override, else the downloaded repo file."""
+ if (os.environ.get(ENV_MODEL_OVERRIDE) or "").strip():
+ # An override that does not exist is a typo, not a reason to read the cache instead.
+ return _env_path(ENV_MODEL_OVERRIDE)
+ return _cached_pair()[0]
+
+
+def mmproj_path() -> Optional[str]:
+ if (os.environ.get(ENV_MMPROJ_OVERRIDE) or "").strip():
+ return _env_path(ENV_MMPROJ_OVERRIDE)
+ if _overridden():
+ # Beside a local model file there is no revision to match.
+ return _cached_at_any_revision(UPSAMPLER_MMPROJ_FILE)
+ return _cached_pair()[1]
+
+
+def _overridden() -> bool:
+ return any(
+ (os.environ.get(name) or "").strip() for name in (ENV_MODEL_OVERRIDE, ENV_MMPROJ_OVERRIDE)
+ )
+
+
+def find_llama_server_binary() -> Optional[str]:
+ try:
+ from core.inference.stt_mtmd_sidecar import find_llama_server_binary as find
+ return find()
+ except Exception: # noqa: BLE001 -- no llama.cpp is "not supported", never a crash
+ return None
+
+
+def _resolve_upsampler_id(model_id: Optional[str]) -> str:
+ if (model_id or UPSAMPLER_ID) != UPSAMPLER_ID:
+ raise ValueError(f"Unknown prompt upsampler '{model_id}'.")
+ return UPSAMPLER_ID
+
+
+_download_state_lock = threading.Lock()
+_download_state = None
+
+
+def _get_download_state():
+ global _download_state
+ with _download_state_lock:
+ if _download_state is None:
+ from core.inference.stt_mtmd_sidecar import _MtmdDownloadState
+ _download_state = _MtmdDownloadState(
+ specs = {UPSAMPLER_ID: UPSAMPLER_SPEC},
+ cached_paths = _cached_download_paths,
+ resolve = _resolve_upsampler_id,
+ label = "FLUX.3 prompt upsampler",
+ )
+ return _download_state
+
+
+def start_download(hf_token: Optional[str] = None) -> str:
+ return _get_download_state().start(UPSAMPLER_ID, hf_token)
+
+
+def download_status() -> dict:
+ return _get_download_state().status()
+
+
+def cancel_download(download_id: Optional[str] = None) -> bool:
+ return _get_download_state().cancel(UPSAMPLER_ID, download_id)
+
+
+def status() -> dict:
+ """The prompt-upsampler status route body."""
+ model = model_path()
+ mmproj = mmproj_path()
+ model_name = (
+ Path(model).name if model and os.environ.get(ENV_MODEL_OVERRIDE) else UPSAMPLER_MODEL_FILE
+ )
+ size = None
+ if model:
+ with suppress(OSError):
+ size = os.path.getsize(model)
+ return {
+ "supported": find_llama_server_binary() is not None,
+ "downloaded": model is not None,
+ "loaded": get_sidecar().loaded,
+ "repo": UPSAMPLER_REPO,
+ "filename": model_name,
+ "size_bytes": size or UPSAMPLER_FILE_SIZES.get(UPSAMPLER_MODEL_FILE),
+ "mmproj_filename": UPSAMPLER_MMPROJ_FILE,
+ "mmproj_size_bytes": UPSAMPLER_FILE_SIZES.get(UPSAMPLER_MMPROJ_FILE),
+ "mmproj_downloaded": mmproj is not None,
+ # Local files from the env overrides: the repo download is not offered beside them.
+ "override": _overridden(),
+ "download": download_status(),
+ }
+
+
+# ── accelerator memory ──────────────────────────────────────────────────────────────────────────────────────────────
+def _resident_cuda_ordinal() -> Optional[int]:
+ """The CUDA card the resident image model sits on, else None."""
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ ordinal = getattr(
+ getattr(get_active_diffusion_engine(), "_state", None), "placed_ordinal", None
+ )
+ return None if ordinal is None else int(ordinal)
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _child_device_mask(ordinal: Optional[int]) -> dict:
+ """Pin llama-server to the image model's card: left alone it splits layers over every visible GPU, onto a card the
+ memory probe never read."""
+ if ordinal is None:
+ return {}
+ try:
+ import torch
+
+ if torch.cuda.device_count() <= 1:
+ return {}
+ var = "CUDA_VISIBLE_DEVICES"
+ if getattr(torch.version, "hip", None) and os.environ.get(var) is None:
+ var = "HIP_VISIBLE_DEVICES"
+ parent = os.environ.get(var)
+ if parent is None:
+ return {var: str(ordinal)}
+ # Torch indexes inside the parent's mask, so the child keeps that entry (an index or a UUID).
+ entries = [entry.strip() for entry in parent.split(",") if entry.strip()]
+ return {var: entries[ordinal]} if ordinal < len(entries) else {}
+ except Exception: # noqa: BLE001 -- no mask: llama.cpp's own device choice
+ return {}
+
+
+def _accelerator_memory_mib(
+ ordinal: Optional[int] = None,
+) -> tuple[Optional[int], Optional[int], str]:
+ """(free, total, device) of the accelerator llama.cpp would offload to (CUDA card ``ordinal`` when given), through
+ the diffusion backend's helper. ``device`` is cuda / mps / cpu; free is None when unknown."""
+ try:
+ import torch
+ from types import SimpleNamespace
+
+ from core.inference.diffusion_memory import snapshot_device_memory
+
+ if torch.cuda.is_available():
+ device = "cuda"
+ elif (
+ getattr(getattr(torch, "backends", None), "mps", None) is not None
+ and torch.backends.mps.is_available()
+ ):
+ device = "mps"
+ else:
+ return None, None, "cpu"
+ from core.inference.diffusion_device import diffusion_device_scope
+
+ with diffusion_device_scope(ordinal if device == "cuda" else None):
+ snap = snapshot_device_memory(SimpleNamespace(device = device, backend = device))
+ return snap.free_mib, snap.total_mib, device
+ except Exception: # noqa: BLE001 -- unknown memory leaves the choice to the defaults
+ return None, None, "unknown"
+
+
+def plan_gpu_layers(
+ free_mib: Optional[int],
+ model_bytes: int,
+ mmproj_bytes: int = 0,
+ device: str = "cuda",
+ training: bool = False,
+) -> int:
+ """How many layers to offload: all when free memory holds weights + overhead, a share of them when it holds some,
+ else none. Unknown memory offloads all (llama.cpp's own default); a CPU-only host and training get none."""
+ if training or device == "cpu":
+ return 0
+ if device == "mps" or free_mib is None:
+ return 99
+ model_mib = model_bytes / (1024 * 1024)
+ budget = free_mib - _GPU_OVERHEAD_MIB - mmproj_bytes / (1024 * 1024)
+ if budget >= model_mib:
+ return 99
+ layers = int(_NUM_LAYERS * max(0.0, budget) / max(1.0, model_mib))
+ return layers if layers >= _MIN_GPU_LAYERS else 0
+
+
+def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:
+ if free_mib is None:
+ return False
+ floor = _TIGHT_FREE_MIB
+ if total_mib:
+ floor = max(floor, int(total_mib * _TIGHT_FREE_FRACTION))
+ return free_mib < floor
+
+
+# ── the sidecar ─────────────────────────────────────────────────────────────────────────────────────────────────────
+class PromptUpsamplerSidecar:
+ """One llama-server serving the upsampler GGUF, one request at a time."""
+
+ def __init__(self, keep_alive_seconds: Optional[float] = None) -> None:
+ if keep_alive_seconds is None:
+ from core.inference.stt_sidecar import STT_KEEP_ALIVE_SECONDS
+ keep_alive_seconds = STT_KEEP_ALIVE_SECONDS
+ self._keep_alive_seconds = float(keep_alive_seconds)
+ self._lock = threading.RLock()
+ # One request at a time (--parallel 1); also serialises load against use.
+ self._request_lock = threading.Lock()
+ self._process: Optional[subprocess.Popen] = None
+ self._port: Optional[int] = None
+ self._api_key: Optional[str] = None
+ self._key: Optional[tuple] = None
+ self._gpu_layers: Optional[int] = None
+ self._penalize_prompt_supported = False
+ self._loading = False
+ self._starting_process: Optional[subprocess.Popen] = None
+ self._load_cancel = threading.Event()
+ self._update_in_progress = False
+ self._idle_timer: Optional[threading.Timer] = None
+ self._generation = 0
+ self._active_requests = 0
+ # A blocking unload bumps the epoch and holds ``_stopping``: requests queued before it never start a server.
+ self._stop_epoch = 0
+ self._stopping = 0
+ self._gpu_ordinal: Optional[int] = None
+
+ @property
+ def loaded(self) -> bool:
+ process = self._process
+ return process is not None and process.poll() is None
+
+ @property
+ def loaded_model(self) -> Optional[str]:
+ return UPSAMPLER_ID if self.loaded else None
+
+ @property
+ def gpu_layers(self) -> Optional[int]:
+ return self._gpu_layers if self.loaded else None
+
+ def is_loading(self) -> bool:
+ return self._loading
+
+ def holds_vram(self) -> bool:
+ return self.loaded and bool(self._gpu_layers)
+
+ # keep-alive
+ def _cancel_idle_locked(self) -> None:
+ if self._idle_timer is not None:
+ self._idle_timer.cancel()
+ self._idle_timer = None
+
+ def _schedule_idle_locked(self) -> None:
+ self._cancel_idle_locked()
+ if self._keep_alive_seconds <= 0 or self._active_requests:
+ return
+ generation = self._generation
+ timer = threading.Timer(self._keep_alive_seconds, self._idle_unload, args = (generation,))
+ timer.daemon = True
+ self._idle_timer = timer
+ timer.start()
+
+ def _idle_unload(self, generation: int) -> None:
+ with self._lock:
+ if generation != self._generation or self._active_requests:
+ return
+ self._release_locked()
+
+ def _release_locked(self) -> None:
+ from core.inference.stt_mtmd_sidecar import _reap
+
+ self._cancel_idle_locked()
+ self._generation += 1
+ process = self._process
+ try:
+ _reap(process)
+ finally:
+ self._process = None
+ self._port = None
+ self._api_key = None
+ self._key = None
+ self._gpu_layers = None
+ self._penalize_prompt_supported = False
+
+ def unload(self, wait: bool = True) -> None:
+ """Stop the server. ``wait=False`` leaves a busy one alone."""
+ if not wait and (self._loading or self._active_requests):
+ return
+ if wait:
+ with self._lock:
+ self._stop_epoch += 1
+ self._stopping += 1
+ try:
+ self.cancel_pending_load()
+ if wait:
+ deadline = time.monotonic() + 30.0
+ while self._active_requests and time.monotonic() < deadline:
+ time.sleep(0.1)
+ with self._lock:
+ if not wait and self._active_requests:
+ return
+ self._release_locked()
+ finally:
+ if wait:
+ with self._lock:
+ self._stopping -= 1
+
+ def cancel_pending_load(self) -> bool:
+ if not self._loading:
+ return False
+ self._load_cancel.set()
+ process = self._starting_process
+ if process is not None and process.poll() is None:
+ with suppress(Exception):
+ process.terminate()
+ return True
+
+ @contextmanager
+ def update_maintenance(self) -> Iterator[bool]:
+ """Refuse loads and stop the server while the llama.cpp tree it runs from is replaced."""
+ self._update_in_progress = True
+ try:
+ self.cancel_pending_load()
+ with self._request_lock, self._lock:
+ was_active = self.loaded
+ self._release_locked()
+ yield was_active
+ finally:
+ self._update_in_progress = False
+
+ def _launch(self, binary: str, model: str, mmproj: Optional[str]) -> None:
+ """Start llama-server for (model, mmproj). Called under ``_request_lock``."""
+ # Published before the training probe: training that starts after it finds a load to cancel.
+ self._load_cancel.clear()
+ with self._lock:
+ self._loading = True
+ try:
+ self._spawn_server(binary, model, mmproj)
+ finally:
+ with self._lock:
+ self._loading = False
+ self._starting_process = None
+
+ def _spawn_server(self, binary: str, model: str, mmproj: Optional[str]) -> None:
+ from core.inference.stt_mtmd_sidecar import (
+ MtmdSttSidecar,
+ _llama_server_child_env,
+ _reap,
+ )
+ from core.inference.stt_sidecar import _training_active
+ from utils.process_lifetime import (
+ adopt_pid,
+ child_popen_kwargs,
+ is_process_shutting_down,
+ spawn_on_lifetime_thread,
+ )
+
+ model_bytes = os.path.getsize(model)
+ mmproj_bytes = os.path.getsize(mmproj) if mmproj else 0
+ ordinal = _resident_cuda_ordinal()
+ free_mib, _total, device = _accelerator_memory_mib(ordinal)
+ training = _training_active()
+ # Nobody holds the GPU (a CPU image load, no model): offloaded layers would sit outside every eviction.
+ cpu_only = training or _gpu_unowned()
+ layers = plan_gpu_layers(free_mib, model_bytes, mmproj_bytes, device, cpu_only)
+ sock, port = MtmdSttSidecar._reserve_free_port()
+ cmd = [
+ binary,
+ "-m",
+ model,
+ "--host",
+ "127.0.0.1",
+ "--port",
+ str(port),
+ "--parallel",
+ "1",
+ "-c",
+ str(_CONTEXT_TOKENS),
+ "-ngl",
+ str(layers),
+ "--jinja",
+ ]
+ if mmproj:
+ cmd += ["--mmproj", mmproj]
+ if layers == 0:
+ cmd.append("--no-mmproj-offload")
+ sock.close()
+ if is_process_shutting_down():
+ raise UpsamplerUnavailableError("Unsloth is shutting down.")
+ from core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _llama_server_api_key_enabled,
+ _llama_server_key_launch,
+ )
+
+ api_key = key_file = None
+ key_env: dict = {}
+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(
+ binary
+ ).get("supports_api_key_file", True):
+ api_key = secrets.token_urlsafe(32)
+ key_argv, key_env, key_file = _llama_server_key_launch(api_key)
+ cmd[1:1] = key_argv
+ logger.info(
+ "flux3.prompt_upsampler: starting llama-server (%s, -ngl %s, free %s MiB on %s%s)",
+ Path(model).name,
+ layers,
+ free_mib,
+ device,
+ ", vision" if mmproj else "",
+ )
+ if self._load_cancel.is_set():
+ raise UpsamplerCancelledError("The prompt upsampler load was cancelled.")
+ # Undrained pipes would block llama-server once its log fills them, so its output goes nowhere unless a log
+ # file is asked for.
+ log_path = (os.environ.get(ENV_SERVER_LOG) or "").strip()
+ log_file = open(log_path, "ab") if log_path else None
+ try:
+ env = {**_llama_server_child_env(binary), **key_env, **_child_device_mask(ordinal)}
+ # Forked from the process-lifetime thread: PDEATHSIG fires when the forking THREAD exits, and requests
+ # run on executor workers, so a server spawned there died with its worker and the keep-alive never held.
+ process = spawn_on_lifetime_thread(
+ lambda: subprocess.Popen(
+ cmd,
+ stdout = log_file or subprocess.DEVNULL,
+ stderr = subprocess.STDOUT if log_file else subprocess.DEVNULL,
+ stdin = subprocess.DEVNULL,
+ env = env,
+ **child_popen_kwargs(),
+ )
+ )
+ with self._lock:
+ self._starting_process = process
+ adopt_pid(process.pid)
+ if is_process_shutting_down():
+ _reap(process)
+ raise UpsamplerUnavailableError("Unsloth is shutting down.")
+ try:
+ ready = _wait_for_server(process, port, self._load_cancel)
+ finally:
+ if key_file is not None:
+ with suppress(OSError):
+ key_file.unlink()
+ if not ready:
+ _reap(process)
+ if self._load_cancel.is_set():
+ raise UpsamplerCancelledError("The prompt upsampler load was cancelled.")
+ raise RuntimeError("llama-server did not become ready for the prompt upsampler.")
+ penalize_prompt_supported = _server_has_penalize_prompt(port, api_key)
+ with self._lock:
+ self._process = process
+ self._port = port
+ self._api_key = api_key
+ self._penalize_prompt_supported = penalize_prompt_supported
+ self._key = (binary, model, mmproj)
+ self._gpu_layers = layers
+ self._gpu_ordinal = ordinal
+ self._generation += 1
+ finally:
+ if log_file is not None:
+ with suppress(OSError):
+ log_file.close()
+
+ def generate(
+ self,
+ payload: dict,
+ *,
+ binary: str,
+ model: str,
+ mmproj: Optional[str],
+ cancel_event: Optional[threading.Event] = None,
+ ) -> str:
+ """Run one chat completion and return the raw text, (re)starting the server for these files."""
+ if self._update_in_progress:
+ raise UpsamplerUnavailableError("llama.cpp is being updated.")
+ epoch = self._stop_epoch
+ with self._request_lock:
+ if cancel_event is not None and cancel_event.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+ with self._lock:
+ self._cancel_idle_locked()
+ current = self._key if self.loaded else None
+ # A server with the vision projector also serves text-only requests, so a t2i after an edit does
+ # not relaunch.
+ if current is not None and not (
+ current[:2] == (binary, model) and (mmproj is None or current[2] == mmproj)
+ ):
+ self._release_locked()
+ if self._stopping or epoch != self._stop_epoch:
+ # Queued behind a request a training start or GPU handover drained: starting now would sit beside
+ # the new owner.
+ raise UpsamplerUnavailableError(
+ "The prompt upsampler was stopped for another GPU user."
+ )
+ # Claimed under the same lock: an idle callback already waiting on it then sees the request.
+ self._active_requests += 1
+ try:
+ if not self.loaded:
+ if not _image_model_owns_gpu():
+ # Chat or video took the GPU after this request read the FLUX.3 status: starting now would
+ # sit beside the new owner for the keep-alive.
+ raise UpsamplerUnavailableError("The image model no longer holds the GPU.")
+ with _forward_cancel(cancel_event, self.cancel_pending_load):
+ self._launch(binary, model, mmproj)
+ with self._lock:
+ port, api_key = self._port, self._api_key
+ generated_only = self._penalize_prompt_supported
+ if generated_only:
+ payload = {**payload, **GENERATED_ONLY_PENALTIES}
+ return _post_chat(port, payload, api_key = api_key, cancel_event = cancel_event)
+ finally:
+ with self._lock:
+ self._active_requests -= 1
+ self._schedule_idle_locked()
+
+ def release_if_tight(self) -> bool:
+ """Unload now when the server holds VRAM and the accelerator is short, so the DiT and text encoder do not
+ denoise next to it. True when it unloaded."""
+ if not self.holds_vram():
+ return False
+ free_mib, total_mib, _device = _accelerator_memory_mib(self._gpu_ordinal)
+ if not vram_is_tight(free_mib, total_mib):
+ return False
+ logger.info(
+ "flux3.prompt_upsampler: unloading after the call (free %s of %s MiB)",
+ free_mib,
+ total_mib,
+ )
+ self.unload(wait = False)
+ return True
+
+
+def _gpu_unowned() -> bool:
+ try:
+ from core.inference import gpu_arbiter
+ return gpu_arbiter.current_owner() is None
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _image_model_owns_gpu() -> bool:
+ """The GPU arbiter's owner is the image model (or nobody yet, as on a fresh start). Fail-open."""
+ try:
+ from core.inference import gpu_arbiter
+ return gpu_arbiter.current_owner() in (None, gpu_arbiter.DIFFUSION)
+ except Exception: # noqa: BLE001
+ return True
+
+
+@contextmanager
+def _forward_cancel(cancel_event: Optional[threading.Event], cancel: Any) -> Iterator[None]:
+ """Stop pressed while llama-server starts cancels the start, not just the request after it."""
+ if cancel_event is None:
+ yield
+ return
+ done = threading.Event()
+
+ def _watch() -> None:
+ # Retried until it lands: a Stop just before the start flips `_loading` would otherwise be dropped.
+ while not done.wait(0.2):
+ if cancel_event.is_set() and cancel():
+ return
+
+ watcher = threading.Thread(target = _watch, name = "flux3-upsampler-cancel", daemon = True)
+ watcher.start()
+ try:
+ yield
+ finally:
+ done.set()
+ watcher.join(timeout = 1)
+
+
+def _wait_for_server(process, port: int, cancel_event: threading.Event) -> bool:
+ from core.inference.stt_mtmd_sidecar import MtmdSttSidecar
+ return MtmdSttSidecar._wait_for_server(process, port, cancel_event)
+
+
+def _server_has_penalize_prompt(port: int, api_key: Optional[str] = None) -> bool:
+ """Whether this llama-server takes the `penalize_prompt` request field: builds that have it echo it among the
+ default generation settings in /props. Any failure reads as an older server."""
+ connection = http.client.HTTPConnection("127.0.0.1", port, timeout = 10)
+ try:
+ connection.request(
+ "GET", "/props", headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+ )
+ response = connection.getresponse()
+ if response.status != 200:
+ return False
+ settings = (json.loads(response.read()) or {}).get("default_generation_settings") or {}
+ return "penalize_prompt" in settings or "penalize_prompt" in (settings.get("params") or {})
+ except (OSError, ValueError, AttributeError, http.client.HTTPException):
+ return False
+ finally:
+ connection.close()
+
+
+def _post_chat(
+ port: int,
+ payload: dict,
+ *,
+ api_key: Optional[str] = None,
+ cancel_event: Optional[threading.Event] = None,
+) -> str:
+ from core.inference.stt_sidecar import _close_connection_on_cancel
+
+ connection = http.client.HTTPConnection("127.0.0.1", port, timeout = _REQUEST_TIMEOUT_SECONDS)
+ done = threading.Event()
+ if cancel_event is not None:
+ threading.Thread(
+ target = _close_connection_on_cancel,
+ args = (connection, cancel_event, done),
+ daemon = True,
+ ).start()
+ try:
+ connection.request(
+ "POST",
+ "/v1/chat/completions",
+ body = json.dumps(payload).encode("utf-8"),
+ headers = {
+ "Content-Type": "application/json",
+ **({"Authorization": f"Bearer {api_key}"} if api_key else {}),
+ },
+ )
+ with connection.getresponse() as response:
+ raw = response.read()
+ if not 200 <= response.status < 300:
+ raise RuntimeError(f"The prompt upsampler returned HTTP {response.status}.")
+ body = json.loads(raw.decode("utf-8"))
+ except Exception:
+ if cancel_event is not None and cancel_event.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+ raise
+ finally:
+ done.set()
+ connection.close()
+ choices = body.get("choices") or []
+ return (choices[0].get("message", {}).get("content") or "") if choices else ""
+
+
+_sidecar: Optional[PromptUpsamplerSidecar] = None
+_sidecar_lock = threading.Lock()
+
+
+def get_sidecar() -> PromptUpsamplerSidecar:
+ global _sidecar
+ with _sidecar_lock:
+ if _sidecar is None:
+ _sidecar = PromptUpsamplerSidecar()
+ return _sidecar
+
+
+def stop_for_handover() -> None:
+ """The image model the server serves is leaving the GPU: stop every run still enhancing and the server, blocking,
+ so nothing of it outlives the DIFFUSION claim."""
+ cancel_active()
+ unload_sidecar(wait = True)
+
+
+def unload_sidecar(wait: bool = False) -> None:
+ """Stop the server if one was ever started (the FLUX.3 model it serves was unloaded). Never creates one.
+ ``wait`` also stops a busy one, for a GPU handover."""
+ sidecar = _sidecar
+ if sidecar is not None:
+ sidecar.unload(wait = wait)
+
+
+# ── upsampling with cache, progress and fallback ────────────────────────────────────────────────────────────────────
+_cache_lock = threading.Lock()
+_cache: "collections.OrderedDict[tuple, str]" = collections.OrderedDict()
+
+
+def _cache_get(key: tuple) -> Optional[str]:
+ with _cache_lock:
+ value = _cache.get(key)
+ if value is not None:
+ _cache.move_to_end(key)
+ return value
+
+
+def _cache_put(key: tuple, value: str) -> None:
+ with _cache_lock:
+ _cache[key] = value
+ _cache.move_to_end(key)
+ while len(_cache) > _CACHE_ENTRIES:
+ _cache.popitem(last = False)
+
+
+def clear_cache() -> None:
+ with _cache_lock:
+ _cache.clear()
+
+
+# Runs in flight, oldest (the one holding llama-server) first, for generate-progress and the Stop button.
+_progress_lock = threading.Lock()
+_runs: list = []
+
+
+def progress() -> Optional[dict]:
+ """``{"done", "total", "message"}`` of the run being enhanced, else None."""
+ with _progress_lock:
+ return dict(_runs[0][0]) if _runs else None
+
+
+def cancel_active() -> bool:
+ """Stop every run still enhancing (a queued one waits on the same server)."""
+ with _progress_lock:
+ events = [event for _state, event in _runs if not event.is_set()]
+ for event in events:
+ event.set()
+ return bool(events)
+
+
+@contextmanager
+def _tracking(total: int, cancel_event: threading.Event) -> Iterator[dict]:
+ run = ({"done": 0, "total": total, "message": "Enhancing prompt..."}, cancel_event)
+ with _progress_lock:
+ _runs.append(run)
+ try:
+ yield run[0]
+ finally:
+ with _progress_lock:
+ _runs.remove(run)
+
+
+def _advance(state: dict) -> None:
+ with _progress_lock:
+ state["done"] += 1
+
+
+def upsample(
+ prompt: str,
+ *,
+ variant: str = "distilled",
+ reference_images: Optional[list] = None,
+ cancel_event: Optional[threading.Event] = None,
+ sidecar: Optional[PromptUpsamplerSidecar] = None,
+) -> str:
+ """The caption for ``prompt`` (cached). Raises UpsamplerUnavailableError when nothing can run, a cancel as
+ UpsamplerCancelledError, and anything else the server does. An empty caption returns ``prompt``."""
+ binary = find_llama_server_binary()
+ if binary is None:
+ raise UpsamplerUnavailableError("llama.cpp (llama-server) is not installed.")
+ model = model_path()
+ if model is None:
+ raise UpsamplerNotDownloadedError("The FLUX.3 prompt upsampler is not downloaded.")
+ edit = bool(reference_images)
+ mmproj = mmproj_path() if edit else None
+ references: list[bytes] = []
+ if edit:
+ if mmproj is None:
+ logger.warning(
+ "flux3.prompt_upsampler: %s is not downloaded; enhancing the edit text-only",
+ UPSAMPLER_MMPROJ_FILE,
+ )
+ else:
+ references = prepare_references(reference_images)
+ payload = build_chat_payload(prompt, variant, references, edit = edit)
+ key = (prompt, variant, edit, references_hash(references), model)
+ cached = _cache_get(key)
+ if cached is not None:
+ # A Stop while the references decoded never reaches the sidecar, which is where it is normally enforced.
+ if cancel_event is not None and cancel_event.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+ return cached
+ sidecar = sidecar or get_sidecar()
+ text = clean_caption(
+ sidecar.generate(
+ payload, binary = binary, model = model, mmproj = mmproj, cancel_event = cancel_event
+ )
+ )
+ if not text:
+ logger.warning("flux3.prompt_upsampler: empty caption; using the prompt as typed")
+ return prompt
+ _cache_put(key, text)
+ return text
+
+
+@dataclass
+class EnhanceOutcome:
+ """Per-prompt captions for one generate call. ``status`` is None for families without an upsampler."""
+
+ status: Optional[str]
+ captions: dict = field(default_factory = dict)
+ statuses: dict = field(default_factory = dict)
+ # The resident model the captions were written for, set by the generate route.
+ load: Any = None
+
+ def caption_for(self, prompt: str) -> Optional[str]:
+ return self.captions.get(prompt)
+
+ def status_for(self, prompt: str) -> Optional[str]:
+ return self.statuses.get(prompt, self.status)
+
+ def encoded(self, prompt: str) -> str:
+ return self.captions.get(prompt) or prompt
+
+
+def enhance_for_generation(
+ prompts: list[str],
+ *,
+ family_name: Optional[str],
+ enhance_prompt: Optional[bool],
+ reference_images: Optional[list] = None,
+ cancel_event: Optional[threading.Event] = None,
+) -> EnhanceOutcome:
+ """Captions for each distinct prompt of a generate call. Never raises except on cancel: anything that stops the
+ upsampler leaves the prompt as typed and says why in the status."""
+ if not supports_family(family_name):
+ return EnhanceOutcome(status = None)
+ if not resolve_enhance(family_name, enhance_prompt):
+ return EnhanceOutcome(status = STATUS_OFF)
+ if find_llama_server_binary() is None or model_path() is None:
+ logger.warning(
+ "flux3.prompt_upsampler: unavailable (%s); generating with the prompt as typed",
+ "llama-server not found"
+ if find_llama_server_binary() is None
+ else "GGUF not downloaded",
+ )
+ return EnhanceOutcome(status = STATUS_UNAVAILABLE)
+ variant = variant_for_family(family_name)
+ distinct = list(dict.fromkeys(p for p in prompts if isinstance(p, str)))
+ cancel_event = cancel_event or threading.Event()
+ outcome = EnhanceOutcome(status = STATUS_APPLIED)
+ sidecar = get_sidecar()
+ # On every exit, a Stop included: a retry with enhancement off must not denoise beside a tight server.
+ try:
+ with _tracking(len(distinct), cancel_event) as tracked:
+ for index, prompt in enumerate(distinct):
+ if cancel_event.is_set():
+ raise UpsamplerCancelledError("Cancelled.")
+ systemic = None
+ try:
+ caption = upsample(
+ prompt,
+ variant = variant,
+ reference_images = reference_images,
+ cancel_event = cancel_event,
+ sidecar = sidecar,
+ )
+ except UpsamplerCancelledError:
+ raise
+ except UpsamplerUnavailableError as exc:
+ logger.warning("flux3.prompt_upsampler: unavailable (%s); prompt as typed", exc)
+ systemic = STATUS_UNAVAILABLE
+ except Exception as exc: # noqa: BLE001 -- an upsampler failure never fails the image
+ if cancel_event.is_set():
+ raise UpsamplerCancelledError("Cancelled.") from exc
+ logger.warning("flux3.prompt_upsampler: failed (%s); prompt as typed", exc)
+ systemic = STATUS_FAILED
+ else:
+ if caption == prompt:
+ outcome.statuses[prompt] = STATUS_FAILED
+ else:
+ outcome.captions[prompt] = caption
+ outcome.statuses[prompt] = STATUS_APPLIED
+ if systemic is not None:
+ # The runtime itself failed (a start that timed out, a dead server): the rest of the batch would wait
+ # out the same timeout each, so they run as typed now.
+ for rest in distinct[index:]:
+ outcome.statuses[rest] = systemic
+ break
+ _advance(tracked)
+ finally:
+ with suppress(Exception):
+ sidecar.release_if_tight()
+ if outcome.statuses and not outcome.captions:
+ # Nothing applied: the batch-level status is the first reason.
+ outcome.status = next(iter(outcome.statuses.values()))
+ return outcome
diff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.py
index 238db506e90..6fc7d184af2 100644
--- a/studio/backend/core/inference/gpu_arbiter.py
+++ b/studio/backend/core/inference/gpu_arbiter.py
@@ -11,6 +11,7 @@ under the lock, so a transfer is atomic vs other acquires.
from __future__ import annotations
+import sys
import threading
from typing import Any, Callable, Optional
@@ -79,6 +80,19 @@ def _evict_diffusion() -> None:
# Unload whichever engine the router has active (diffusers or native sd.cpp).
from core.inference.diffusion_engine_router import get_active_diffusion_engine
get_active_diffusion_engine().unload()
+ _stop_prompt_upsampler()
+
+
+def _stop_prompt_upsampler() -> None:
+ """The FLUX.3 prompt upsampler serves the image model, so it leaves with every DIFFUSION claim: an evicted, a
+ released or a replaced one. Only if Images ever started it."""
+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if module is None:
+ return
+ try:
+ module.stop_for_handover()
+ except Exception as exc: # noqa: BLE001 - never block an acquire on this
+ logger.warning("gpu_arbiter: could not stop the FLUX.3 prompt upsampler: %s", exc)
def _release_idle_video_servers(reason: str) -> None:
@@ -252,6 +266,8 @@ def release(owner: str) -> None:
global _owner, _owner_epoch, _owner_account, _prior_account
with _lock:
if _owner == owner:
+ if owner == DIFFUSION:
+ _stop_prompt_upsampler()
_owner = None
_owner_account = None
_prior_account = None
@@ -269,6 +285,8 @@ def release_if(owner: str, predicate: Callable[[], bool]) -> bool:
with _lock:
if _owner != owner or not predicate():
return False
+ if owner == DIFFUSION:
+ _stop_prompt_upsampler()
_owner = None
_owner_account = None
_prior_account = None
diff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.py
index 03f4f627496..aa01968ef6a 100644
--- a/studio/backend/core/inference/image_gallery.py
+++ b/studio/backend/core/inference/image_gallery.py
@@ -50,7 +50,9 @@ _IMG2IMG_WORKFLOWS = ("img2img", "inpaint", "outpaint", "edit", "upscale")
def _params_text(meta: dict[str, Any]) -> str:
"""Automatic1111-style ``parameters`` string for cross-tool interop."""
- lines = [str(meta.get("prompt", ""))]
+ # The caption the model encoded (FLUX.3 prompt upsampler): other tools reproduce the image from this line. The
+ # typed prompt stays in the Unsloth chunk.
+ lines = [str(meta.get("enhanced_prompt") or meta.get("prompt", ""))]
negative = meta.get("negative_prompt")
if negative:
lines.append(f"Negative prompt: {negative}")
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index 9dbe23f5ed8..ab6e113a839 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -314,9 +314,23 @@ def is_model_downloaded(model_id: str) -> bool:
class _MtmdDownloadState:
- """Tracks the two-file download of one mtmd dictation model."""
+ """Tracks the two-file download of one mtmd dictation model.
- def __init__(self) -> None:
+ ``specs`` / ``cached_paths`` / ``resolve`` let another llama.cpp sidecar (the FLUX.3 prompt upsampler) reuse the
+ same pinned two-file download; None keeps the dictation catalogue, looked up at call time.
+ """
+
+ def __init__(
+ self,
+ specs: Optional[dict] = None,
+ cached_paths = None,
+ resolve = None,
+ label: str = "mtmd STT",
+ ) -> None:
+ self._specs = specs
+ self._cached_paths_fn = cached_paths
+ self._resolve_fn = resolve
+ self._label = label
self._lock = threading.Lock()
self._thread: Optional[threading.Thread] = None
self._process: Optional[subprocess.Popen] = None
@@ -330,6 +344,15 @@ class _MtmdDownloadState:
self._hub_cache: Optional[Path] = None
self._cancelled = False
+ def _spec(self, model_id: str):
+ return (self._specs if self._specs is not None else MTMD_STT_MODELS)[model_id]
+
+ def _resolve(self, model_id: Optional[str]) -> str:
+ return (self._resolve_fn or resolve_mtmd_model_id)(model_id)
+
+ def _cached_paths(self, model_id: str, **kwargs):
+ return (self._cached_paths_fn or _cached_model_paths)(model_id, **kwargs)
+
def status(self) -> dict:
with self._lock:
downloading = self._thread is not None and self._thread.is_alive()
@@ -403,7 +426,7 @@ class _MtmdDownloadState:
total = total if total is not None else self._total_bytes
if not model_id or hub_cache is None or not selected_files:
return None
- repo = MTMD_STT_MODELS[model_id].repo
+ repo = self._spec(model_id).repo
done = sum(
_downloaded_file_bytes(
hub_cache = hub_cache,
@@ -424,7 +447,7 @@ class _MtmdDownloadState:
model_id: str,
hf_token: Optional[str] = None,
) -> str:
- model_id = resolve_mtmd_model_id(model_id)
+ model_id = self._resolve(model_id)
hub_cache = _capture_stt_hub_cache()
with self._lock:
if self._thread is not None and self._thread.is_alive():
@@ -465,7 +488,7 @@ class _MtmdDownloadState:
if hub_cache is None:
from core.inference.stt_sidecar import _active_hf_hub_cache
hub_cache = self._hub_cache or _active_hf_hub_cache()
- spec = MTMD_STT_MODELS[model_id]
+ spec = self._spec(model_id)
registry = None
owner = None
try:
@@ -527,7 +550,7 @@ class _MtmdDownloadState:
cancelled = self._cancelled
if process.returncode == 0 and not cancelled:
if (
- _cached_model_paths(
+ self._cached_paths(
model_id,
hub_cache = hub_cache,
revision = revision,
@@ -548,13 +571,13 @@ class _MtmdDownloadState:
self._cancelled = True
return
detail = stderr.decode("utf-8", "replace").strip()
- logger.warning("mtmd STT download failed for %s: %s", model_id, detail)
+ logger.warning("%s download failed for %s: %s", self._label, model_id, detail)
with self._lock:
self._error = modelscope_missing(detail) or f"Download failed for '{model_id}'."
except Exception as exc:
with self._lock:
if not self._cancelled:
- logger.warning("mtmd STT download failed for %s: %s", model_id, exc)
+ logger.warning("%s download failed for %s: %s", self._label, model_id, exc)
self._error = modelscope_missing(exc) or f"Download failed for '{model_id}'."
finally:
# Release first: it is the half that would wedge the repository.
diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.py
index 8699ae1c751..f87efa24c52 100644
--- a/studio/backend/core/inference/stt_sidecar.py
+++ b/studio/backend/core/inference/stt_sidecar.py
@@ -168,6 +168,17 @@ def _downloaded_file_bytes(
if blob_key:
blobs = repo_cache / "blobs"
candidates.extend((blobs / f"{blob_key}.incomplete", blobs / blob_key))
+ # huggingface_hub >= 1.18 writes a process-unique `<etag>.<8 hex>.incomplete` partial instead.
+ from hub.utils.hf_cache_state import incomplete_blob_hash
+
+ try:
+ candidates.extend(
+ partial
+ for partial in blobs.glob(f"{blob_key}.*.incomplete")
+ if incomplete_blob_hash(partial.name) == blob_key
+ )
+ except OSError:
+ pass
if revision:
candidates.append(repo_cache / "snapshots" / revision / filename)
sizes: list[int] = []
diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.py
index 5331f2fd3cc..5f32f469e54 100644
--- a/studio/backend/loggers/handlers.py
+++ b/studio/backend/loggers/handlers.py
@@ -120,6 +120,8 @@ _QUIET_SUCCESS_PATHS = {
"/api/inference/video/load-progress",
"/api/inference/images/generate-progress",
"/api/inference/video/generate-progress",
+ # Polled by the Images page while the FLUX.3 prompt upsampler downloads.
+ "/api/inference/images/prompt-upsampler/status",
"/api/llama/update-status",
# Engine status, re-read every 5s by the Hub while the vLLM GPU probe runs and by the open Settings engine list.
# The UI shows each answer, install jobs log their own progress, and a failed read (non-2xx) still logs.
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 0e06a1c273d..189cd85b209 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -4843,6 +4843,12 @@ class DiffusionGenerateRequest(BaseModel):
"forward pass (plain text-to-image only). Overrides `prompt` for the images; "
"`prompt` is still required as the fallback/display value.",
)
+ enhance_prompt: Optional[bool] = Field(
+ None,
+ description = "Rewrite the prompt with the FLUX.3 prompt upsampler before encoding. Null = the "
+ "family default (on for every FLUX.3 Klein family); ignored by every other family. An "
+ "unavailable or failing upsampler leaves the prompt as typed.",
+ )
seeds: Optional[list[int]] = Field(
None,
min_length = 1,
@@ -5040,6 +5046,16 @@ class GalleryImage(BaseModel):
id: str = Field(..., description = "Stable id (the on-disk filename stem)")
url: str = Field(..., description = "Relative URL to fetch the PNG bytes")
prompt: str = Field(..., description = "Prompt used")
+ enhanced_prompt: Optional[str] = Field(
+ None,
+ description = "The caption the prompt upsampler wrote and the model encoded; null when "
+ "not enhanced. Replaying it with enhance_prompt=false reproduces the image.",
+ )
+ prompt_enhancement: Optional[Literal["applied", "off", "unavailable", "failed"]] = Field(
+ None,
+ description = "Prompt upsampling outcome for families that have one: applied | off | "
+ "unavailable (not downloaded or no llama-server) | failed; null otherwise.",
+ )
negative_prompt: Optional[str] = Field(None, description = "Negative prompt, if any")
width: int = Field(..., description = "Image width")
height: int = Field(..., description = "Image height")
@@ -5204,6 +5220,44 @@ class GalleryListResponse(BaseModel):
has_more: bool = Field(False, description = "Whether older images remain past this page")
+class PromptUpsamplerDownloadRequest(BaseModel):
+ """Body of POST /images/prompt-upsampler/download (optional)."""
+
+ hf_token: Optional[str] = Field(None, description = "Hugging Face token for the download")
+
+
+class PromptUpsamplerPreviewRequest(BaseModel):
+ """Enhance one prompt with the FLUX.3 prompt upsampler without generating."""
+
+ prompt: str = Field(..., min_length = 1, description = "The prompt as typed")
+ reference_images: Optional[list[str]] = Field(
+ None,
+ max_length = 10,
+ description = "Reference images (base64/data-URL, as /images/generate) for an edit, in order",
+ )
+ variant: Optional[Literal["distilled", "base"]] = Field(
+ None,
+ description = "Sampling variant; null = the loaded FLUX.3 family's (distilled when none is loaded)",
+ )
+
+ @field_validator("reference_images")
+ @classmethod
+ def _bounded_reference_items(cls, value: Optional[list[str]]) -> Optional[list[str]]:
+ if value is not None:
+ for item in value:
+ if len(item) > 32 * 1024 * 1024:
+ raise ValueError("each reference image must be at most 32 MiB (base64)")
+ if sum(len(item) for item in value) > _MAX_CONDITION_PAYLOAD:
+ raise ValueError(
+ "the input images together must be at most 128 MiB (base64); use smaller images"
+ )
+ return value
+
+
+class PromptUpsamplerPreviewResponse(BaseModel):
+ enhanced_prompt: str = Field(..., description = "The caption the upsampler wrote")
+
+
class DiffusionGenerateProgressResponse(BaseModel):
"""Live per-step progress for an in-flight generation."""
@@ -5214,12 +5268,17 @@ class DiffusionGenerateProgressResponse(BaseModel):
eta_seconds: Optional[float] = Field(None, description = "Estimated seconds remaining")
phase: Optional[str] = Field(
None,
- description = "encode | denoise | decode; null from engines that report no phase (sd.cpp)",
+ description = "enhance (FLUX.3 prompt upsampling, before the run) | encode | denoise | decode; null "
+ "from engines that report no phase (sd.cpp)",
)
preview: Optional[str] = Field(
None, description = "Live latent preview of the image being denoised, as a JPEG data URL"
)
preview_seq: int = Field(0, description = "Moves each time a new preview is published")
+ message: Optional[str] = Field(
+ None,
+ description = "Status text for a phase with no steps (phase 'enhance': 'Enhancing prompt...')",
+ )
target: Optional[Literal["connection"]] = Field(
None,
description = "'connection' when the run is on a saved image connection, so a page that "
@@ -5352,6 +5411,13 @@ class DiffusionStatusResponse(BaseModel):
modular_families: List[str] = Field(
default_factory = list, description = "Loadable families shipped as Modular Diffusers pipelines"
)
+ supports_prompt_upsampler: bool = Field(
+ False,
+ description = "Whether the loaded family can enhance prompts with the FLUX.3 prompt upsampler",
+ )
+ prompt_upsampler_default: bool = Field(
+ False, description = "Whether enhance_prompt defaults to on for the loaded family"
+ )
base_repo: Optional[str] = Field(None, description = "Companion diffusers base repo")
device: Optional[str] = Field(None, description = "Device the pipeline is on")
dtype: Optional[str] = Field(None, description = "Compute dtype")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 659e0879fb8..1b2ee886b3b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3779,6 +3779,9 @@ from models.inference import (
DiffusionLoadRequest,
DiffusionGenerateRequest,
DiffusionGenerateResponse,
+ PromptUpsamplerDownloadRequest,
+ PromptUpsamplerPreviewRequest,
+ PromptUpsamplerPreviewResponse,
DiffusionGenerateProgressResponse,
DiffusionStatusResponse,
DiffusionDownloadPlanResponse,
@@ -50206,6 +50209,10 @@ async def load_diffusion_model_gated(
# worker is handed the credential.
for _ref in _media_repos_to_record:
_note_load_fetched_with_a_request_token(_ref, request.hf_token)
+ # Past admission: the upsampler serves the model being replaced, and a same-owner swap runs no eviction.
+ from core.inference.flux3_prompt_upsampler import stop_for_handover
+
+ stop_for_handover()
# Kicks the slow load onto a background thread and returns at once (the client polls images/load-progress).
return engine.begin_load(
request.model_path,
@@ -50552,6 +50559,56 @@ async def _external_image_generation(
_stopped_external_image_runs.discard(task)
+async def _enhance_generate_prompts(backend, request: DiffusionGenerateRequest):
+ """FLUX.3 prompt upsampling for one generate call (``EnhanceOutcome``). Every other family, and FLUX.3 with
+ ``enhance_prompt=false``, passes its prompts through untouched. Never fails the generation: only a Stop during
+ the upsampling surfaces, as the cancelled 409."""
+ from core.inference import flux3_prompt_upsampler as pup
+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG
+
+ try:
+ status = await asyncio.to_thread(backend.status)
+ family = status.get("family") if status.get("loaded") else None
+ except Exception as exc: # noqa: BLE001 -- no status, no upsampling
+ logger.debug("flux3.prompt_upsampler: status probe failed: %s", exc)
+ status, family = {}, None
+ if not pup.supports_family(family):
+ return pup.EnhanceOutcome(status = None)
+ # A workflow the family refuses fails in the run anyway: no upsampler start (or vision load) ahead of that 400.
+ workflows = status.get("workflows") or []
+ if request.workflow and workflows and request.workflow not in workflows:
+ return pup.EnhanceOutcome(status = None)
+ if not pup.resolve_enhance(family, request.enhance_prompt):
+ return pup.EnhanceOutcome(status = pup.STATUS_OFF)
+ if account_access.managed_account():
+ # The run below refuses a resident model this account may not use; refuse before the sidecar works for it.
+ await asyncio.to_thread(account_access.require_media_generation_access, status, "diffusion")
+ references = ([request.init_image] if request.init_image else []) + list(
+ request.reference_images or []
+ )
+ # The sidecar takes GPU memory of its own: never beside another account's chat, image or video run.
+ try:
+ account_access.require_idle_other_accounts()
+ except (HTTPException, account_access.GpuBusyForAnotherAccountError):
+ return pup.EnhanceOutcome(status = pup.STATUS_UNAVAILABLE)
+ try:
+ with account_access.media_generation("diffusion"):
+ outcome = await asyncio.to_thread(
+ pup.enhance_for_generation,
+ list(request.prompts or [request.prompt]),
+ family_name = family,
+ enhance_prompt = request.enhance_prompt,
+ reference_images = references or None,
+ )
+ except pup.UpsamplerCancelledError:
+ raise HTTPException(status_code = 409, detail = DIFFUSION_CANCELLED_MSG)
+ if outcome.status not in (None, pup.STATUS_OFF):
+ # Every attempted enhancement, a fallback too: a model swapped in meanwhile retries with its own default.
+ from core.inference.diffusion_families import load_identity
+ outcome.load = load_identity(status.get("repo_id"), status.get("base_repo"), family)
+ return outcome
+
+
@studio_router.post("/images/generate", response_model = DiffusionGenerateResponse)
@account_access.gpu_busy_route
async def generate_diffusion_image(
@@ -50590,6 +50647,8 @@ async def generate_diffusion_image(
if request.localized_edit is not None
else None
)
+ # FLUX.3 prompt upsampling runs ahead of the run, so the sidecar can leave the VRAM before the DiT denoises.
+ enhancement = await _enhance_generate_prompts(backend, request)
result = None
for attempt in range(2):
expected_load = None
@@ -50602,6 +50661,9 @@ async def generate_diffusion_image(
expected_load = load_identity(
status.get("repo_id"), status.get("base_repo"), status.get("family")
)
+ if enhancement.load is not None:
+ # Captions are written for one family: a model swapped in since then retries with fresh ones.
+ expected_load = enhancement.load
# Ahead of the run: milestones key off the previous poll, so a resumed range logs nothing.
reset_media_generation_progress("image")
try:
@@ -50609,7 +50671,7 @@ async def generate_diffusion_image(
result = await asyncio.to_thread(
backend.generate,
expected_load = expected_load,
- prompt = request.prompt,
+ prompt = enhancement.encoded(request.prompt),
negative_prompt = request.negative_prompt,
width = None if size_omitted else request.width,
height = None if size_omitted else request.height,
@@ -50617,7 +50679,11 @@ async def generate_diffusion_image(
guidance = request.guidance,
seed = request.seed,
batch_size = request.batch_size,
- prompts = request.prompts,
+ prompts = (
+ [enhancement.encoded(p) for p in request.prompts]
+ if request.prompts
+ else request.prompts
+ ),
seeds = request.seeds,
init_image = request.init_image,
mask_image = request.mask_image,
@@ -50660,6 +50726,8 @@ async def generate_diffusion_image(
except DiffusionModelReplacedError as exc:
if attempt > 0:
raise HTTPException(status_code = 409, detail = str(exc))
+ # The caption was written for the replaced model's family.
+ enhancement = await _enhance_generate_prompts(backend, request)
continue
except RuntimeError as exc:
# Match these two EXACT client-state messages (409); other RuntimeErrors are failures.
@@ -50686,16 +50754,21 @@ async def generate_diffusion_image(
if per_image_seeds and index < len(per_image_seeds)
else result["seed"]
)
+ # A prompts-list batch records each image's OWN prompt so its recipe replays exactly.
+ typed_prompt = (
+ request.prompts[index]
+ if request.prompts and index < len(request.prompts)
+ else request.prompt
+ )
records.append(
image_gallery.save(
image,
{
- # A prompts-list batch records each image's OWN prompt so its recipe replays exactly.
- "prompt": (
- request.prompts[index]
- if request.prompts and index < len(request.prompts)
- else request.prompt
- ),
+ "prompt": typed_prompt,
+ # The caption actually encoded (FLUX.3 prompt upsampling); replaying it with
+ # enhance_prompt=false reproduces the image.
+ "enhanced_prompt": enhancement.caption_for(typed_prompt),
+ "prompt_enhancement": enhancement.status_for(typed_prompt),
"negative_prompt": result.get("negative_prompt"),
# Persist the ACTUAL output size, not the request sliders: the conditioned workflows derive it from the upload.
"width": getattr(image, "width", None) or request.width,
@@ -51451,6 +51524,12 @@ async def unload_diffusion_model(
if account_access.account_scope() is not None:
unload = functools.partial(unload, expected_account = current_account_id())
status_dict = await asyncio.to_thread(unload)
+ # The FLUX.3 prompt upsampler serves the image model: it leaves with it instead of holding VRAM to its keep-alive.
+ from core.inference import flux3_prompt_upsampler as pup
+
+ # Blocking: DIFFUSION ownership is dropped below, so a sidecar still starting or answering would sit beside the
+ # next owner without an eviction to stop it.
+ await asyncio.to_thread(pup.stop_for_handover)
# Drop DIFFUSION ownership only if nothing is resident AND no load is in flight, or a later chat load skips eviction and
# OOMs the new pipeline. An in-flight load reads is_loaded False, so gate on loading_repo_ids() and use release_if.
engine = get_active_diffusion_engine()
@@ -51556,6 +51635,16 @@ async def diffusion_generate_progress(current_subject: str = Depends(get_current
progress = get_active_diffusion_engine().generate_progress()
log_media_generation_progress("image", progress)
+ if not progress["active"]:
+ from core.inference.flux3_prompt_upsampler import progress as upsampler_progress
+ enhancing = upsampler_progress()
+ if enhancing is not None:
+ progress = {
+ **progress,
+ "active": True,
+ "phase": "enhance",
+ "message": enhancing.get("message"),
+ }
if external and not progress["active"]:
progress = {**progress, "active": True, "target": "connection"}
# A finished generation still persisting its gallery record counts as active, so a reload probe keeps polling.
@@ -51597,13 +51686,116 @@ async def cancel_diffusion_generation(
):
return {"cancelled": False}
+ # A run still upsampling its prompt has not reached the engine yet.
+ from core.inference.flux3_prompt_upsampler import cancel_active as cancel_upsampling
+
+ # Both: another tab's request may be enhancing while this one denoises.
+ upsampling = cancel_upsampling()
# The slot can change hands before the executor callback, so the engine rechecks under the lock.
expected = account_access.tracked_generation_account()
cancel = get_active_diffusion_engine().cancel_generate
if expected is not None:
cancel = functools.partial(cancel, expected_account = expected)
cancelled = await asyncio.get_running_loop().run_in_executor(_CANCEL_EXECUTOR, cancel)
- return {"cancelled": cancelled}
+ return {"cancelled": bool(cancelled or upsampling)}
+
+
+# ── FLUX.3 prompt upsampler (llama.cpp sidecar) ──────────────────────────────────────────────────────────
+
+
+@studio_router.get("/images/prompt-upsampler/status")
+async def prompt_upsampler_status(current_subject: str = Depends(get_current_subject)):
+ """Whether the FLUX.3 prompt upsampler can run (llama-server present), is downloaded, is loaded, and the
+ download progress (the dictation download's shape)."""
+ from core.inference import flux3_prompt_upsampler
+ return JSONResponse(content = await asyncio.to_thread(flux3_prompt_upsampler.status))
+
+
+@studio_router.post("/images/prompt-upsampler/download")
+async def prompt_upsampler_download(
+ payload: Optional[PromptUpsamplerDownloadRequest] = None,
+ current_subject: str = Depends(get_current_subject),
+ hf_token: HfTokenArg = Depends(get_request_hf_token),
+):
+ """Start the background download of the upsampler GGUF and its vision projector; progress on /status."""
+ from core.inference import flux3_prompt_upsampler
+ from core.inference.stt_sidecar import SttDownloadCacheError, SttModelIdError
+
+ account_access.require_installation_owner()
+ if flux3_prompt_upsampler.status().get("override"):
+ raise HTTPException(
+ status_code = 409,
+ detail = "The prompt upsampler files come from UNSLOTH_FLUX3_UPSAMPLER_GGUF / _MMPROJ; set both instead.",
+ )
+ token = account_access.account_hf_token(
+ (payload.hf_token if payload is not None and payload.hf_token else None) or hf_token
+ )
+ try:
+ download_id = await asyncio.to_thread(flux3_prompt_upsampler.start_download, token)
+ except (SttModelIdError, SttDownloadCacheError) as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ return JSONResponse(
+ content = {**flux3_prompt_upsampler.download_status(), "download_id": download_id}
+ )
+
+
+@studio_router.post("/images/prompt-upsampler/cancel")
+async def prompt_upsampler_cancel(current_subject: str = Depends(get_current_subject)):
+ """Stop the upsampler download; partial files stay cached, so the next download resumes."""
+ from core.inference import flux3_prompt_upsampler
+
+ account_access.require_installation_owner()
+ cancelled = await asyncio.to_thread(flux3_prompt_upsampler.cancel_download)
+ return JSONResponse(
+ content = {**flux3_prompt_upsampler.download_status(), "cancelled": cancelled}
+ )
+
+
+@studio_router.post(
+ "/images/prompt-upsampler/preview", response_model = PromptUpsamplerPreviewResponse
+)
+@account_access.gpu_busy_route
+async def prompt_upsampler_preview(
+ request: PromptUpsamplerPreviewRequest, current_subject: str = Depends(get_current_subject)
+):
+ """The caption the upsampler writes for ``prompt`` (and references), so the page can show or edit it before
+ generating. 409 when not downloaded, 501 without llama-server."""
+ from core.inference import flux3_prompt_upsampler as pup
+
+ variant = request.variant
+ if variant is None:
+ try:
+ from core.inference.diffusion_engine_router import get_active_diffusion_engine
+ status = await asyncio.to_thread(get_active_diffusion_engine().status)
+ variant = pup.variant_for_family(status.get("family") if status.get("loaded") else None)
+ except Exception: # noqa: BLE001 -- nothing loaded reads as distilled
+ variant = "distilled"
+ account_access.require_idle_other_accounts()
+ try:
+ with account_access.media_generation("diffusion"):
+ caption = await asyncio.to_thread(
+ pup.upsample,
+ request.prompt,
+ variant = variant,
+ reference_images = request.reference_images or None,
+ )
+ except pup.UpsamplerNotDownloadedError as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ except pup.UpsamplerUnavailableError as exc:
+ raise HTTPException(status_code = 501, detail = str(exc))
+ except pup.UpsamplerCancelledError as exc:
+ raise HTTPException(status_code = 409, detail = str(exc))
+ except ValueError as exc:
+ raise HTTPException(status_code = 400, detail = str(exc))
+ except Exception as exc: # noqa: BLE001
+ logger.error("flux3.prompt_upsampler.preview_failed: %s", exc, exc_info = True)
+ raise HTTPException(status_code = 500, detail = "The prompt upsampler failed.")
+ finally:
+ try:
+ await asyncio.to_thread(pup.get_sidecar().release_if_tight)
+ except Exception: # noqa: BLE001 -- the keep-alive reclaims it anyway
+ pass
+ return PromptUpsamplerPreviewResponse(enhanced_prompt = caption)
# ──────────────────────────────────────────────────────────────────────────
diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.py
index 9b0d532cd41..90f08dbdd2b 100644
--- a/studio/backend/routes/training_vram.py
+++ b/studio/backend/routes/training_vram.py
@@ -5,6 +5,7 @@
models when they fit. STT is evicted before chat when training needs memory.
"""
+import sys
from typing import Any, Callable, Dict, List, Optional, Tuple
from hub.utils.hf_tokens import HfTokenArg, normalize_token
@@ -668,10 +669,35 @@ def free_stt_model_for_training(reason: str) -> List[str]:
return freed
+def free_prompt_upsampler_for_training(reason: str) -> List[str]:
+ """Stop the FLUX.3 prompt upsampler's llama-server, only if Images ever started it. Never raises."""
+ upsampler_module = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if upsampler_module is None:
+ return []
+ try:
+ # A run still decoding its references has not reached the sidecar: stopped here, it never starts one.
+ upsampler_module.cancel_active()
+ upsampler = upsampler_module.get_sidecar()
+ # Loaded, not only on the GPU: a -ngl 0 server still holds ~6 GB of host RAM and the CPU through training.
+ if upsampler.loaded or upsampler.is_loading():
+ logger.info("Unloading the FLUX.3 prompt upsampler for training (%s)", reason)
+ upsampler.unload()
+ return ["image:prompt-upsampler"]
+ except Exception as e:
+ logger.warning("Could not unload the FLUX.3 prompt upsampler: %s", e)
+ return []
+
+
def coordinate_models_for_training(
can_keep: Callable[[], Tuple[bool, Dict[str, Any]]],
) -> List[str]:
"""Keep resident models when they fit, evicting STT before chat."""
+ # The upsampler restarts on demand, so it never holds memory through training.
+ freed = free_prompt_upsampler_for_training(reason = "training start")
+ return freed + _coordinate_resident_models(can_keep)
+
+
+def _coordinate_resident_models(can_keep: Callable[[], Tuple[bool, Dict[str, Any]]]) -> List[str]:
resident_chat = summarize_resident_chat()
resident_stt = summarize_resident_stt()
if not resident_chat["any"] and not resident_stt["any"]:
diff --git a/studio/backend/tests/test_flux3_prompt_upsampler.py b/studio/backend/tests/test_flux3_prompt_upsampler.py
new file mode 100644
index 00000000000..c139c5be465
--- /dev/null
+++ b/studio/backend/tests/test_flux3_prompt_upsampler.py
@@ -0,0 +1,1202 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 prompt upsampling: request building, reference preparation, family defaults, fallbacks, the gallery
+record and the routes. The llama-server is mocked; no GPU and no network."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import threading
+import time
+
+import pytest
+
+from core.inference import flux3_prompt_upsampler as pup
+
+
[email protected](autouse = True)
+def _clean(monkeypatch):
+ pup.clear_cache()
+ monkeypatch.delenv(pup.ENV_MODEL_OVERRIDE, raising = False)
+ monkeypatch.delenv(pup.ENV_MMPROJ_OVERRIDE, raising = False)
+ yield
+ pup.clear_cache()
+
+
+# ── prompts and sampling ────────────────────────────────────────────────────────────────────────────────────────────
+# sha256 of BFL's system_prompts.json entries (black-forest-labs/flux-3-image-klein-pup @ 8b069751), equal to
+# diffusers SYSTEM_PROMPTS at d42ebf9c6e.
+_T2I_SHA = "8b235a421bce2213d11ada01f5a1244443dd72303ffbb4cc9e5a0d8b0de5ab6f"
+_I2I_SHA = "95cec384d5b6d341caa9fb3f2c7ca45ee80c2364b3a95d4044aebf976f6dd3bf"
+
+
+def test_system_prompts_are_bfls_verbatim():
+ assert hashlib.sha256(pup.SYSTEM_PROMPTS["t2i"].encode()).hexdigest() == _T2I_SHA
+ assert hashlib.sha256(pup.SYSTEM_PROMPTS["i2i"].encode()).hexdigest() == _I2I_SHA
+
+
+ "variant, repeat, presence",
+ [("distilled", 1.05, 1.5), ("base", 1.0, 1.5)],
+)
+def test_t2i_payload(variant, repeat, presence):
+ body = pup.build_chat_payload("a gymnast doing a backflip", variant)
+ assert body["messages"] == [
+ {"role": "system", "content": pup.SYSTEM_PROMPTS["t2i"]},
+ {"role": "user", "content": [{"type": "text", "text": "a gymnast doing a backflip"}]},
+ ]
+ assert body["top_k"] == 1 and body["temperature"] == 1.0 and body["top_p"] == 0.95
+ assert body["min_p"] == 0.0 and body["seed"] == 0
+ assert body["repeat_penalty"] == repeat and body["presence_penalty"] == presence
+ assert body["max_tokens"] == 3584
+ assert body["chat_template_kwargs"] == {"enable_thinking": False}
+
+
[email protected]("variant, repeat", [("distilled", 1.05), ("base", 1.0)])
+def test_edit_payload_images_then_scaffold(variant, repeat):
+ refs = [b"jpeg-one", b"jpeg-two"]
+ body = pup.build_chat_payload("make it red", variant, refs)
+ system, user = body["messages"]
+ assert system == {"role": "system", "content": pup.SYSTEM_PROMPTS["i2i"]}
+ parts = user["content"]
+ assert [p["type"] for p in parts] == ["image_url", "image_url", "text"]
+ assert (
+ parts[0]["image_url"]["url"]
+ == "data:image/jpeg;base64," + base64.b64encode(b"jpeg-one").decode()
+ )
+ assert parts[2]["text"] == (
+ "Reference images follow in order (the first image, the second image, ...).\n"
+ "User request: make it red\n"
+ "Output the caption."
+ )
+ assert body["repeat_penalty"] == repeat and body["presence_penalty"] == 0.0
+
+
+def test_edit_without_images_uses_the_edit_prompt_text_only():
+ body = pup.build_chat_payload("make it red", "distilled", [], edit = True)
+ system, user = body["messages"]
+ assert system["content"] == pup.SYSTEM_PROMPTS["i2i"]
+ assert [p["type"] for p in user["content"]] == ["text"]
+ assert "User request: make it red" in user["content"][0]["text"]
+
+
+def test_unknown_variant_is_refused():
+ with pytest.raises(ValueError):
+ pup.build_chat_payload("x", "turbo")
+
+
+def test_clean_caption():
+ assert pup.clean_caption("<think>\n\n</think>\n\n A caption. ") == "A caption."
+ assert pup.clean_caption('"quoted"') == "quoted"
+ assert pup.clean_caption(None) == ""
+
+
+# ── reference preparation ───────────────────────────────────────────────────────────────────────────────────────────
+ "size, count, expected",
+ [
+ ((1024, 1024), 1, (1024, 1024)),
+ ((4000, 3000), 1, (1152, 864)),
+ ((4000, 3000), 2, (832, 608)),
+ ((4000, 3000), 3, (576, 416)),
+ ((4000, 3000), 10, (576, 416)),
+ ((100, 50), 1, (96, 64)),
+ ((1000, 31), 1, (992, 32)),
+ ],
+)
+def test_reference_size(size, count, expected):
+ w, h = pup.get_reference_size(*size, count)
+ assert (w, h) == expected
+ assert w % 32 == 0 and h % 32 == 0
+ assert w * h <= pup.MAX_PIXELS_BY_REFS[count]
+
+
+def test_reference_size_refuses_extreme_aspect():
+ with pytest.raises(ValueError):
+ pup.get_reference_size(10000, 10, 1)
+
+
+def test_prepare_reference_rgba_over_white_and_jpeg():
+ from PIL import Image
+
+ image = Image.new("RGBA", (300, 200), (0, 0, 0, 0))
+ out = Image.open(io.BytesIO(pup.prepare_reference(image, 1)))
+ assert out.format == "JPEG" and out.mode == "RGB"
+ assert out.size == (288, 192)
+ r, g, b = out.getpixel((10, 10))
+ assert min(r, g, b) > 245 # transparent became white, not black
+
+
+def test_prepare_references_caps_at_ten_and_decodes_data_urls():
+ from PIL import Image
+
+ buf = io.BytesIO()
+ Image.new("RGB", (800, 600), (10, 20, 30)).save(buf, format = "PNG")
+ url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+ refs = pup.prepare_references([url] * 12)
+ assert len(refs) == 10
+ assert Image.open(io.BytesIO(refs[0])).size == pup.get_reference_size(800, 600, 10)
+
+
+# ── family defaults ─────────────────────────────────────────────────────────────────────────────────────────────────
+ "family, flag, expected",
+ [
+ ("flux.3-klein", None, True),
+ ("flux.3-klein-base", None, True),
+ ("flux.3-klein", False, False),
+ ("flux.3-klein-base", True, True),
+ ("flux.2-klein", None, False),
+ ("flux.2-klein", True, False),
+ ("z-image", True, False),
+ (None, True, False),
+ ],
+)
+def test_resolve_enhance(family, flag, expected):
+ assert pup.resolve_enhance(family, flag) is expected
+
+
+def test_variant_and_capabilities():
+ assert pup.variant_for_family("flux.3-klein") == "distilled"
+ assert pup.variant_for_family("flux.3-klein-base") == "base"
+ assert pup.family_capabilities("flux.3-klein-base") == {
+ "supports_prompt_upsampler": True,
+ "prompt_upsampler_default": True,
+ }
+ assert pup.family_capabilities("qwen-image") == {
+ "supports_prompt_upsampler": False,
+ "prompt_upsampler_default": False,
+ }
+
+
+def test_status_annotation_carries_the_capabilities(monkeypatch):
+ from core.inference import diffusion_engine_router as router
+ from models.inference import DiffusionStatusResponse
+
+ monkeypatch.setattr(router, "_supported_family_capabilities", lambda: ("flux.3-klein",))
+ out = router.annotate_status({"loaded": True, "repo_id": "x", "family": "flux.3-klein"})
+ assert out["supports_prompt_upsampler"] is True and out["prompt_upsampler_default"] is True
+ assert DiffusionStatusResponse(**out).supports_prompt_upsampler is True
+ out = router.annotate_status({"loaded": False, "repo_id": None, "family": None})
+ assert out["supports_prompt_upsampler"] is False
+
+
+# ── memory planning ─────────────────────────────────────────────────────────────────────────────────────────────────
+GIB = 1024**3
+
+
+def test_plan_gpu_layers():
+ model = int(5.24 * GIB)
+ assert pup.plan_gpu_layers(40_000, model, 900 * 1024 * 1024) == 99
+ assert 4 <= pup.plan_gpu_layers(6_000, model) < 32
+ assert pup.plan_gpu_layers(2_000, model) == 0
+ assert pup.plan_gpu_layers(40_000, model, training = True) == 0
+ assert pup.plan_gpu_layers(None, model, device = "cpu") == 0
+ assert pup.plan_gpu_layers(None, model) == 99
+ assert pup.plan_gpu_layers(1_000, model, device = "mps") == 99
+
+
+def test_vram_is_tight():
+ assert pup.vram_is_tight(4_000, 24_000)
+ assert not pup.vram_is_tight(20_000, 24_000)
+ assert pup.vram_is_tight(15_000, 80_000) # under 20% of the card
+ assert not pup.vram_is_tight(None, None)
+
+
+# ── paths and fallbacks ─────────────────────────────────────────────────────────────────────────────────────────────
+def test_env_override_paths(monkeypatch, tmp_path):
+ model = tmp_path / "pup.gguf"
+ model.write_bytes(b"x" * 10)
+ monkeypatch.setenv(pup.ENV_MODEL_OVERRIDE, str(model))
+ monkeypatch.setenv(pup.ENV_MMPROJ_OVERRIDE, str(tmp_path / "missing.gguf"))
+ monkeypatch.setattr(pup, "_cached_at_any_revision", lambda *a, **k: "/cache/should-not-be-used")
+ assert pup.model_path() == str(model)
+ # A set but missing override is a typo, not a reason to read the cache.
+ assert pup.mmproj_path() is None
+
+
+class _FakeSidecar:
+ def __init__(
+ self,
+ reply = "A long caption.",
+ error = None,
+ ):
+ self.reply = reply
+ self.error = error
+ self.calls: list[dict] = []
+
+ def generate(
+ self,
+ payload,
+ *,
+ binary,
+ model,
+ mmproj,
+ cancel_event = None,
+ ):
+ self.calls.append({"payload": payload, "model": model, "mmproj": mmproj})
+ if self.error is not None:
+ raise self.error
+ return self.reply(payload) if callable(self.reply) else self.reply
+
+ def release_if_tight(self):
+ return False
+
+
+def runnable(monkeypatch):
+ sidecar = _FakeSidecar(
+ reply = lambda payload: "Caption for " + payload["messages"][1]["content"][-1]["text"]
+ )
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")
+ monkeypatch.setattr(pup, "mmproj_path", lambda: "/m/mmproj.gguf")
+ monkeypatch.setattr(pup, "get_sidecar", lambda: sidecar)
+ return sidecar
+
+
+def test_one_upsample_per_distinct_prompt_and_cache(runnable):
+ out = pup.enhance_for_generation(
+ ["a cat", "a dog", "a cat"], family_name = "flux.3-klein", enhance_prompt = None
+ )
+ assert out.status == "applied"
+ assert len(runnable.calls) == 2
+ assert out.encoded("a cat") == "Caption for a cat"
+ assert out.status_for("a dog") == "applied"
+ # Same prompt + variant + refs: the cache answers, so a re-run encodes the same caption.
+ again = pup.enhance_for_generation(["a cat"], family_name = "flux.3-klein", enhance_prompt = True)
+ assert len(runnable.calls) == 2 and again.encoded("a cat") == "Caption for a cat"
+ # A different variant is a different caption request.
+ pup.enhance_for_generation(["a cat"], family_name = "flux.3-klein-base", enhance_prompt = True)
+ assert len(runnable.calls) == 3
+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.0
+
+
+def test_off_and_other_families_never_call(runnable):
+ assert (
+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = False).status
+ == "off"
+ )
+ other = pup.enhance_for_generation(["a"], family_name = "flux.2-klein", enhance_prompt = True)
+ assert other.status is None and other.encoded("a") == "a"
+ assert runnable.calls == []
+
+
+def test_unavailable_without_weights_or_binary(monkeypatch):
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: None)
+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)
+ assert out.status == "unavailable" and out.encoded("a") == "a" and out.caption_for("a") is None
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)
+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")
+ assert (
+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None).status
+ == "unavailable"
+ )
+
+
+def test_server_failure_falls_back_to_the_typed_prompt(runnable):
+ runnable.error = RuntimeError("boom")
+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)
+ assert out.status == "failed" and out.encoded("a") == "a" and out.caption_for("a") is None
+
+
+def test_empty_caption_falls_back(runnable):
+ runnable.reply = " "
+ assert pup.upsample("a", variant = "distilled") == "a"
+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)
+ assert out.status == "failed" and out.encoded("a") == "a"
+
+
+def test_a_stopped_enhancement_still_releases_a_tight_server(runnable, monkeypatch):
+ released = []
+ monkeypatch.setattr(runnable, "release_if_tight", lambda: released.append(True) or True)
+ runnable.error = pup.UpsamplerCancelledError("Cancelled.")
+ with pytest.raises(pup.UpsamplerCancelledError):
+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)
+ assert released == [True]
+
+
+def test_cancel_propagates(runnable):
+ event = threading.Event()
+ event.set()
+ with pytest.raises(pup.UpsamplerCancelledError):
+ pup.enhance_for_generation(
+ ["a"], family_name = "flux.3-klein", enhance_prompt = None, cancel_event = event
+ )
+ assert pup.progress() is None
+
+
+def _png_url(size = (640, 480)):
+ from PIL import Image
+
+ buf = io.BytesIO()
+ Image.new("RGB", size, (200, 10, 10)).save(buf, format = "PNG")
+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+
+
+def test_edit_sends_references_with_mmproj(runnable):
+ caption = pup.upsample("make it blue", variant = "distilled", reference_images = [_png_url()])
+ call = runnable.calls[-1]
+ assert call["mmproj"] == "/m/mmproj.gguf"
+ parts = call["payload"]["messages"][1]["content"]
+ assert [p["type"] for p in parts] == ["image_url", "text"]
+ assert caption.startswith("Caption for Reference images follow in order")
+
+
+def test_edit_without_mmproj_runs_text_only(runnable, monkeypatch):
+ monkeypatch.setattr(pup, "mmproj_path", lambda: None)
+ pup.upsample("make it blue", variant = "distilled", reference_images = [_png_url()])
+ call = runnable.calls[-1]
+ assert call["mmproj"] is None
+ system, user = call["payload"]["messages"]
+ assert system["content"] == pup.SYSTEM_PROMPTS["i2i"]
+ assert [p["type"] for p in user["content"]] == ["text"]
+
+
+def test_references_change_the_cache_key(runnable):
+ pup.upsample("edit", reference_images = [_png_url((640, 480))])
+ pup.upsample("edit", reference_images = [_png_url((640, 480))])
+ pup.upsample("edit", reference_images = [_png_url((480, 640))])
+ assert len(runnable.calls) == 2
+
+
+def test_upsample_raises_when_not_downloaded(monkeypatch):
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: None)
+ with pytest.raises(pup.UpsamplerNotDownloadedError):
+ pup.upsample("a")
+
+
+# ── the server request (mocked HTTP) ────────────────────────────────────────────────────────────────────────────────
+def test_post_chat_parses_the_reply(monkeypatch):
+ import json
+
+ sent = {}
+
+ class _Response:
+ status = 200
+
+ def read(self):
+ return json.dumps({"choices": [{"message": {"content": "The caption."}}]}).encode()
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ class _Connection:
+ def __init__(
+ self,
+ host,
+ port,
+ timeout = None,
+ ):
+ sent["target"] = (host, port)
+ self.sock = None
+
+ def request(
+ self,
+ method,
+ path,
+ body = None,
+ headers = None,
+ ):
+ sent.update(method = method, path = path, body = json.loads(body), headers = headers)
+
+ def getresponse(self):
+ return _Response()
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(pup.http.client, "HTTPConnection", _Connection)
+ payload = pup.build_chat_payload("a", "distilled")
+ assert pup._post_chat(4321, payload, api_key = "k") == "The caption."
+ assert sent["target"] == ("127.0.0.1", 4321)
+ assert sent["path"] == "/v1/chat/completions" and sent["body"] == payload
+ assert sent["headers"]["Authorization"] == "Bearer k"
+
+
+ "status, props, expected",
+ [
+ (200, {"default_generation_settings": {"params": {"penalize_prompt": True}}}, True),
+ (200, {"default_generation_settings": {"penalize_prompt": True}}, True),
+ (200, {"default_generation_settings": {"params": {"presence_penalty": 0.0}}}, False),
+ (200, {}, False),
+ (404, {}, False),
+ (200, None, False),
+ ],
+)
+def test_penalize_prompt_probe(monkeypatch, status, props, expected):
+ import json
+
+ class _Response:
+ def __init__(self):
+ self.status = status
+
+ def read(self):
+ return b"not json" if props is None else json.dumps(props).encode()
+
+ class _Connection:
+ def __init__(
+ self,
+ host,
+ port,
+ timeout = None,
+ ):
+ pass
+
+ def request(
+ self,
+ method,
+ path,
+ headers = None,
+ ):
+ assert (method, path) == ("GET", "/props")
+
+ def getresponse(self):
+ return _Response()
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(pup.http.client, "HTTPConnection", _Connection)
+ assert pup._server_has_penalize_prompt(4321) is expected
+
+
[email protected]("supported", [True, False])
+def test_generated_only_penalties_follow_the_server(monkeypatch, supported):
+ sent = {}
+ sidecar = pup.PromptUpsamplerSidecar()
+
+ def _fake_launch(binary, model, mmproj):
+ sidecar._process = type("P", (), {"poll": lambda self: None})()
+ sidecar._port, sidecar._key = 4321, (binary, model, mmproj)
+ sidecar._penalize_prompt_supported = supported
+
+ monkeypatch.setattr(sidecar, "_launch", _fake_launch)
+ monkeypatch.setattr(sidecar, "_schedule_idle_locked", lambda: None)
+ monkeypatch.setattr(
+ pup, "_post_chat", lambda port, payload, **kw: sent.update(payload) or "caption"
+ )
+ payload = pup.build_chat_payload("a", "base")
+ assert sidecar.generate(payload, binary = "b", model = "m", mmproj = None) == "caption"
+ if supported:
+ assert sent["penalize_prompt"] is False and sent["repeat_last_n"] == 4096
+ else:
+ assert "penalize_prompt" not in sent and "repeat_last_n" not in sent
+ assert sent["presence_penalty"] == 1.5 and "penalize_prompt" not in payload
+
+
+# ── the download state reuses the dictation worker ──────────────────────────────────────────────────────────────────
+def test_download_state_targets_the_unsloth_repo(monkeypatch):
+ state = pup._get_download_state()
+ assert state._spec(pup.UPSAMPLER_ID).repo == "unsloth/flux-3-image-klein-pup-GGUF"
+ assert state._spec(pup.UPSAMPLER_ID).model_file == "flux-3-image-klein-pup-Q4_K_M.gguf"
+ assert state._spec(pup.UPSAMPLER_ID).mmproj_file == "mmproj-F16.gguf"
+ with pytest.raises(ValueError):
+ state._resolve("black-forest-labs/flux-3-image-klein-pup")
+ snapshot = state.status()
+ assert {"downloading", "download_id", "error", "cancelled", "bytes_total", "bytes_done"} <= set(
+ snapshot
+ )
+
+
+def test_status_shape(monkeypatch):
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)
+ monkeypatch.setattr(pup, "model_path", lambda: None)
+ monkeypatch.setattr(pup, "mmproj_path", lambda: None)
+ out = pup.status()
+ assert out["supported"] is False and out["downloaded"] is False and out["loaded"] is False
+ assert out["mmproj_downloaded"] is False
+ assert out["repo"] == pup.UPSAMPLER_REPO and out["filename"] == pup.UPSAMPLER_MODEL_FILE
+ assert out["size_bytes"] == pup.UPSAMPLER_FILE_SIZES[pup.UPSAMPLER_MODEL_FILE]
+ assert isinstance(out["download"], dict)
+
+
+# ── gallery record ──────────────────────────────────────────────────────────────────────────────────────────────────
+def test_gallery_record_round_trip(monkeypatch, tmp_path):
+ from PIL import Image
+
+ from core.inference import image_gallery
+ from models.inference import GalleryImage
+
+ monkeypatch.setattr(image_gallery, "gallery_dir", lambda: tmp_path)
+ meta = {
+ "prompt": "a gymnast doing a backflip",
+ "enhanced_prompt": "Full-color indoor photograph of a gymnast ...",
+ "prompt_enhancement": "applied",
+ "width": 64,
+ "height": 64,
+ "steps": 4,
+ "guidance": 1.0,
+ "seed": 7,
+ "created_at": 1.0,
+ }
+ record = image_gallery.save(Image.new("RGB", (64, 64)), meta)
+ listed = [r for r in image_gallery.list_images(10, 0) if r["id"] == record["id"]][0]
+ image = GalleryImage(**listed)
+ assert image.prompt == "a gymnast doing a backflip"
+ assert image.enhanced_prompt == meta["enhanced_prompt"]
+ assert image.prompt_enhancement == "applied"
+ # Records written before this read as unset.
+ old = {k: v for k, v in listed.items() if k not in ("enhanced_prompt", "prompt_enhancement")}
+ image = GalleryImage(**old)
+ assert image.enhanced_prompt is None and image.prompt_enhancement is None
+
+
+# ── routes ──────────────────────────────────────────────────────────────────────────────────────────────────────────
+class _FluxBackend:
+ def __init__(self, family = "flux.3-klein"):
+ self.family = family
+ self.calls: list[dict] = []
+
+ def status(self):
+ return {"loaded": True, "repo_id": "unsloth/flux", "family": self.family}
+
+ def generate(
+ self,
+ *,
+ seed = None,
+ prompts = None,
+ batch_size = 1,
+ **kwargs,
+ ):
+ self.calls.append({"prompts": prompts, "seed": seed, **kwargs})
+ count = len(prompts) if prompts else batch_size
+ return {"images": [object()] * count, "seed": 5, "seeds": [5 + i for i in range(count)]}
+
+ def generate_progress(self):
+ return {"active": False, "step": 0, "total_steps": 0, "fraction": 0.0, "eta_seconds": None}
+
+ def cancel_generate(self, expected_account = None):
+ return False
+
+
+def routes(monkeypatch):
+ from fastapi import FastAPI
+ from fastapi.testclient import TestClient
+
+ import core.inference.diffusion_engine_router as engine_router
+ import core.inference.image_gallery as gallery_module
+ from auth.authentication import authenticated_via_api_key, get_current_subject
+ from routes.inference import studio_router
+
+ backend = _FluxBackend()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: backend)
+ saved: list[dict] = []
+
+ def _save(image, meta):
+ record = {**meta, "id": f"img{len(saved)}", "url": "/x"}
+ saved.append(record)
+ return record
+
+ monkeypatch.setattr(gallery_module, "save", _save)
+ app = FastAPI()
+ app.include_router(studio_router, prefix = "/api/inference")
+ app.dependency_overrides[get_current_subject] = lambda: "test-user"
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
+ return TestClient(app), backend, saved
+
+
+def test_generate_encodes_the_caption_and_records_both(routes, runnable):
+ client, backend, saved = routes
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ assert backend.calls[-1]["prompt"] == "Caption for a cat"
+ image = r.json()["images"][0]
+ assert image["prompt"] == "a cat"
+ assert image["enhanced_prompt"] == "Caption for a cat"
+ assert image["prompt_enhancement"] == "applied"
+
+
+def test_generate_restore_with_enhance_off_is_untouched(routes, runnable):
+ client, backend, saved = routes
+ r = client.post(
+ "/api/inference/images/generate",
+ json = {"prompt": "Caption for a cat", "enhance_prompt": False, "steps": 4},
+ )
+ assert r.status_code == 200, r.text
+ assert backend.calls[-1]["prompt"] == "Caption for a cat"
+ assert runnable.calls == []
+ image = r.json()["images"][0]
+ assert image["enhanced_prompt"] is None and image["prompt_enhancement"] == "off"
+
+
+def test_generate_prompts_list_one_upsample_each(routes, runnable):
+ client, backend, saved = routes
+ r = client.post(
+ "/api/inference/images/generate",
+ json = {"prompt": "a", "prompts": ["a", "b", "a"], "steps": 4},
+ )
+ assert r.status_code == 200, r.text
+ assert backend.calls[-1]["prompts"] == ["Caption for a", "Caption for b", "Caption for a"]
+ assert len(runnable.calls) == 2
+ assert [i["enhanced_prompt"] for i in r.json()["images"]] == [
+ "Caption for a",
+ "Caption for b",
+ "Caption for a",
+ ]
+
+
+def test_generate_other_family_passes_through(routes, runnable):
+ client, backend, saved = routes
+ backend.family = "flux.2-klein"
+ r = client.post(
+ "/api/inference/images/generate",
+ json = {"prompt": "a cat", "enhance_prompt": True, "steps": 4},
+ )
+ assert r.status_code == 200, r.text
+ assert backend.calls[-1]["prompt"] == "a cat" and runnable.calls == []
+ image = r.json()["images"][0]
+ assert image["enhanced_prompt"] is None and image["prompt_enhancement"] is None
+
+
+def test_generate_skips_the_upsampler_for_a_workflow_the_family_refuses(
+ routes, runnable, monkeypatch
+):
+ client, backend, saved = routes
+ monkeypatch.setattr(
+ backend,
+ "status",
+ lambda: {**_FluxBackend.status(backend), "workflows": ["txt2img", "reference"]},
+ )
+ client.post(
+ "/api/inference/images/generate",
+ json = {
+ "prompt": "make it night time",
+ "workflow": "edit",
+ "init_image": "data:,",
+ "steps": 4,
+ },
+ )
+ assert runnable.calls == []
+ client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert len(runnable.calls) == 1
+
+
+def test_unload_sidecar_never_creates_one(monkeypatch):
+ monkeypatch.setattr(pup, "_sidecar", None)
+ pup.unload_sidecar()
+ assert pup._sidecar is None
+ released = []
+ sidecar = pup.PromptUpsamplerSidecar()
+ monkeypatch.setattr(sidecar, "unload", lambda wait = True: released.append(wait))
+ monkeypatch.setattr(pup, "_sidecar", sidecar)
+ pup.unload_sidecar()
+ assert released == [False]
+
+
+def test_generate_upsampler_failure_never_fails_the_image(routes, runnable):
+ client, backend, saved = routes
+ runnable.error = RuntimeError("server died")
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ assert backend.calls[-1]["prompt"] == "a cat"
+ assert r.json()["images"][0]["prompt_enhancement"] == "failed"
+
+
+def test_generate_edit_sends_init_and_references(routes, runnable):
+ client, backend, saved = routes
+ r = client.post(
+ "/api/inference/images/generate",
+ json = {
+ "prompt": "swap the hat",
+ "init_image": _png_url(),
+ "reference_images": [_png_url((320, 320))],
+ "workflow": "edit",
+ "steps": 4,
+ },
+ )
+ assert r.status_code == 200, r.text
+ parts = runnable.calls[-1]["payload"]["messages"][1]["content"]
+ assert [p["type"] for p in parts] == ["image_url", "image_url", "text"]
+
+
+def test_upsampler_routes(routes, runnable, monkeypatch):
+ client, backend, saved = routes
+ monkeypatch.setattr(pup, "status", lambda: {"supported": True, "downloaded": True})
+ assert client.get("/api/inference/images/prompt-upsampler/status").json() == {
+ "supported": True,
+ "downloaded": True,
+ }
+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a gymnast"})
+ assert r.status_code == 200, r.text
+ assert r.json() == {"enhanced_prompt": "Caption for a gymnast"}
+ backend.family = "flux.3-klein-base"
+ client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "b"})
+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.0
+ client.post(
+ "/api/inference/images/prompt-upsampler/preview",
+ json = {"prompt": "c", "variant": "distilled"},
+ )
+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.05
+
+ started = {}
+ monkeypatch.setattr(
+ pup, "start_download", lambda token: started.setdefault("token", token) and "dl-1"
+ )
+ monkeypatch.setattr(pup, "download_status", lambda: {"downloading": True})
+ r = client.post("/api/inference/images/prompt-upsampler/download", json = {"hf_token": "hf_x"})
+ assert r.status_code == 200 and r.json()["download_id"] == "dl-1" and started["token"] == "hf_x"
+ r = client.post("/api/inference/images/prompt-upsampler/download")
+ assert r.status_code == 200
+ monkeypatch.setattr(pup, "cancel_download", lambda: True)
+ r = client.post("/api/inference/images/prompt-upsampler/cancel")
+ assert r.status_code == 200 and r.json()["cancelled"] is True
+
+
+def test_preview_errors(routes, monkeypatch):
+ client, backend, saved = routes
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")
+ monkeypatch.setattr(pup, "model_path", lambda: None)
+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a"})
+ assert r.status_code == 409
+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)
+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a"})
+ assert r.status_code == 501
+
+
+def test_images_status_route_carries_the_capabilities(routes, monkeypatch):
+ client, backend, saved = routes
+ from core.inference import diffusion_engine_router as engine_router
+
+ monkeypatch.setattr(engine_router, "_supported_family_capabilities", lambda: ("flux.3-klein",))
+ body = client.get("/api/inference/images/status").json()
+ assert body["supports_prompt_upsampler"] is True and body["prompt_upsampler_default"] is True
+ backend.family = "z-image"
+ body = client.get("/api/inference/images/status").json()
+ assert body["supports_prompt_upsampler"] is False and body["prompt_upsampler_default"] is False
+
+
+def test_download_status_keys_match_the_frontend():
+ assert {"downloading", "bytes_total", "bytes_done", "error", "cancelled"} <= set(
+ pup.download_status()
+ )
+
+
+def test_progress_reports_the_enhance_phase(routes, monkeypatch):
+ client, backend, saved = routes
+ monkeypatch.setattr(
+ pup, "progress", lambda: {"done": 0, "total": 1, "message": "Enhancing prompt..."}
+ )
+ body = client.get("/api/inference/images/generate-progress").json()
+ assert body["active"] is True and body["phase"] == "enhance"
+ assert body["message"] == "Enhancing prompt..."
+ monkeypatch.setattr(pup, "cancel_active", lambda: True)
+ stopped = []
+ monkeypatch.setattr(
+ backend, "cancel_generate", lambda **_: stopped.append(True) or False, raising = False
+ )
+ assert client.post("/api/inference/images/generate/cancel").json() == {"cancelled": True}
+ # Another tab's denoise is stopped too, not only the enhancement.
+ assert stopped == [True]
+
+
+def test_generate_retry_recomputes_the_caption_for_the_new_model(routes, runnable, monkeypatch):
+ from core.inference.diffusion_families import DiffusionModelReplacedError, load_identity
+
+ client, backend, saved = routes
+ original = backend.generate
+
+ def replaced_once(**kwargs):
+ if not backend.calls:
+ backend.calls.append({"replaced": True})
+ backend.family = "flux.2-klein"
+ raise DiffusionModelReplacedError(
+ load_identity("unsloth/flux", None, "flux.3-klein"),
+ load_identity("unsloth/flux", None, "flux.2-klein"),
+ )
+ return original(**kwargs)
+
+ monkeypatch.setattr(backend, "generate", replaced_once)
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ # FLUX.2 has no upsampler: the retry encodes the typed prompt, not the FLUX.3 caption.
+ assert backend.calls[-1]["prompt"] == "a cat"
+ assert r.json()["images"][0]["prompt_enhancement"] is None
+
+
+def test_single_user_generate_binds_the_caption_to_its_model(routes, runnable):
+ from core.inference.diffusion_families import load_identity
+
+ client, backend, saved = routes
+ status = backend.status()
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ # Bound on single-user installs too: a model swapped in after the caption raises and retries.
+ assert backend.calls[-1]["expected_load"] == load_identity(
+ status.get("repo_id"), status.get("base_repo"), "flux.3-klein"
+ )
+
+
+def test_a_cached_caption_still_honors_a_stop(runnable):
+ assert pup.upsample("a cat", variant = "distilled") == "Caption for a cat"
+ event = threading.Event()
+ event.set()
+ with pytest.raises(pup.UpsamplerCancelledError):
+ pup.upsample("a cat", variant = "distilled", cancel_event = event)
+
+
+def test_launch_publishes_loading_before_the_training_probe(monkeypatch):
+ sidecar = pup.PromptUpsamplerSidecar()
+ seen = []
+ monkeypatch.setattr(sidecar, "_spawn_server", lambda *a: seen.append(sidecar.is_loading()))
+ sidecar._launch("b", "m", None)
+ assert seen == [True] and not sidecar.is_loading()
+
+
+def test_an_idle_callback_waiting_on_the_lock_sees_the_request(monkeypatch):
+ sidecar = pup.PromptUpsamplerSidecar()
+ reaped = []
+ monkeypatch.setattr(pup, "_image_model_owns_gpu", lambda: True)
+ monkeypatch.setattr(sidecar, "_release_locked", lambda: reaped.append(True))
+
+ def launch(binary, model, mmproj):
+ # The keep-alive timer fired just before this request: its callback runs while the server starts.
+ sidecar._idle_unload(sidecar._generation)
+
+ monkeypatch.setattr(sidecar, "_launch", launch)
+ monkeypatch.setattr(pup, "_post_chat", lambda *a, **k: "caption")
+ assert sidecar.generate({}, binary = "b", model = "m", mmproj = None) == "caption"
+ assert reaped == [] and sidecar._active_requests == 0
+
+
+def test_unload_route_stops_the_upsampler_blocking(routes, monkeypatch):
+ client, backend, saved = routes
+ waits, cancels = [], []
+ monkeypatch.setattr(pup, "unload_sidecar", lambda wait = False: waits.append(wait))
+ monkeypatch.setattr(pup, "cancel_active", lambda: cancels.append(True))
+ monkeypatch.setattr(backend, "unload", lambda **_: {"loaded": False}, raising = False)
+ r = client.post("/api/inference/images/unload", json = {})
+ assert r.status_code == 200, r.text
+ assert waits == [True] and cancels == [True]
+
+
+def test_a_request_queued_before_a_blocking_unload_never_starts(monkeypatch):
+ sidecar = pup.PromptUpsamplerSidecar()
+ launched = []
+ monkeypatch.setattr(pup, "_image_model_owns_gpu", lambda: True)
+ monkeypatch.setattr(sidecar, "_launch", lambda *a: launched.append(a))
+ monkeypatch.setattr(pup, "_post_chat", lambda *a, **k: "caption")
+ sidecar._request_lock.acquire()
+ errors = []
+
+ def queued():
+ try:
+ sidecar.generate({}, binary = "b", model = "m", mmproj = None)
+ except pup.UpsamplerUnavailableError as exc:
+ errors.append(exc)
+
+ worker = threading.Thread(target = queued)
+ worker.start()
+ time.sleep(0.2)
+ # Training starts while that request waits on the lock: the drain finds nothing active and returns.
+ sidecar.unload(wait = True)
+ sidecar._request_lock.release()
+ worker.join(5)
+ assert errors and launched == []
+ # A request after the handover is a new one.
+ assert sidecar.generate({}, binary = "b", model = "m", mmproj = None) == "caption"
+
+
+def test_an_unowned_gpu_starts_the_upsampler_on_the_cpu(monkeypatch):
+ from core.inference import gpu_arbiter
+
+ monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: None)
+ assert pup._gpu_unowned()
+ monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: gpu_arbiter.DIFFUSION)
+ assert not pup._gpu_unowned()
+
+
+def test_the_child_is_pinned_to_the_image_models_card(monkeypatch):
+ import torch
+
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 4)
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,5,GPU-abc")
+ assert pup._child_device_mask(2) == {"CUDA_VISIBLE_DEVICES": "GPU-abc"}
+ assert pup._child_device_mask(1) == {"CUDA_VISIBLE_DEVICES": "5"}
+ assert pup._child_device_mask(None) == {}
+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")
+ assert pup._child_device_mask(1) == {"CUDA_VISIBLE_DEVICES": "1"}
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ assert pup._child_device_mask(0) == {}
+
+
+def test_references_follow_the_generate_decoders_side_limit():
+ with pytest.raises(ValueError):
+ pup.prepare_references([_png_url((4100, 64))])
+
+
+def test_generate_skips_the_upsampler_while_another_account_runs(routes, runnable, monkeypatch):
+ from hub.services.models import account_access
+
+ client, backend, saved = routes
+
+ def busy(path = None):
+ # The combined guard: another account's image or video job, not only chat.
+ raise account_access.GpuBusyForAnotherAccountError("other", 1)
+
+ monkeypatch.setattr(account_access, "require_idle_other_accounts", busy)
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ assert runnable.calls == [] and backend.calls[-1]["prompt"] == "a cat"
+ assert r.json()["images"][0]["prompt_enhancement"] == "unavailable"
+
+
+def test_preview_is_refused_while_another_account_runs(routes, runnable, monkeypatch):
+ from hub.services.models import account_access
+
+ client, backend, saved = routes
+
+ def busy(path = None):
+ raise account_access.GpuBusyForAnotherAccountError("other", 1)
+
+ monkeypatch.setattr(account_access, "require_idle_other_accounts", busy)
+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a cat"})
+ assert r.status_code == 409
+ assert runnable.calls == []
+
+
+def test_references_are_decoded_one_at_a_time(monkeypatch):
+ live = []
+ peak = []
+ real_decode, real_prepare = pup.decode_reference, pup.prepare_reference
+
+ def decode(data):
+ live.append(1)
+ peak.append(len(live))
+ return real_decode(data)
+
+ def prepare(image, count):
+ out = real_prepare(image, count)
+ live.pop()
+ return out
+
+ monkeypatch.setattr(pup, "decode_reference", decode)
+ monkeypatch.setattr(pup, "prepare_reference", prepare)
+ refs = pup.prepare_references([_png_url((256, 256))] * 4)
+ assert len(refs) == 4 and max(peak) == 1
+
+
+def test_stop_during_server_start_cancels_the_start(monkeypatch):
+ sidecar = pup.PromptUpsamplerSidecar()
+ started, cancelled = threading.Event(), threading.Event()
+
+ def launch(binary, model, mmproj):
+ sidecar._loading = True
+ started.set()
+ try:
+ assert sidecar._load_cancel.wait(5), "Stop never reached the starting server"
+ cancelled.set()
+ raise pup.UpsamplerCancelledError("The prompt upsampler load was cancelled.")
+ finally:
+ sidecar._loading = False
+
+ monkeypatch.setattr(sidecar, "_launch", launch)
+ event = threading.Event()
+ threading.Thread(target = lambda: (started.wait(5), event.set()), daemon = True).start()
+ with pytest.raises(pup.UpsamplerCancelledError):
+ sidecar.generate({}, binary = "b", model = "m", mmproj = None, cancel_event = event)
+ assert cancelled.is_set()
+
+
+def test_training_stops_an_enhancement_that_has_not_reached_the_sidecar(monkeypatch):
+ import sys
+
+ from routes import training_vram
+
+ stopped = []
+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ monkeypatch.setattr(pup, "cancel_active", lambda: stopped.append(True) or True)
+ monkeypatch.setattr(pup, "get_sidecar", lambda: pup.PromptUpsamplerSidecar())
+ assert training_vram.free_prompt_upsampler_for_training("test") == []
+ assert stopped == [True]
+
+
+class _ResidentUpsampler:
+ loaded = True
+
+ def __init__(self):
+ self.unloaded = []
+
+ def holds_vram(self):
+ return True
+
+ def is_loading(self):
+ return False
+
+ def unload(self, wait = True):
+ self.unloaded.append(wait)
+
+
+def test_training_stops_the_upsampler_even_when_it_is_the_only_resident(monkeypatch):
+ import sys
+
+ from routes import training_vram
+
+ resident = _ResidentUpsampler()
+ monkeypatch.setattr(pup, "get_sidecar", lambda: resident)
+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ idle = {"model": None, "device": None, "loading": False, "any": False}
+ monkeypatch.setattr(training_vram, "summarize_resident_chat", lambda: dict(idle))
+ monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: dict(idle))
+ freed = training_vram.coordinate_models_for_training(lambda: (True, {}))
+ assert freed == ["image:prompt-upsampler"] and resident.unloaded
+
+
+def test_arbiter_diffusion_eviction_stops_the_upsampler(monkeypatch):
+ import core.inference.diffusion_engine_router as engine_router
+ from core.inference import gpu_arbiter
+
+ class _Engine:
+ def unload(self):
+ pass
+
+ sidecar = _ResidentUpsampler()
+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: _Engine())
+ monkeypatch.setattr(pup, "_sidecar", sidecar)
+ import sys
+
+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ event = threading.Event()
+ with pup._tracking(1, event):
+ gpu_arbiter._evict_diffusion()
+ # The run answering is stopped first, so the handover does not wait out its caption.
+ assert event.is_set()
+ assert sidecar.unloaded == [True]
+
+
+def test_every_drop_of_the_diffusion_claim_stops_the_upsampler(monkeypatch):
+ import sys
+
+ from core.inference import gpu_arbiter
+
+ stops = []
+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)
+ monkeypatch.setattr(pup, "stop_for_handover", lambda: stops.append(gpu_arbiter.current_owner()))
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)
+ # A switch to a CPU-only image model releases the claim without an eviction.
+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)
+ assert gpu_arbiter.release_if(gpu_arbiter.DIFFUSION, lambda: True)
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ gpu_arbiter.release(gpu_arbiter.CHAT)
+ # Stopped while DIFFUSION still held the claim, so no other owner can start beside it.
+ assert stops == [gpu_arbiter.DIFFUSION, gpu_arbiter.DIFFUSION]
+
+
+def test_a_failed_enhancement_still_binds_the_generation_to_its_model(routes, runnable):
+ from core.inference.diffusion_families import load_identity
+
+ client, backend, saved = routes
+ runnable.error = RuntimeError("boom")
+ status = backend.status()
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 200, r.text
+ assert r.json()["images"][0]["prompt_enhancement"] == "failed"
+ assert backend.calls[-1]["expected_load"] == load_identity(
+ status.get("repo_id"), status.get("base_repo"), "flux.3-klein"
+ )
+
+
+def test_a_queued_enhancement_keeps_the_active_one_tracked_and_stop_reaches_both():
+ first, second = threading.Event(), threading.Event()
+ with pup._tracking(1, first) as active:
+ with pup._tracking(2, second):
+ # The run holding llama-server is reported, not the one queued behind it.
+ assert pup.progress()["total"] == 1
+ pup._advance(active)
+ assert pup.progress()["done"] == 1
+ # The queued run leaving does not clear the active one.
+ assert pup.progress()["total"] == 1
+ with pup._tracking(2, second):
+ assert pup.cancel_active() is True
+ assert first.is_set() and second.is_set()
+ assert pup.progress() is None and pup.cancel_active() is False
+
+
+def test_model_and_projector_come_from_one_revision(monkeypatch):
+ files = {
+ ("rev_new", pup.UPSAMPLER_MODEL_FILE): "/c/new/model",
+ ("rev_old", pup.UPSAMPLER_MODEL_FILE): "/c/old/model",
+ ("rev_old", pup.UPSAMPLER_MMPROJ_FILE): "/c/old/mmproj",
+ }
+ monkeypatch.delenv(pup.ENV_MODEL_OVERRIDE, raising = False)
+ monkeypatch.delenv(pup.ENV_MMPROJ_OVERRIDE, raising = False)
+ monkeypatch.setattr(pup, "_candidate_revisions", lambda hub_cache = None: ["rev_new", "rev_old"])
+ monkeypatch.setattr(
+ pup, "_cached_file", lambda name, hub_cache = None, revision = None: files.get((revision, name))
+ )
+ # rev_new has no projector, so the pair is rev_old's, never rev_new's model with rev_old's projector.
+ assert (pup.model_path(), pup.mmproj_path()) == ("/c/old/model", "/c/old/mmproj")
+ del files[("rev_old", pup.UPSAMPLER_MMPROJ_FILE)]
+ files[("rev_new", pup.UPSAMPLER_MMPROJ_FILE)] = None
+ assert (pup.model_path(), pup.mmproj_path()) == ("/c/new/model", None)
+ assert pup._cached_download_paths(pup.UPSAMPLER_ID) is None
+
+
+def test_no_repo_download_beside_an_override(routes, monkeypatch):
+ client, backend, saved = routes
+ started = []
+ monkeypatch.setattr(pup, "status", lambda: {"override": True})
+ monkeypatch.setattr(pup, "start_download", lambda token: started.append(token) or "dl")
+ r = client.post("/api/inference/images/prompt-upsampler/download")
+ assert r.status_code == 409 and "UNSLOTH_FLUX3_UPSAMPLER_GGUF" in r.text and started == []
+
+
+def test_a_systemic_failure_runs_the_rest_of_the_batch_as_typed(runnable):
+ runnable.error = RuntimeError("llama-server did not become ready for the prompt upsampler.")
+ out = pup.enhance_for_generation(
+ ["a", "b", "c"], family_name = "flux.3-klein", enhance_prompt = None
+ )
+ # One start attempt, not one per prompt; every prompt falls back.
+ assert len(runnable.calls) == 1
+ assert out.statuses == {"a": "failed", "b": "failed", "c": "failed"} and out.captions == {}
+
+
+def test_no_sidecar_start_after_the_gpu_moved_to_another_owner(monkeypatch):
+ from core.inference import gpu_arbiter
+
+ sidecar = pup.PromptUpsamplerSidecar()
+ launched = []
+ monkeypatch.setattr(sidecar, "_launch", lambda *a: launched.append(a))
+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)
+ with pytest.raises(pup.UpsamplerUnavailableError, match = "no longer holds the GPU"):
+ sidecar.generate({}, binary = "b", model = "m", mmproj = None)
+ assert launched == []
+
+
+def test_the_preview_caps_the_aggregate_image_payload():
+ from pydantic import ValidationError
+
+ from models.inference import PromptUpsamplerPreviewRequest
+
+ big = "x" * (30 * 1024 * 1024)
+ PromptUpsamplerPreviewRequest(prompt = "p", reference_images = [big] * 4)
+ with pytest.raises(ValidationError, match = "at most 128 MiB"):
+ PromptUpsamplerPreviewRequest(prompt = "p", reference_images = [big] * 5)
+
+
+def test_a_managed_account_is_authorized_before_the_upsampler_runs(routes, runnable, monkeypatch):
+ from fastapi import HTTPException
+
+ from hub.services.models import account_access
+
+ client, backend, saved = routes
+ monkeypatch.setattr(account_access, "managed_account", lambda: True)
+
+ def deny(status, modality = None):
+ raise HTTPException(status_code = 404, detail = "Model not found")
+
+ monkeypatch.setattr(account_access, "require_media_generation_access", deny)
+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})
+ assert r.status_code == 404 and runnable.calls == []
diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.py
index d1a913dad41..10f3895870f 100644
--- a/studio/backend/tests/test_image_gallery.py
+++ b/studio/backend/tests/test_image_gallery.py
@@ -706,3 +706,10 @@ def test_connection_images_record_no_local_engine(monkeypatch):
remote = gallery.save(_img(), {**base, "model_kind": "connection"})
assert local["engine"] == "diffusers"
assert remote["engine"] is None
+
+
+def test_parameters_chunk_carries_the_encoded_caption():
+ meta = {**_meta(), "prompt": "a cat", "enhanced_prompt": "A tabby cat on a sunlit sill."}
+ text = gallery._params_text(meta)
+ assert text.splitlines()[0] == "A tabby cat on a sunlit sill."
+ assert gallery._params_text({**_meta(), "prompt": "a cat"}).splitlines()[0] == "a cat"
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index df0063cad8c..b28be02b76c 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -3871,6 +3871,7 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():
"core/inference/audio_cpp_server.py",
"core/inference/diffusion_vae_prebuild.py",
"core/inference/engine_install.py",
+ "core/inference/flux3_prompt_upsampler.py",
"core/inference/lemonade_server.py",
"core/inference/llama_cpp.py",
"core/inference/managed_engine.py",
diff --git a/studio/backend/tests/test_stt_http_downloads.py b/studio/backend/tests/test_stt_http_downloads.py
index 6d8b5636ff2..0991a8a5bc1 100644
--- a/studio/backend/tests/test_stt_http_downloads.py
+++ b/studio/backend/tests/test_stt_http_downloads.py
@@ -272,6 +272,25 @@ def test_selected_file_progress_covers_partial_final_and_windows_snapshot(tmp_pa
assert stt_sidecar._downloaded_file_bytes(**kwargs) == 100
+def test_selected_file_progress_counts_the_process_unique_partial(tmp_path):
+ # huggingface_hub >= 1.18 names the partial `<etag>.<8 hex>.incomplete`.
+ repo = "Org/Repo"
+ blob_key = "c" * 64
+ blobs = _repo_dir(tmp_path, repo) / "blobs"
+ blobs.mkdir(parents = True)
+ (blobs / f"{blob_key}.ca4ea4fe.incomplete").write_bytes(b"x" * 30)
+ (blobs / f"{'d' * 64}.0badf00d.incomplete").write_bytes(b"x" * 90)
+ kwargs = {
+ "hub_cache": tmp_path,
+ "repo": repo,
+ "filename": "model.gguf",
+ "size": 100,
+ "blob_key": blob_key,
+ "revision": None,
+ }
+ assert stt_sidecar._downloaded_file_bytes(**kwargs) == 30
+
+
def test_mtmd_progress_counts_only_its_two_exact_files(tmp_path):
model_id = "qwen3-asr-0.6b"
spec = stt_mtmd_sidecar.MTMD_STT_MODELS[model_id]
diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py
index 2a99e717d3e..902bc2b2515 100644
--- a/studio/backend/utils/llama_cpp_update.py
+++ b/studio/backend/utils/llama_cpp_update.py
@@ -777,6 +777,7 @@ def _run_llama_phase(
# The mtmd dictation sidecar serves Qwen3-ASR from this same llama-server out of this same tree, so a live one locks the exe on Windows and a concurrent load would start against a half-swapped install.
model_was_active = _block_mtmd_sidecar(mtmd_guard) or model_was_active
+ model_was_active = _block_prompt_upsampler(mtmd_guard) or model_was_active
cmd = [
sys.executable,
@@ -961,6 +962,19 @@ def _block_mtmd_sidecar(stack: ExitStack) -> bool:
return False
+def _block_prompt_upsampler(stack: ExitStack) -> bool:
+ """The FLUX.3 prompt upsampler runs the same llama-server; hold its maintenance guard for the install, only
+ if Images ever loaded the module. Not fail-closed, like the mtmd guard."""
+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")
+ if module is None:
+ return False
+ try:
+ return stack.enter_context(module.get_sidecar().update_maintenance())
+ except Exception as exc: # noqa: BLE001 - the update proceeds without it
+ logger.debug("llama update: prompt upsampler coordination failed", error = str(exc))
+ return False
+
+
# Combined-job progress split when every phase runs (the llama bundle dwarfs the others); normalized to 0..1 over the phases that run.
_LLAMA_PHASE_WEIGHT = 0.7
_WHISPER_PHASE_WEIGHT = 0.3
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index cd6a6f4da10..abe4ba05efb 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -9,6 +9,13 @@ import {
MEMORY_REFUSAL_HEADER,
MemoryEstimateRefusalError,
} from "./lib/memory-refusal";
+import {
+ parsePromptUpsamplerStatus,
+ PROMPT_UPSAMPLER_ROUTES,
+ type PromptEnhancement,
+ type PromptUpsamplerPreviewRequest,
+ type PromptUpsamplerStatus,
+} from "./lib/prompt-enhance";
// One Advanced control's resolved value and provenance, for the Advanced-panel badges. `value` is the engaged
// value (null when off), `requested` is what the caller asked for (null = left to the backend), `source` is "auto"
@@ -83,6 +90,10 @@ export interface DiffusionStatus {
resolved?: Record<string, DiffusionResolvedControl> | null;
// Default steps / guidance the backend renders the loaded model with (file header before base repo).
generation_defaults?: { steps?: number; guidance?: number } | null;
+ // Whether the loaded family can run the prompt upsampler (FLUX.3), and whether it is on by default. Absent on
+ // an older backend: no Enhance prompt toggle.
+ supports_prompt_upsampler?: boolean;
+ prompt_upsampler_default?: boolean;
}
export interface DiffusionSamplerOption {
@@ -113,8 +124,10 @@ export interface DiffusionGenerateProgress {
total_steps: number;
fraction: number;
eta_seconds: number | null;
- // Absent (sd.cpp engine) means "denoise". "encode" runs before the denoise loop starts.
- phase?: "encode" | "denoise" | "decode" | null;
+ // Absent (sd.cpp engine) means "denoise". "encode" runs before the denoise loop starts; "enhance" is the FLUX.3
+ // prompt upsampler writing the caption before the run (with `message`).
+ phase?: "enhance" | "encode" | "denoise" | "decode" | null;
+ message?: string | null;
// Live latent preview of the image being denoised (small JPEG data URL), and a counter that moves with each one.
preview?: string | null;
preview_seq?: number;
@@ -215,6 +228,9 @@ export interface DiffusionGenerateRequest {
controlnet?: ControlNetSpecInput;
sampler?: string;
schedule_type?: string;
+ // Rewrite the prompt with the family's prompt upsampler first. null/absent = the family default. Sent only for a
+ // family that supports it.
+ enhance_prompt?: boolean | null;
}
// One LoRA selection sent with a generation.
@@ -303,6 +319,9 @@ export interface GalleryImage {
// null = the model's shipped scheduler; schedule_type is the resolved schedule.
sampler?: string | null;
schedule_type?: string | null;
+ // The caption the prompt upsampler wrote and the model actually ran; `prompt` stays the user's text.
+ enhanced_prompt?: string | null;
+ prompt_enhancement?: PromptEnhancement | null;
created_at: number;
// Library state, not recipe: stored beside the PNG, absent on records written before this existed.
pinned?: boolean;
@@ -494,6 +513,43 @@ export async function unloadDiffusionModel(): Promise<DiffusionStatus> {
return parseJson(await authFetch("/api/inference/images/unload", { method: "POST" }));
}
+export async function getPromptUpsamplerStatus(signal?: AbortSignal): Promise<PromptUpsamplerStatus> {
+ return parsePromptUpsamplerStatus(
+ await parseJson<unknown>(await authFetch(PROMPT_UPSAMPLER_ROUTES.status, { signal })),
+ );
+}
+
+/** Start the background download of the prompt upsampler. Progress is read back from the status route. */
+export async function startPromptUpsamplerDownload(hfToken?: string): Promise<void> {
+ await parseJson<unknown>(
+ await authFetch(PROMPT_UPSAMPLER_ROUTES.download, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(hfToken ? { hf_token: hfToken } : {}),
+ }),
+ );
+}
+
+/** Stop the download. Partial files stay cached, so a later download resumes. */
+export async function cancelPromptUpsamplerDownload(): Promise<void> {
+ await parseJson<unknown>(
+ await authFetch(PROMPT_UPSAMPLER_ROUTES.cancel, { method: "POST" }),
+ );
+}
+
+/** Rewrite a prompt without generating, so the caption can be read or edited first. */
+export async function previewEnhancedPrompt(
+ body: PromptUpsamplerPreviewRequest,
+): Promise<{ enhanced_prompt: string }> {
+ return parseJson(
+ await authFetch(PROMPT_UPSAMPLER_ROUTES.preview, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(body),
+ }),
+ );
+}
+
/** List diffusion LoRA adapters, optionally filtered to a model family. */
export async function listDiffusionLoras(family?: string): Promise<DiffusionLoraInfo[]> {
const qs = family ? `?family=${encodeURIComponent(family)}` : "";
diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx
index 8e91a080c21..7520888b6d4 100644
--- a/studio/frontend/src/features/images/images-page.tsx
+++ b/studio/frontend/src/features/images/images-page.tsx
@@ -320,6 +320,20 @@ import {
SCHEDULE_LABELS,
withSamplerChoice,
} from "./lib/sampler-choice";
+import {
+ enhancePromptRequestField,
+ enhanceRestoreFor,
+ enhancementNotice,
+ hasEnhancedPrompt,
+ ENHANCE_UNSUPPORTED_FAMILY,
+ overrideAfterFamilyChange,
+ promptUpsamplerCapability,
+ readEnhanceChoices,
+ resolveEnhancePrompt,
+ upsamplerVariantFor,
+ withEnhanceChoice,
+} from "./lib/prompt-enhance";
+import { EnhancedPromptRecipeRow, EnhancePromptControls } from "./prompt-enhance-panel";
import { useNavigate, useSearch } from "@tanstack/react-router";
import { useStagedDownload, type StagedDownloadEntry } from "@/features/hub/download-manager";
import { DiffusionTrainPanel } from "./train/diffusion-train-panel";
@@ -1146,7 +1160,7 @@ function RecipePopover({
active,
}: {
image: GalleryImage;
- onRestore: (image: GalleryImage) => void;
+ onRestore: (image: GalleryImage, exactCaption?: boolean) => void;
active: boolean;
}) {
// Controlled + force-closed off-tab: PopoverContent portals to body, so the inert page wrapper cannot contain it.
@@ -1177,6 +1191,9 @@ function RecipePopover({
</div>
<div className="flex min-h-0 flex-col gap-2 overflow-y-auto overscroll-contain px-4 py-3 text-xs">
<RecipeRow label="Prompt" value={image.prompt} wrap />
+ {hasEnhancedPrompt(image) ? (
+ <EnhancedPromptRecipeRow caption={(image.enhanced_prompt ?? "").trim()} />
+ ) : null}
{image.negative_prompt ? (
<RecipeRow label="Negative" value={image.negative_prompt} wrap />
) : null}
@@ -1267,6 +1284,17 @@ function RecipePopover({
<HugeiconsIcon icon={Refresh01Icon} className="size-4" />
Restore these settings
</Button>
+ {/* The caption that actually ran, with Enhance prompt off, so the run reproduces without the upsampler. */}
+ {image.prompt_enhancement === "applied" && hasEnhancedPrompt(image) ? (
+ <Button
+ size="sm"
+ variant="ghost"
+ className="mt-1 w-full gap-1.5"
+ onClick={() => onRestore(image, true)}
+ >
+ Restore with the enhanced prompt
+ </Button>
+ ) : null}
</div>
</PopoverContent>
</Popover>
@@ -1495,6 +1523,9 @@ export function ImagesPage({
const [steps, setSteps] = useState(DEFAULT_GEN.steps);
const [guidance, setGuidance] = useState(DEFAULT_GEN.guidance);
const [samplerChoices, setSamplerChoices] = useState(readSamplerChoices);
+ const [enhanceChoices, setEnhanceChoices] = useState(readEnhanceChoices);
+ // The family it was set under ("" = before a model loaded, which applies to whichever family loads next).
+ const [enhanceOverrideState, setEnhanceOverrideState] = useState<{ family: string; on: boolean } | null>(null);
// A recipe restored while unloaded: Generate recalls the model, uses this, and stores it for that model.
const [restoredSampler, setRestoredSampler] = useState<Partial<SamplerChoice> | null>(null);
// Whether the user has taken the recipe since the pick still waiting for its status: a preset
@@ -1975,6 +2006,41 @@ export function ImagesPage({
},
[samplerChoice, samplerModelKey, samplerOptions],
);
+ // Enhance prompt (FLUX.3's prompt upsampler): stored per family, defaulting to the family's own default. A
+ // restore or "Use as prompt" sets a one-off override that the next toggle or family change drops.
+ const upsamplerCapability = promptUpsamplerCapability(status, Boolean(external));
+ const enhanceFamilyKey = upsamplerCapability.supported ? (status?.family ?? "") : "";
+ // A family change consumes the override (overrideAfterFamilyChange), a switch to a model or connection without an
+ // upsampler included.
+ const enhanceTrackKey =
+ enhanceFamilyKey || (external || status?.loaded ? ENHANCE_UNSUPPORTED_FAMILY : "");
+ useEffect(() => {
+ if (enhanceTrackKey) setEnhanceOverrideState((prev) => overrideAfterFamilyChange(prev, enhanceTrackKey));
+ }, [enhanceTrackKey]);
+ const enhanceOverride =
+ enhanceOverrideState &&
+ (enhanceOverrideState.family === "" || enhanceOverrideState.family === enhanceFamilyKey)
+ ? enhanceOverrideState.on
+ : null;
+ const setEnhanceOverride = useCallback(
+ (on: boolean | null) =>
+ setEnhanceOverrideState(on === null ? null : { family: enhanceFamilyKey, on }),
+ [enhanceFamilyKey],
+ );
+ const enhanceOn = resolveEnhancePrompt({
+ capability: upsamplerCapability,
+ stored: enhanceChoices[enhanceFamilyKey],
+ override: enhanceOverride,
+ });
+ const changeEnhancePrompt = useCallback(
+ (on: boolean) => {
+ setEnhanceOverride(null);
+ setEnhanceChoices((all) =>
+ withEnhanceChoice(all, enhanceFamilyKey, on, upsamplerCapability.defaultOn),
+ );
+ },
+ [enhanceFamilyKey, setEnhanceOverride, upsamplerCapability.defaultOn],
+ );
useEffect(() => {
if (!controlnetSupported) {
setAvailableControlNets([]);
@@ -2469,7 +2535,7 @@ export function ImagesPage({
[dropFromStrip],
);
- const restoreSettings = useCallback((image: GalleryImage) => {
+ const restoreSettings = useCallback((image: GalleryImage, exactCaption = false) => {
const viaConnection = image.model_kind === "connection";
// Negative prompt only applies when guidance>0; do not restore a hidden value.
const restoredNegative = image.guidance > 0 ? (image.negative_prompt ?? "") : "";
@@ -2519,7 +2585,11 @@ export function ImagesPage({
const reopened: WorkflowId =
image.workflow === "edit" ? "edit" : image.workflow === "reference" ? "reference" : "create";
setWorkflow(reopened);
- setPromptFor(reopened, image.prompt);
+ // An enhanced run restores the user's prompt with Enhance prompt on, or (exactCaption) the caption it ran with
+ // the toggle off. Records without an outcome leave the toggle alone.
+ const enhanceRestore = enhanceRestoreFor(image, exactCaption);
+ setPromptFor(reopened, enhanceRestore.prompt);
+ if (enhanceRestore.enhance !== null) setEnhanceOverride(enhanceRestore.enhance);
setInitImage(null);
setMaskImage(null);
setReferenceImages(
@@ -2549,7 +2619,7 @@ export function ImagesPage({
} else {
toast.success("Settings restored to inputs", rescaled);
}
- }, [changeSamplerChoice, samplerOptions.length, setPromptFor, setWorkflow, sizeLimits]);
+ }, [changeSamplerChoice, samplerOptions.length, setEnhanceOverride, setPromptFor, setWorkflow, sizeLimits]);
// Like dropping an image on ComfyUI: Unsloth's recipe first, else AUTOMATIC1111 / ComfyUI metadata.
const importFileRef = useRef<HTMLInputElement>(null);
@@ -4601,6 +4671,7 @@ export function ImagesPage({
allow_oversized: allowOversizedSent ? true : undefined,
live_preview: livePreview,
...samplerRequestFields(samplerOptions, samplerChoice),
+ ...enhancePromptRequestField(upsamplerCapability, enhanceOn),
...condFields,
// Drop empty and zero-weight rows and trim hand-typed repo ids, so the recipe records only
// adapters that applied. Gated on loraCapable, since a restore can leave adapters in state.
@@ -4647,6 +4718,11 @@ export function ImagesPage({
setImages((prev) => mergeGenerated(prev, res.images));
res.images.forEach((image) => knownIds.add(image.id));
if (res.images[0]) setSelectedId(res.images[0].id);
+ // Asked for enhancement and did not get it: say so once, without blocking the result.
+ if (enhanceOn && upsamplerCapability.supported) {
+ const notice = enhancementNotice(res.images[0]?.prompt_enhancement);
+ if (notice) toast.info(notice.title, { id: "prompt-enhancement-notice", description: notice.description });
+ }
res.images.forEach((image) => void ensureSrc(image));
setGenDone(i + 1);
}
@@ -4688,7 +4764,7 @@ export function ImagesPage({
setGenStep(null);
setStopping(false);
}
- }, [external, allowOversized, livePreview, prompt, negativePrompt, negativeCapable, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
+ }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeCapable, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);
// Stop the in-flight generation. Latch FIRST, so a multi-run request stops even if the POST
// races the run that is already finishing.
@@ -5644,6 +5720,29 @@ export function ImagesPage({
onChange={(e) => setPrompt(e.target.value)}
/>
</Field>
+ {upsamplerCapability.supported && (
+ <EnhancePromptControls
+ enabled={enhanceOn}
+ onEnabledChange={changeEnhancePrompt}
+ prompt={prompt}
+ previewRequest={() => {
+ // Edit and Reference condition on their images, so the caption describes them too.
+ const conditioned = workflow === "edit" || workflow === "reference";
+ const images = conditioned
+ ? [initImage, ...referenceImages].filter((src): src is string => Boolean(src))
+ : [];
+ return {
+ prompt: prompt.trim(),
+ variant: upsamplerVariantFor(status?.family),
+ ...(images.length ? { reference_images: images } : {}),
+ };
+ }}
+ onUseCaption={(caption) => {
+ setPrompt(caption);
+ setEnhanceOverride(false);
+ }}
+ />
+ )}
{negativeCapable && (
<NegativePromptField
value={negativePrompt}
diff --git a/studio/frontend/src/features/images/lib/image-metadata.ts b/studio/frontend/src/features/images/lib/image-metadata.ts
index 755f03ba2b4..e466d1c0c8b 100644
--- a/studio/frontend/src/features/images/lib/image-metadata.ts
+++ b/studio/frontend/src/features/images/lib/image-metadata.ts
@@ -619,6 +619,9 @@ export interface RestorableRecipe {
model_kind?: string;
sampler?: string;
schedule_type?: string;
+ // The prompt upsampler's caption and outcome; restore decides the Enhance prompt toggle from them.
+ enhanced_prompt?: string | null;
+ prompt_enhancement?: "applied" | "off" | "unavailable" | "failed" | null;
}
export const APPLIED_RECIPE_KEYS = [
@@ -640,6 +643,8 @@ export const APPLIED_RECIPE_KEYS = [
"localized_edit",
"sampler",
"schedule_type",
+ "enhanced_prompt",
+ "prompt_enhancement",
] as const;
/** Known but not applied: loading a model or a ControlNet image stays the user's call. */
@@ -767,6 +772,14 @@ export function recipeSettings(
model_kind: r.model_kind === "connection" ? "connection" : undefined,
sampler: str(r.sampler),
schedule_type: str(r.schedule_type),
+ enhanced_prompt: str(r.enhanced_prompt) ?? null,
+ prompt_enhancement:
+ r.prompt_enhancement === "applied" ||
+ r.prompt_enhancement === "off" ||
+ r.prompt_enhancement === "unavailable" ||
+ r.prompt_enhancement === "failed"
+ ? r.prompt_enhancement
+ : null,
};
}
diff --git a/studio/frontend/src/features/images/lib/prompt-enhance.ts b/studio/frontend/src/features/images/lib/prompt-enhance.ts
new file mode 100644
index 00000000000..cd5c819ac7a
--- /dev/null
+++ b/studio/frontend/src/features/images/lib/prompt-enhance.ts
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// "Enhance prompt": the FLUX.3 prompt upsampler rewrites a short request into the detailed caption the model was
+// trained on. Pure logic only (no "@/" imports) so node:test can load it directly.
+
+/** Every upsampler route, in one place. */
+export const PROMPT_UPSAMPLER_ROUTES = {
+ status: "/api/inference/images/prompt-upsampler/status",
+ download: "/api/inference/images/prompt-upsampler/download",
+ cancel: "/api/inference/images/prompt-upsampler/cancel",
+ preview: "/api/inference/images/prompt-upsampler/preview",
+} as const;
+
+export const ENHANCE_PROMPT_LABEL = "Enhance prompt";
+export const ENHANCE_PROMPT_HINT =
+ "Rewrites your prompt into a detailed caption with FLUX.3's prompt upsampler before generating. Fixes most broken poses and hands. Turn off to use your prompt exactly as typed.";
+
+export type PromptEnhancement = "applied" | "off" | "unavailable" | "failed";
+
+/** Same shape as the speech-to-text model download status (SttDownloadStatus). */
+export interface PromptUpsamplerDownload {
+ downloading: boolean;
+ model: string | null;
+ download_id?: string | null;
+ completed_download_ids?: string[];
+ variant?: string | null;
+ error: string | null;
+ cancelled?: boolean;
+ cancelled_model?: string | null;
+ bytes_total: number | null;
+ bytes_done: number | null;
+}
+
+export interface PromptUpsamplerStatus {
+ supported: boolean;
+ downloaded: boolean;
+ loaded: boolean;
+ repo: string;
+ filename: string;
+ size_bytes: number;
+ mmproj_size_bytes: number;
+ mmproj_downloaded: boolean;
+ // Local files from the env overrides: no repo download is offered beside them.
+ override: boolean;
+ download: PromptUpsamplerDownload;
+}
+
+export interface PromptUpsamplerPreviewRequest {
+ prompt: string;
+ reference_images?: string[];
+ variant?: "distilled" | "base";
+}
+
+const IDLE_DOWNLOAD: PromptUpsamplerDownload = {
+ downloading: false,
+ model: null,
+ error: null,
+ bytes_total: null,
+ bytes_done: null,
+};
+
+const num = (v: unknown): number | null =>
+ typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
+const str = (v: unknown): string | null => (typeof v === "string" ? v : null);
+
+/** Tolerant parse of the status body: a missing or malformed field reads as "not there", never a throw. */
+export function parsePromptUpsamplerStatus(raw: unknown): PromptUpsamplerStatus {
+ const body = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};
+ const d =
+ body.download && typeof body.download === "object"
+ ? (body.download as Record<string, unknown>)
+ : {};
+ const download: PromptUpsamplerDownload = {
+ ...IDLE_DOWNLOAD,
+ downloading: d.downloading === true,
+ model: str(d.model),
+ error: str(d.error),
+ bytes_total: num(d.bytes_total),
+ bytes_done: num(d.bytes_done),
+ };
+ if (typeof d.download_id === "string") download.download_id = d.download_id;
+ if (typeof d.cancelled === "boolean") download.cancelled = d.cancelled;
+ return {
+ supported: body.supported === true,
+ downloaded: body.downloaded === true,
+ loaded: body.loaded === true,
+ repo: str(body.repo) ?? "",
+ filename: str(body.filename) ?? "",
+ size_bytes: num(body.size_bytes) ?? 0,
+ mmproj_size_bytes: num(body.mmproj_size_bytes) ?? 0,
+ mmproj_downloaded: body.mmproj_downloaded === true,
+ override: body.override === true,
+ download,
+ };
+}
+
+/** The model or its vision projector (edits) is missing, so the Download control stays offered. */
+export function upsamplerNeedsDownload(status: PromptUpsamplerStatus): boolean {
+ return status.supported && !status.override && (!status.downloaded || !status.mmproj_downloaded);
+}
+
+/** Bytes the Download button fetches: whichever of the model and the vision projector is still missing. */
+export function upsamplerDownloadBytes(status: PromptUpsamplerStatus): number {
+ return (
+ (status.downloaded ? 0 : status.size_bytes) +
+ (status.mmproj_downloaded ? 0 : status.mmproj_size_bytes)
+ );
+}
+
+/** 0..1 download fraction, or null when the total is not known yet. */
+export function upsamplerDownloadFraction(download: PromptUpsamplerDownload): number | null {
+ const total = download.bytes_total ?? 0;
+ if (!(total > 0)) return null;
+ return Math.min(1, Math.max(0, (download.bytes_done ?? 0) / total));
+}
+
+/** What the loaded model advertises. Absent on an older backend: unsupported, default off. */
+export interface PromptUpsamplerCapability {
+ supported: boolean;
+ defaultOn: boolean;
+}
+
+export function promptUpsamplerCapability(
+ status:
+ | {
+ loaded?: boolean;
+ supports_prompt_upsampler?: boolean;
+ prompt_upsampler_default?: boolean;
+ }
+ | null
+ | undefined,
+ external = false,
+): PromptUpsamplerCapability {
+ const supported = !external && status?.loaded === true && status.supports_prompt_upsampler === true;
+ return { supported, defaultOn: supported && status?.prompt_upsampler_default === true };
+}
+
+/** The toggle's value: a one-off restore override, else the stored per-family choice, else the family default. */
+export function resolveEnhancePrompt(args: {
+ capability: PromptUpsamplerCapability;
+ stored: boolean | undefined;
+ override?: boolean | null;
+}): boolean {
+ if (!args.capability.supported) return false;
+ if (typeof args.override === "boolean") return args.override;
+ if (typeof args.stored === "boolean") return args.stored;
+ return args.capability.defaultOn;
+}
+
+/** The request field. Only a supporting family sends it, so every other request stays byte-identical. */
+export function enhancePromptRequestField(
+ capability: PromptUpsamplerCapability,
+ on: boolean,
+): { enhance_prompt?: boolean } {
+ return capability.supported ? { enhance_prompt: on } : {};
+}
+
+/** The preview variant for the loaded family: base checkpoints get the base caption style. */
+export function upsamplerVariantFor(family: string | null | undefined): "distilled" | "base" {
+ return (family ?? "").toLowerCase().includes("base") ? "base" : "distilled";
+}
+
+const KEY = "unsloth:images:enhance-prompt-by-family";
+
+export function readEnhanceChoices(): Record<string, boolean> {
+ try {
+ const value = JSON.parse(localStorage.getItem(KEY) ?? "{}");
+ if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+ const out: Record<string, boolean> = {};
+ for (const [k, v] of Object.entries(value)) if (typeof v === "boolean") out[k] = v;
+ return out;
+ } catch {
+ return {};
+ }
+}
+
+/** Store the choice for one family. A choice equal to the family default is dropped, so a changed default reaches
+ * users who never touched the toggle. */
+export function withEnhanceChoice(
+ choices: Record<string, boolean>,
+ familyKey: string,
+ on: boolean,
+ defaultOn: boolean,
+): Record<string, boolean> {
+ if (!familyKey) return choices;
+ const next = { ...choices };
+ if (on === defaultOn) delete next[familyKey];
+ else next[familyKey] = on;
+ try {
+ localStorage.setItem(KEY, JSON.stringify(next));
+ } catch {
+ // storage can be unavailable in private browsing
+ }
+ return next;
+}
+
+/** What a recipe restore puts back. `exact` reuses the caption that actually ran, with enhancement off. */
+export function enhanceRestoreFor(
+ image: { prompt: string; enhanced_prompt?: string | null; prompt_enhancement?: string | null },
+ exact = false,
+): { prompt: string; enhance: boolean | null } {
+ const applied = image.prompt_enhancement === "applied";
+ const caption = (image.enhanced_prompt ?? "").trim();
+ if (exact && applied && caption) return { prompt: caption, enhance: false };
+ if (applied) return { prompt: image.prompt, enhance: true };
+ // "off" ran the prompt as typed; unavailable / failed did too, but the user had asked for it, so keep asking.
+ if (image.prompt_enhancement === "off") return { prompt: image.prompt, enhance: false };
+ if (image.prompt_enhancement === "unavailable" || image.prompt_enhancement === "failed") {
+ return { prompt: image.prompt, enhance: true };
+ }
+ // Records from before the upsampler, or another family: leave the toggle alone.
+ return { prompt: image.prompt, enhance: null };
+}
+
+/** Whether a record carries an enhanced caption worth showing next to the user's prompt. */
+export function hasEnhancedPrompt(image: {
+ prompt: string;
+ enhanced_prompt?: string | null;
+ prompt_enhancement?: string | null;
+}): boolean {
+ const caption = (image.enhanced_prompt ?? "").trim();
+ return Boolean(caption) && caption !== image.prompt.trim();
+}
+
+/** Two 32-bit hashes (FNV-1a, djb2) over the whole data URL: the same-format headers and trailers of two different
+ * images often match, so the ends alone do not tell them apart. */
+export function referenceDigest(src: string): string {
+ let fnv = 0x811c9dc5;
+ let djb = 5381;
+ for (let i = 0; i < src.length; i++) {
+ const code = src.charCodeAt(i);
+ fnv = Math.imul(fnv ^ code, 0x01000193);
+ djb = (Math.imul(djb, 33) + code) | 0;
+ }
+ return `${src.length}:${(fnv >>> 0).toString(16)}:${(djb >>> 0).toString(16)}`;
+}
+
+// The stale check runs on every render: digests of the images the last call named, and only those, so a removed
+// image is not kept alive by the cache.
+let referenceDigests = new Map<string, string>();
+
+/** Drop the cached digests (and with them the uploads they are keyed on) once no preview is shown. */
+export function clearPreviewDigests(): void {
+ referenceDigests = new Map();
+}
+
+/** What a preview caption was written for: the prompt, the variant and the reference images. */
+export function previewSignature(body: PromptUpsamplerPreviewRequest): string {
+ const next = new Map<string, string>();
+ const refs = (body.reference_images ?? []).map((src) => {
+ const digest = next.get(src) ?? referenceDigests.get(src) ?? referenceDigest(src);
+ next.set(src, digest);
+ return digest;
+ });
+ referenceDigests = next;
+ return JSON.stringify([body.prompt.trim(), body.variant ?? null, refs]);
+}
+
+/** Non-blocking notice after a run that asked for enhancement and did not get it, else null. */
+export function enhancementNotice(
+ outcome: string | null | undefined,
+): { title: string; description: string } | null {
+ if (outcome === "unavailable") {
+ return {
+ title: "Prompt used as typed",
+ description:
+ "The prompt upsampler could not run (not downloaded yet, or llama.cpp is unavailable), so the prompt ran unchanged.",
+ };
+ }
+ if (outcome === "failed") {
+ return {
+ title: "Prompt used as typed",
+ description: "The prompt upsampler could not rewrite this prompt, so it ran unchanged.",
+ };
+ }
+ return null;
+}
+
+/** The family key of a loaded model or connection with no upsampler. */
+export const ENHANCE_UNSUPPORTED_FAMILY = "\u0000unsupported";
+
+/** The one-off Enhance override after the loaded family becomes ``familyKey`` (non-empty): one set before any model
+ * loaded ("") binds to the first FLUX.3 family and survives an unsupported model, one set under another family is
+ * dropped, so it cannot come back on a later switch. */
+export function overrideAfterFamilyChange(
+ prev: { family: string; on: boolean } | null,
+ familyKey: string,
+): { family: string; on: boolean } | null {
+ if (!prev || !familyKey || prev.family === familyKey) return prev;
+ if (prev.family === "") return familyKey === ENHANCE_UNSUPPORTED_FAMILY ? prev : { ...prev, family: familyKey };
+ return null;
+}
+
diff --git a/studio/frontend/src/features/images/prompt-enhance-panel.tsx b/studio/frontend/src/features/images/prompt-enhance-panel.tsx
new file mode 100644
index 00000000000..b41b3256487
--- /dev/null
+++ b/studio/frontend/src/features/images/prompt-enhance-panel.tsx
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import { Cancel01Icon, Copy01Icon, Download04Icon, SparklesIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { ChevronDown } from "lucide-react";
+import { useCallback, useEffect, useRef, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { InfoHint } from "@/components/ui/info-hint";
+import { Progress } from "@/components/ui/progress";
+import { Spinner } from "@/components/ui/spinner";
+import { Switch } from "@/components/ui/switch";
+import { useIsAccountOwner } from "@/features/auth/account-session";
+import { formatBytes, getHfToken, hfApiToken } from "@/features/hub";
+import { copyToClipboard } from "@/lib/copy-to-clipboard";
+import { toast } from "@/lib/toast";
+import { cn } from "@/lib/utils";
+
+import {
+ cancelPromptUpsamplerDownload,
+ getPromptUpsamplerStatus,
+ previewEnhancedPrompt,
+ startPromptUpsamplerDownload,
+} from "./api";
+import {
+ ENHANCE_PROMPT_HINT,
+ ENHANCE_PROMPT_LABEL,
+ type PromptUpsamplerPreviewRequest,
+ type PromptUpsamplerStatus,
+ upsamplerDownloadBytes,
+ upsamplerNeedsDownload,
+ clearPreviewDigests,
+ previewSignature,
+ upsamplerDownloadFraction,
+} from "./lib/prompt-enhance";
+
+/** Upsampler status, polled every second while a download runs and once otherwise. */
+function usePromptUpsamplerStatus(active: boolean) {
+ const [status, setStatus] = useState<PromptUpsamplerStatus | null>(null);
+ const [tick, setTick] = useState(0);
+ const refresh = useCallback(() => setTick((t) => t + 1), []);
+ const downloading = status?.download.downloading === true;
+ useEffect(() => {
+ if (!active) return;
+ const controller = new AbortController();
+ getPromptUpsamplerStatus(controller.signal)
+ .then(setStatus)
+ .catch(() => {});
+ return () => controller.abort();
+ }, [active, tick]);
+ useEffect(() => {
+ if (!active || !downloading) return;
+ const id = window.setInterval(refresh, 1000);
+ return () => window.clearInterval(id);
+ }, [active, downloading, refresh]);
+ return { status, refresh };
+}
+
+function CopyButton({ text, label }: { text: string; label: string }) {
+ return (
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className="h-7 gap-1 px-2 text-xs"
+ aria-label={label}
+ onClick={() => {
+ void copyToClipboard(text).then((ok) =>
+ ok ? toast.success("Copied") : toast.error("Could not copy"),
+ );
+ }}
+ >
+ <HugeiconsIcon icon={Copy01Icon} className="size-3.5" />
+ Copy
+ </Button>
+ );
+}
+
+function UpsamplerDownloadNotice({
+ status,
+ onChanged,
+}: {
+ status: PromptUpsamplerStatus;
+ onChanged: () => void;
+}) {
+ const [busy, setBusy] = useState(false);
+ // The download and cancel routes are owner-only; a managed account is told who can fetch it instead.
+ const owner = useIsAccountOwner();
+ const { download } = status;
+ const fraction = upsamplerDownloadFraction(download);
+ const downloadBytes = upsamplerDownloadBytes(status);
+ const size = downloadBytes > 0 ? formatBytes(downloadBytes) : null;
+ const start = async () => {
+ setBusy(true);
+ try {
+ await startPromptUpsamplerDownload(hfApiToken(getHfToken()));
+ } catch (err) {
+ toast.error("Could not download the prompt upsampler", {
+ description: err instanceof Error ? err.message : undefined,
+ });
+ } finally {
+ setBusy(false);
+ onChanged();
+ }
+ };
+ const cancel = async () => {
+ try {
+ await cancelPromptUpsamplerDownload();
+ } catch (err) {
+ toast.error(err instanceof Error ? err.message : "Could not cancel the download");
+ } finally {
+ onChanged();
+ }
+ };
+ return (
+ <div
+ data-testid="prompt-upsampler-download"
+ className="flex flex-col gap-1.5 rounded-md border border-border/60 bg-muted/30 px-2.5 py-2 text-ui-11"
+ >
+ {download.downloading ? (
+ <>
+ <div className="flex items-center justify-between gap-2">
+ <span className="text-muted-foreground">
+ {download.bytes_total
+ ? `Downloading prompt upsampler: ${formatBytes(download.bytes_done ?? 0)} of ${formatBytes(download.bytes_total)}`
+ : "Downloading prompt upsampler..."}
+ </span>
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className={cn("h-6 gap-1 px-1.5 text-xs", !owner && "hidden")}
+ onClick={() => void cancel()}
+ >
+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />
+ Cancel
+ </Button>
+ </div>
+ <Progress
+ className="h-1.5"
+ value={fraction === null ? undefined : fraction * 100}
+ indeterminate={fraction === null}
+ />
+ </>
+ ) : (
+ <>
+ <span className="text-muted-foreground">
+ {status.downloaded
+ ? `The prompt upsampler's vision projector${size ? ` (${size})` : ""} is not downloaded. Until it is, edits are enhanced from the text alone.`
+ : `The prompt upsampler${size ? ` (${size})` : ""} is not downloaded. Until it is, prompts run as typed.`}
+ </span>
+ {download.error && !download.cancelled ? (
+ <span className="text-destructive">{download.error}</span>
+ ) : null}
+ {owner ? (
+ <Button
+ type="button"
+ size="sm"
+ variant="secondary"
+ className="h-7 gap-1.5 self-start text-xs"
+ disabled={busy}
+ onClick={() => void start()}
+ >
+ {busy ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Download04Icon} className="size-3.5" />}
+ {size ? `Download (${size})` : "Download"}
+ </Button>
+ ) : (
+ <span className="text-muted-foreground">Ask the owner of this Unsloth install to download it.</span>
+ )}
+ </>
+ )}
+ </div>
+ );
+}
+
+/** The Enhance prompt toggle, the download notice, and the preview panel. Rendered only for a supporting family. */
+export function EnhancePromptControls({
+ enabled,
+ onEnabledChange,
+ prompt,
+ previewRequest,
+ onUseCaption,
+}: {
+ enabled: boolean;
+ onEnabledChange: (on: boolean) => void;
+ prompt: string;
+ /** The preview body for the current form (prompt, reference images, variant). */
+ previewRequest: () => PromptUpsamplerPreviewRequest;
+ /** Put the caption in the prompt box and turn enhancement off. */
+ onUseCaption: (caption: string) => void;
+}) {
+ const { status, refresh } = usePromptUpsamplerStatus(true);
+ // Re-read once a download ends, so the notice clears without a reload.
+ const wasDownloading = useRef(false);
+ useEffect(() => {
+ const now = status?.download.downloading === true;
+ if (wasDownloading.current && !now) refresh();
+ wasDownloading.current = now;
+ }, [status?.download.downloading, refresh]);
+
+ const [preview, setPreview] = useState<{ signature: string; caption: string } | null>(null);
+ // The stale check stops running without a preview, so its cache would otherwise keep the last uploads alive.
+ useEffect(() => {
+ if (preview === null) clearPreviewDigests();
+ }, [preview]);
+ useEffect(() => clearPreviewDigests, []);
+ const [previewOpen, setPreviewOpen] = useState(true);
+ const [previewing, setPreviewing] = useState(false);
+ const previewSeq = useRef(0);
+ const runPreview = async () => {
+ const body = previewRequest();
+ if (!body.prompt.trim()) {
+ toast.error("Prompt is empty");
+ return;
+ }
+ const seq = ++previewSeq.current;
+ setPreviewing(true);
+ try {
+ const res = await previewEnhancedPrompt(body);
+ if (seq !== previewSeq.current) return;
+ setPreview({ signature: previewSignature(body), caption: res.enhanced_prompt });
+ setPreviewOpen(true);
+ } catch (err) {
+ if (seq !== previewSeq.current) return;
+ toast.error("Could not enhance the prompt", {
+ description: err instanceof Error ? err.message : undefined,
+ });
+ } finally {
+ if (seq === previewSeq.current) setPreviewing(false);
+ }
+ };
+
+ const needsDownload = enabled && status !== null && upsamplerNeedsDownload(status);
+ const canPreview = status === null || status.downloaded;
+ // A caption for an older prompt, other images or the other variant stays readable but is marked stale.
+ const stale = preview !== null && preview.signature !== previewSignature(previewRequest());
+
+ return (
+ <div className="flex flex-col gap-1.5" data-testid="enhance-prompt-controls">
+ <div className="flex items-center justify-between gap-2">
+ <span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">
+ {ENHANCE_PROMPT_LABEL}
+ <InfoHint>{ENHANCE_PROMPT_HINT}</InfoHint>
+ </span>
+ <div className="flex items-center gap-1.5">
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className="h-7 gap-1 px-2 text-xs"
+ disabled={previewing || !canPreview || !prompt.trim()}
+ title={canPreview ? "Rewrite the prompt now and show the caption" : "Download the prompt upsampler first"}
+ onClick={() => void runPreview()}
+ >
+ {previewing ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={SparklesIcon} className="size-3.5" />}
+ {previewing ? "Enhancing..." : "Preview"}
+ </Button>
+ <Switch checked={enabled} onCheckedChange={onEnabledChange} aria-label={ENHANCE_PROMPT_LABEL} />
+ </div>
+ </div>
+ {needsDownload && status ? <UpsamplerDownloadNotice status={status} onChanged={refresh} /> : null}
+ {preview ? (
+ <div
+ data-testid="enhanced-prompt-preview"
+ className="flex flex-col gap-1 rounded-md border border-border/60 px-2.5 py-2 text-xs"
+ >
+ <div className="flex items-center justify-between gap-2">
+ <button
+ type="button"
+ aria-expanded={previewOpen}
+ onClick={() => setPreviewOpen((o) => !o)}
+ className="flex items-center gap-1 font-medium text-muted-foreground transition-colors hover:text-foreground"
+ >
+ Enhanced prompt{stale ? " (for an earlier request)" : ""}
+ <ChevronDown className={cn("size-3 transition-transform", previewOpen && "rotate-180")} />
+ </button>
+ <button
+ type="button"
+ aria-label="Dismiss enhanced prompt"
+ className="text-muted-foreground hover:text-foreground"
+ onClick={() => setPreview(null)}
+ >
+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />
+ </button>
+ </div>
+ {previewOpen ? (
+ <>
+ <p className="max-h-48 overflow-y-auto whitespace-pre-wrap break-words text-foreground">
+ {preview.caption}
+ </p>
+ <div className="flex items-center justify-end gap-1">
+ <CopyButton text={preview.caption} label="Copy enhanced prompt" />
+ <Button
+ type="button"
+ size="sm"
+ variant="secondary"
+ className="h-7 px-2 text-xs"
+ onClick={() => {
+ onUseCaption(preview.caption);
+ setPreview(null);
+ }}
+ >
+ Use as prompt
+ </Button>
+ </div>
+ </>
+ ) : null}
+ </div>
+ ) : null}
+ </div>
+ );
+}
+
+/** The recipe's "Enhanced prompt" row: collapsible, with copy. */
+export function EnhancedPromptRecipeRow({ caption }: { caption: string }) {
+ const [open, setOpen] = useState(false);
+ return (
+ <div className="grid grid-cols-[72px_1fr] items-start gap-2">
+ <button
+ type="button"
+ aria-expanded={open}
+ onClick={() => setOpen((o) => !o)}
+ className="flex items-start gap-0.5 text-left text-muted-foreground hover:text-foreground"
+ >
+ Enhanced prompt
+ <ChevronDown className={cn("mt-0.5 size-3 shrink-0 transition-transform", open && "rotate-180")} />
+ </button>
+ <div className="flex min-w-0 flex-col gap-1">
+ <span
+ className={cn(
+ "min-w-0 text-foreground",
+ open ? "whitespace-pre-wrap break-words" : "truncate",
+ )}
+ >
+ {caption}
+ </span>
+ {open ? (
+ <div className="flex justify-end">
+ <CopyButton text={caption} label="Copy enhanced prompt" />
+ </div>
+ ) : null}
+ </div>
+ </div>
+ );
+}
diff --git a/studio/frontend/src/lib/media-generate-phase.ts b/studio/frontend/src/lib/media-generate-phase.ts
index abf691f76de..68d3f48f652 100644
--- a/studio/frontend/src/lib/media-generate-phase.ts
+++ b/studio/frontend/src/lib/media-generate-phase.ts
@@ -2,7 +2,8 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
export interface MediaGeneratePhaseProgress {
- // encode | denoise | decode | export (video); absent or null (sd.cpp) reads as denoise.
+ // enhance (FLUX.3 prompt upsampler) | encode | denoise | decode | export (video); absent or null (sd.cpp) reads as
+ // denoise.
phase?: string | null;
step: number;
total: number;
@@ -14,6 +15,7 @@ export function generatePhaseLabel(
p: MediaGeneratePhaseProgress,
opts: { hasAudio?: boolean; formatEta?: (seconds: number) => string } = {},
): string {
+ if (p.phase === "enhance") return "Enhancing prompt…";
if (p.phase === "encode") return "Encoding prompt…";
if (p.phase === "decode") return opts.hasAudio ? "Decoding video and audio…" : "Decoding…";
if (p.phase === "export") return "Encoding video…";
diff --git a/studio/frontend/tests/image-prompt-enhance.test.ts b/studio/frontend/tests/image-prompt-enhance.test.ts
new file mode 100644
index 00000000000..a36abeb8f69
--- /dev/null
+++ b/studio/frontend/tests/image-prompt-enhance.test.ts
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { installLocalStorageFake, readSrc } from "./helpers/kit.ts";
+import {
+ ENHANCE_PROMPT_HINT,
+ enhancePromptRequestField,
+ enhanceRestoreFor,
+ enhancementNotice,
+ ENHANCE_UNSUPPORTED_FAMILY,
+ overrideAfterFamilyChange,
+ previewSignature,
+ hasEnhancedPrompt,
+ parsePromptUpsamplerStatus,
+ PROMPT_UPSAMPLER_ROUTES,
+ promptUpsamplerCapability,
+ readEnhanceChoices,
+ resolveEnhancePrompt,
+ upsamplerDownloadBytes,
+ upsamplerDownloadFraction,
+ upsamplerNeedsDownload,
+ upsamplerVariantFor,
+ withEnhanceChoice,
+} from "../src/features/images/lib/prompt-enhance.ts";
+import { importedRecipe, recipeSettings } from "../src/features/images/lib/image-metadata.ts";
+
+const FLUX3 = { loaded: true, family: "flux.3-klein", supports_prompt_upsampler: true, prompt_upsampler_default: true };
+const SDXL = { loaded: true, family: "sdxl", supports_prompt_upsampler: false, prompt_upsampler_default: false };
+
+test("the toggle defaults to the family default and is off for families without an upsampler", () => {
+ const flux = promptUpsamplerCapability(FLUX3);
+ assert.deepEqual(flux, { supported: true, defaultOn: true });
+ assert.equal(resolveEnhancePrompt({ capability: flux, stored: undefined }), true);
+
+ // A supporting family whose default is off.
+ const offByDefault = promptUpsamplerCapability({ ...FLUX3, prompt_upsampler_default: false });
+ assert.equal(resolveEnhancePrompt({ capability: offByDefault, stored: undefined }), false);
+
+ for (const status of [SDXL, null, undefined, { loaded: true }, { ...FLUX3, loaded: false }]) {
+ const cap = promptUpsamplerCapability(status);
+ assert.equal(cap.supported, false);
+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: true, override: true }), false);
+ }
+ // An external image connection never runs the local upsampler.
+ assert.equal(promptUpsamplerCapability(FLUX3, true).supported, false);
+});
+
+test("a stored choice beats the default, and a restore override beats both", () => {
+ const cap = promptUpsamplerCapability(FLUX3);
+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: false }), false);
+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: false, override: true }), true);
+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: true, override: false }), false);
+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: undefined, override: null }), true);
+});
+
+test("the choice persists per family and a choice equal to the default is not stored", () => {
+ const { store } = installLocalStorageFake();
+ assert.deepEqual(readEnhanceChoices(), {});
+ let choices = withEnhanceChoice({}, "flux.3-klein", false, true);
+ assert.deepEqual(choices, { "flux.3-klein": false });
+ choices = withEnhanceChoice(choices, "flux.3-klein-base", false, true);
+ assert.deepEqual(readEnhanceChoices(), { "flux.3-klein": false, "flux.3-klein-base": false });
+ choices = withEnhanceChoice(choices, "flux.3-klein", true, true);
+ assert.deepEqual(readEnhanceChoices(), { "flux.3-klein-base": false });
+ // No family key: nothing is written.
+ assert.equal(withEnhanceChoice(choices, "", true, false), choices);
+ // Corrupt storage reads as empty, non-boolean entries are dropped.
+ store.set("unsloth:images:enhance-prompt-by-family", "not json");
+ assert.deepEqual(readEnhanceChoices(), {});
+ store.set("unsloth:images:enhance-prompt-by-family", JSON.stringify({ a: true, b: "yes", c: 1 }));
+ assert.deepEqual(readEnhanceChoices(), { a: true });
+});
+
+test("enhance_prompt is sent only when the loaded family supports it", () => {
+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(FLUX3), true), { enhance_prompt: true });
+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(FLUX3), false), { enhance_prompt: false });
+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(SDXL), false), {});
+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(null), true), {});
+ // Wired into the generate request.
+ const page = readSrc("features/images/images-page.tsx");
+ const call = page.slice(page.indexOf("res = await generateDiffusionImage({"));
+ assert.match(call.slice(0, 4000), /\.\.\.enhancePromptRequestField\(upsamplerCapability, enhanceOn\)/);
+});
+
+test("recipe restore: an applied run restores the user's prompt with the toggle on, or the caption with it off", () => {
+ const applied = {
+ prompt: "a cat",
+ enhanced_prompt: "A photograph of a ginger cat sitting on a windowsill.",
+ prompt_enhancement: "applied",
+ };
+ assert.deepEqual(enhanceRestoreFor(applied), { prompt: "a cat", enhance: true });
+ assert.deepEqual(enhanceRestoreFor(applied, true), {
+ prompt: "A photograph of a ginger cat sitting on a windowsill.",
+ enhance: false,
+ });
+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: "off" }), { prompt: "a cat", enhance: false });
+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: "unavailable" }, true), {
+ prompt: "a cat",
+ enhance: true,
+ });
+ // An older record, or another family's, leaves the toggle alone.
+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat" }), { prompt: "a cat", enhance: null });
+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: null }, true), {
+ prompt: "a cat",
+ enhance: null,
+ });
+ // Exact with no caption falls back to the user's prompt.
+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", enhanced_prompt: " ", prompt_enhancement: "applied" }, true), {
+ prompt: "a cat",
+ enhance: true,
+ });
+ assert.equal(hasEnhancedPrompt(applied), true);
+ assert.equal(hasEnhancedPrompt({ prompt: "a cat", enhanced_prompt: "a cat" }), false);
+ assert.equal(hasEnhancedPrompt({ prompt: "a cat" }), false);
+});
+
+test("an embedded recipe carries the enhancement fields through to restore", () => {
+ const recipe = importedRecipe({
+ recipe: JSON.stringify({
+ prompt: "a cat",
+ enhanced_prompt: "A ginger cat.",
+ prompt_enhancement: "applied",
+ width: 1024,
+ height: 1024,
+ steps: 16,
+ guidance: 1,
+ seed: 3,
+ }),
+ });
+ assert.ok(recipe);
+ const settings = recipeSettings(recipe, { width: 1024, height: 1024, steps: 8, guidance: 0 });
+ assert.equal(settings.enhanced_prompt, "A ginger cat.");
+ assert.equal(settings.prompt_enhancement, "applied");
+ const bogus = importedRecipe({ recipe: JSON.stringify({ prompt: "x", prompt_enhancement: "maybe" }) });
+ assert.ok(bogus);
+ assert.equal(recipeSettings(bogus, { width: 1, height: 1, steps: 1, guidance: 0 }).prompt_enhancement, null);
+});
+
+test("status parsing tolerates missing and malformed fields", () => {
+ const full = parsePromptUpsamplerStatus({
+ supported: true,
+ downloaded: false,
+ loaded: false,
+ repo: "unsloth/FLUX.3-prompt-upsampler-GGUF",
+ filename: "upsampler-Q4_K_M.gguf",
+ size_bytes: 5_600_000_000,
+ mmproj_size_bytes: 900_000_000,
+ mmproj_downloaded: false,
+ download: { downloading: true, model: "x", error: null, bytes_total: 200, bytes_done: 50, download_id: "d1" },
+ });
+ assert.equal(full.supported, true);
+ assert.equal(full.size_bytes, 5_600_000_000);
+ assert.equal(full.download.downloading, true);
+ assert.equal(full.download.download_id, "d1");
+ assert.equal(upsamplerDownloadFraction(full.download), 0.25);
+ // The button fetches the vision projector too, until it is on disk.
+ assert.equal(upsamplerDownloadBytes(full), 6_500_000_000);
+ assert.equal(upsamplerDownloadBytes({ ...full, mmproj_downloaded: true }), 5_600_000_000);
+ // Model cached, projector missing: Download stays offered and fetches only the projector.
+ const noProjector = { ...full, downloaded: true };
+ assert.equal(upsamplerNeedsDownload(noProjector), true);
+ assert.equal(upsamplerDownloadBytes(noProjector), 900_000_000);
+ assert.equal(upsamplerNeedsDownload({ ...noProjector, mmproj_downloaded: true }), false);
+ assert.equal(upsamplerNeedsDownload({ ...full, supported: false }), false);
+
+ const empty = parsePromptUpsamplerStatus(null);
+ assert.deepEqual(
+ { supported: empty.supported, downloaded: empty.downloaded, size: empty.size_bytes, repo: empty.repo },
+ { supported: false, downloaded: false, size: 0, repo: "" },
+ );
+ assert.equal(empty.download.downloading, false);
+ assert.equal(upsamplerDownloadFraction(empty.download), null);
+
+ const junk = parsePromptUpsamplerStatus({ supported: "yes", size_bytes: -4, download: "nope" });
+ assert.equal(junk.supported, false);
+ assert.equal(junk.size_bytes, 0);
+ assert.equal(junk.download.bytes_total, null);
+ assert.equal(
+ upsamplerDownloadFraction({ ...full.download, bytes_total: 100, bytes_done: 500 }),
+ 1,
+ );
+});
+
+test("routes live in one place; variant and notices", () => {
+ for (const route of Object.values(PROMPT_UPSAMPLER_ROUTES)) {
+ assert.match(route, /^\/api\/inference\/images\/prompt-upsampler\//);
+ }
+ const api = readSrc("features/images/api.ts");
+ assert.doesNotMatch(api, /"\/api\/inference\/images\/prompt-upsampler/);
+ assert.equal(upsamplerVariantFor("flux.3-klein-base"), "base");
+ assert.equal(upsamplerVariantFor("flux.3-klein"), "distilled");
+ assert.equal(upsamplerVariantFor(null), "distilled");
+ assert.ok(enhancementNotice("unavailable"));
+ assert.ok(enhancementNotice("failed"));
+ assert.equal(enhancementNotice("applied"), null);
+ assert.equal(enhancementNotice("off"), null);
+ assert.equal(enhancementNotice(undefined), null);
+ // No em dashes in the copy.
+ assert.doesNotMatch(ENHANCE_PROMPT_HINT, /—/);
+ assert.doesNotMatch(readSrc("features/images/prompt-enhance-panel.tsx"), /—/);
+ assert.doesNotMatch(readSrc("features/images/lib/prompt-enhance.ts"), /—/);
+});
+
+test("a preview caption goes stale when the prompt, the variant or a reference image changes", () => {
+ const base = { prompt: "make it night", variant: "distilled" as const, reference_images: ["data:image/png;base64,AAAA"] };
+ const sig = previewSignature(base);
+ assert.equal(previewSignature({ ...base, prompt: " make it night " }), sig);
+ assert.notEqual(previewSignature({ ...base, prompt: "make it day" }), sig);
+ assert.notEqual(previewSignature({ ...base, variant: "base" }), sig);
+ assert.notEqual(previewSignature({ ...base, reference_images: ["data:image/png;base64,BBBB"] }), sig);
+ assert.notEqual(previewSignature({ prompt: base.prompt, variant: base.variant }), sig);
+ // Same length and the same 48 characters at each end: only the body differs.
+ const head = "data:image/png;base64," + "A".repeat(60);
+ const tail = "Z".repeat(60);
+ assert.notEqual(
+ previewSignature({ ...base, reference_images: [head + "QQQQ" + tail] }),
+ previewSignature({ ...base, reference_images: [head + "RRRR" + tail] }),
+ );
+});
+
+test("an unavailable outcome does not claim the weights are missing, and overrides offer no download", () => {
+ assert.match(enhancementNotice("unavailable")!.description, /llama\.cpp is unavailable/);
+ const status = parsePromptUpsamplerStatus({ supported: true, downloaded: true, mmproj_downloaded: false, override: true });
+ assert.equal(status.override, true);
+ assert.equal(upsamplerNeedsDownload(status), false);
+});
+
+test("a family change consumes the one-off override instead of leaving it dormant", () => {
+ const unloaded = { family: "", on: false };
+ // Set before any model loaded: binds to the first FLUX.3 family that loads.
+ const bound = overrideAfterFamilyChange(unloaded, "flux.3-klein");
+ assert.deepEqual(bound, { family: "flux.3-klein", on: false });
+ assert.equal(overrideAfterFamilyChange(bound, "flux.3-klein"), bound);
+ // A switch to the other variant drops it, so switching back does not revive it.
+ assert.equal(overrideAfterFamilyChange(bound, "flux.3-klein-base"), null);
+ assert.equal(overrideAfterFamilyChange(null, "flux.3-klein"), null);
+ // A model without an upsampler consumes a bound override but keeps one set before any model loaded.
+ assert.equal(overrideAfterFamilyChange(bound, ENHANCE_UNSUPPORTED_FAMILY), null);
+ assert.equal(overrideAfterFamilyChange(unloaded, ENHANCE_UNSUPPORTED_FAMILY), unloaded);
+});
+
diff --git a/studio/frontend/tests/image-restore-negative-prompt.test.ts b/studio/frontend/tests/image-restore-negative-prompt.test.ts
index 616fdb95eb5..baed9c65431 100644
--- a/studio/frontend/tests/image-restore-negative-prompt.test.ts
+++ b/studio/frontend/tests/image-restore-negative-prompt.test.ts
@@ -5,6 +5,7 @@ import assert from "node:assert/strict";
import test from "node:test";
import ts from "@typescript/typescript6";
import { readSrc } from "./helpers/kit.ts";
+import { enhanceRestoreFor } from "../src/features/images/lib/prompt-enhance.ts";
function restoreSettingsWith(negativeCapable: boolean) {
const source = readSrc("features/images/images-page.tsx");
@@ -46,6 +47,8 @@ function restoreSettingsWith(negativeCapable: boolean) {
AUTOMATIC_SCHEDULE: "automatic",
samplerOptions: [],
changeSamplerChoice: () => {},
+ // Its return value is read, so it cannot be a recording no-op.
+ enhanceRestoreFor,
toast: { success: () => {} },
};
// Every closure name the callback reads that is not a real global resolves to a recording no-op,
@@ -82,3 +85,30 @@ test("restoring an image keeps its negative prompt while a model that ignores on
});
assert.equal(state.setNegativePrompt, "blurry, text");
});
+
+test("restoring an enhanced FLUX.3 image sets the Enhance prompt toggle from its outcome", () => {
+ const image = {
+ prompt: "a cat",
+ enhanced_prompt: "A ginger cat on a windowsill.",
+ prompt_enhancement: "applied",
+ guidance: 1,
+ steps: 16,
+ seed: 7,
+ width: 1024,
+ height: 1024,
+ workflow: "create",
+ };
+ const asTyped = restoreSettingsWith(true);
+ asTyped.restore(image);
+ assert.deepEqual(asTyped.state.setPromptFor, ["create", "a cat"]);
+ assert.equal(asTyped.state.setEnhanceOverride, true);
+
+ const exact = restoreSettingsWith(true);
+ exact.restore(image, true);
+ assert.deepEqual(exact.state.setPromptFor, ["create", "A ginger cat on a windowsill."]);
+ assert.equal(exact.state.setEnhanceOverride, false);
+
+ const legacy = restoreSettingsWith(true);
+ legacy.restore({ ...image, enhanced_prompt: undefined, prompt_enhancement: undefined });
+ assert.equal("setEnhanceOverride" in legacy.state, false);
+});
diff --git a/studio/frontend/tests/media-generate-phase.test.ts b/studio/frontend/tests/media-generate-phase.test.ts
index 7606ca3f6c9..44579db104a 100644
--- a/studio/frontend/tests/media-generate-phase.test.ts
+++ b/studio/frontend/tests/media-generate-phase.test.ts
@@ -38,3 +38,7 @@ test("a new preview re-renders even when nothing else moved", () => {
assert.equal(sameGenerateProgress(a, { ...a, preview_seq: 2 }), false);
assert.equal(sameGenerateProgress({ step: 1, phase: "encode" }, { step: 1, phase: "denoise" }), false);
});
+
+test("the FLUX.3 enhance phase is labelled as such, not as denoising", () => {
+ assert.equal(generatePhaseLabel({ phase: "enhance", step: 0, total: 16 }), "Enhancing prompt…");
+});