Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: FLUX.3 prompt upsampling (Enhance prompt) through a llama.cpp sidecar (#14137)
Daniel Handef3451e4e17
diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.pyindex 7a75d4d4684..06b64905740 100644--- a/studio/backend/core/inference/diffusion_engine_router.py+++ b/studio/backend/core/inference/diffusion_engine_router.py@@ -933,6 +933,10 @@ def annotate_status(status: dict[str, Any]) -> dict[str, Any]:out["supported_families"] = list(_supported_family_capabilities())# The family override picker classes an opaque folder as a modular pipeline only for these (FLUX.3).out["modular_families"] = list(modular_image_family_names(_supported_family_capabilities()))+ # FLUX.3 prompt upsampling, by the loaded family (pure: no binary or cache probe on a status poll).+ from .flux3_prompt_upsampler import family_capabilities++ out.update(family_capabilities(out.get("family") if out.get("loaded") else None))return outdiff --git a/studio/backend/core/inference/flux3_prompt_upsampler.py b/studio/backend/core/inference/flux3_prompt_upsampler.pynew file mode 100644index 00000000000..08d875bc2c6--- /dev/null+++ b/studio/backend/core/inference/flux3_prompt_upsampler.py@@ -0,0 +1,1351 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 Image Klein prompt upsampling through a llama.cpp sidecar.++FLUX.3 was trained on ~250-word captions, and a short request ("a gymnast doing a backflip") renders broken+anatomy. BFL ships a prompt upsampler (a Qwen3.5-9B fine-tune) that rewrites the request into such a caption;+ComfyUI's FLUX.3 templates run it by default. Studio runs a GGUF of it in one ``llama-server`` on an ephemeral+127.0.0.1 port, loaded on demand and unloaded after the dictation keep-alive (or right after the call when VRAM is+tight, so the DiT and text encoder never denoise next to it). The diffusers graph's own ``prompt_upsampler`` step+stays dropped (``diffusion_flux3.flux3_blocks``); this replaces it.++Prompts and sampling follow ComfyUI's FLUX.3 templates. The system prompts are BFL's ``system_prompts.json``+(black-forest-labs/flux-3-image-klein-pup at commit 8b069751), byte-identical to ``SYSTEM_PROMPTS`` in diffusers'+``modular_pipelines/flux3/prompt_upsampler.py`` at d42ebf9c6e (huggingface/diffusers#15058). The reference+preparation is adapted from that file's ``get_reference_size`` / ``prepare_reference`` (Apache-2.0).++Weights only ever come from the unsloth GGUF repo below or a local override; nothing is fetched implicitly by a+generate call, the UI drives the download. Any failure leaves the prompt as typed.+"""++from __future__ import annotations++import collections+import hashlib+import http.client+import io+import json+import math+import os+import re+import secrets+import subprocess+import threading+import time+from contextlib import contextmanager, suppress+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any, Iterator, Optional++from loggers import get_logger++logger = get_logger(__name__)++# ── weights: one place for repo, files and sizes ────────────────────────────────────────────────────────────────────+UPSAMPLER_ID = "flux-3-image-klein-pup"+UPSAMPLER_REPO = "unsloth/flux-3-image-klein-pup-GGUF"+UPSAMPLER_MODEL_FILES = {+ "Q4_K_M": "flux-3-image-klein-pup-Q4_K_M.gguf",+ "Q8_0": "flux-3-image-klein-pup-Q8_0.gguf",+}+UPSAMPLER_DEFAULT_QUANT = "Q4_K_M"+UPSAMPLER_MODEL_FILE = UPSAMPLER_MODEL_FILES[UPSAMPLER_DEFAULT_QUANT]+# Vision projector, needed only for edits (reference images).+UPSAMPLER_MMPROJ_FILE = "mmproj-F16.gguf"+# Expected sizes of the files as built, for the UI before the download starts (the download reads the real ones).+UPSAMPLER_FILE_SIZES = {+ "flux-3-image-klein-pup-Q4_K_M.gguf": 5_629_109_248,+ "flux-3-image-klein-pup-Q8_0.gguf": 9_527_501_824,+ "mmproj-F16.gguf": 918_165_984,+}+# Local files instead of the repo, for testing a build before upload.+ENV_MODEL_OVERRIDE = "UNSLOTH_FLUX3_UPSAMPLER_GGUF"+ENV_MMPROJ_OVERRIDE = "UNSLOTH_FLUX3_UPSAMPLER_MMPROJ"+# Append llama-server's output to this file (debugging only).+ENV_SERVER_LOG = "UNSLOTH_FLUX3_UPSAMPLER_SERVER_LOG"++FLUX3_UPSAMPLER_FAMILIES = {"flux.3-klein": "distilled", "flux.3-klein-base": "base"}+VARIANTS = ("distilled", "base")++# ── prompts (ComfyUI FLUX.3 templates; BFL system_prompts.json, verbatim) ───────────────────────────────────────────+SYSTEM_PROMPTS = {+ "t2i": (+ "You are the prompt upsampler of a text-to-image model. Expand the user's short request into ONE detailed, "+ "self-contained caption: the exact text the image model receives. Output only the caption, with no greetings, "+ "explanations, refusals, questions, Markdown, headings, JSON, coordinates, ids or angle-bracket tokens, and no "+ "quotes around it. If the input is not clearly an image request, caption a visualization of its concept. The "+ 'request may begin with a "TARGET CANVAS: W:H" line: compose for that canvas (wide spreads content sideways, '+ "tall stacks it) and never mention the ratio or the canvas.\n"+ "\n"+ "Fidelity and planning: keep every explicit subject, attribute, count, spatial constraint, style cue and piece "+ "of wording; never contradict, substitute, drop or narrow the request, and keep an abstract concept abstract. "+ "Decide everything left open as ONE concrete, consistent plan; never offer alternatives or hedge. Never "+ "re-express a stated constraint through another mechanism; add camera optics (depth of field, bokeh, blur) only "+ "when asked for or implied. Without a stated style, plan an actual photograph and open with the medium "+ 'convention ("Full-color outdoor photograph, ...", "Full-color indoor photograph, ...", "Studio '+ 'photograph, ...", "Close-up photograph, ..."); keep "Photorealistic" for renders, and turn '+ '"photorealistic" or "realistic" requests for photos into the photograph convention. Give a photograph a '+ "few ordinary, non-ideal specifics drawn from the scene's own objects instead of the stock-photo ideal. Do not "+ "invent names, brands, logos or celebrities; keep user-named ones. A long request keeps its content but not "+ "its wording: drop appraisal and sales vocabulary, turn moods into observable physical facts, and add only new "+ "concrete content with positions. Time of day is lighting, not rendered text.\n"+ "\n"+ 'Caption format: one continuous prose block in the generative voice, never "this image shows", "we can '+ 'see" or "is visible". Lead with medium and style, then scene, lighting and camera, then the subjects with '+ "concrete colors, materials and state. Every person in focus gets an observable profile: ethnicity, gender, "+ 'age range, skin tone ("fair skin", "brown skin"), hair and clothing. Make the layout reconstructable from '+ "words: frame-relative positions (frame-left, frame-right, upper, lower, foreground, midground, background), "+ 'sentence-initial anchors ("On the left, ...", "In the background, ..."), relations (holds, faces, '+ "behind) and a consistent order; describe up to about ten instances one by one, beyond that give the exact "+ "count and arrangement. Out-of-focus or distant planes are shapes, colors and silhouettes, never identities or "+ "readable text, and the background takes about a sixth of the caption. Be definite: no hedging adverbs, no "+ "quality or emotion judgments, no closing sentence about mood; state negations as what is present. Rendered "+ "text: transcribe every wording verbatim inside double quotes in its original language and script, keep the "+ "line breaks of multi-line text, and give its typography, color and placement; author concrete, domain-correct "+ "wording when the request needs text it does not supply, and name canonical structures (calendar, keyboard, "+ "periodic table) exactly with their full layout. All prose is English; only quoted text keeps its language. "+ "Typical length is 100 to 350 words, more for complex scenes.\n"+ "\n"+ "Output only the caption."+ ),+ "i2i": (+ "You are the prompt upsampler of an image-editing model. You see the reference image(s), attached in order "+ "(the first image, the second image, ...), and the user's short request. Expand it into ONE detailed, "+ "self-contained instruction: the caption the editing model receives together with the same images. Output only "+ "the instruction as one continuous English prose block, with no greetings, explanations, Markdown, JSON, "+ "coordinates, ids or angle-bracket tokens. Read a vague or conversational request as the most plausible edit "+ "of the supplied images.\n"+ "\n"+ "Pick the voice by task shape. When one reference stays and gets bounded changes, write an imperative editing "+ 'instruction on it ("Remove the ...", "Change the ... to ...", "Zoom in on ..."); a whole-image '+ 'transformation opens with its scope ("Convert the entire image into ...", "Apply a ... filter over the '+ 'entire scene", "Horizontally flip the entire image so that ...", "Shift the camera viewpoint to ...", '+ "then where the major elements should now be). When the references are drawn into a newly composed frame "+ '(combining subjects, a new scene, a garment, object or style donor), write a generation request ("Create a '+ 'photograph of ...", "Create an anime-style illustration of ...", "Generate a new image featuring ...") '+ "that says what to take from each image, by ordinal, and what is newly generated.\n"+ "\n"+ "Keep every explicit edit, constraint, count, position, style cue and piece of wording, and decide what is left "+ "open as one concrete plan. Ground everything in what is actually visible: identify each referent "+ "unambiguously by its real appearance, never mention content that is not there, and use every image the user "+ "attached. State every end state concretely (which color, how large, which pose, which material) and give new "+ "content enough appearance to be generated. Cover the knock-on effects of the change (shadows, reflections, "+ "occlusion, perspective, scale, lighting) and say what fills a removed region. A zoom-in names what stays in "+ "frame and which neighbors are cropped out; an outpainting continues the scene in the same medium. Describe "+ "the change, not the whole scene, and close an editing instruction with one compact clause naming what stays "+ "unchanged; transferred people keep their identity, face and accessories unless asked otherwise. Transcribe "+ "on-image text verbatim in its original language inside double quotes, with line breaks as \\n, and match the "+ "original font, size and color when replacing it. Name recognizable public figures, brands and landmarks you "+ "can see. Directions are frame-relative; use no evaluative or quality words. Typical length is 50 to 150 "+ "words.\n"+ "\n"+ "Output only the instruction."+ ),+}+# The user turn of an edit: the reference images, then this text.+I2I_SCAFFOLD = (+ "Reference images follow in order (the first image, the second image, ...).\n"+ "User request: {instruction}\n"+ "Output the caption."+)++# ── sampling (ComfyUI FLUX.3 templates) ─────────────────────────────────────────────────────────────────────────────+MAX_NEW_TOKENS = 3584+_BASE_SAMPLING = {+ "temperature": 1.0,+ "top_k": 1,+ "top_p": 0.95,+ "min_p": 0.0,+ "seed": 0,+}+# (mode, variant) -> (repeat_penalty, presence_penalty)+PENALTIES = {+ ("t2i", "distilled"): (1.05, 1.5),+ ("t2i", "base"): (1.0, 1.5),+ ("i2i", "distilled"): (1.05, 0.0),+ ("i2i", "base"): (1.0, 0.0),+}+# ComfyUI and BFL's reference penalize every generated token and never the prompt. llama-server feeds the prompt into+# the penalty history unless it has `penalize_prompt` (unslothai/llama.cpp#283); with it the window spans the whole+# caption. Older servers keep their default 64-token window: a wider one there would mostly penalize the system prompt.+GENERATED_ONLY_PENALTIES = {"penalize_prompt": False, "repeat_last_n": 4096}++# ── reference preparation (diffusers prepare_reference) ─────────────────────────────────────────────────────────────+MAX_REFERENCES = 10+REFERENCE_MULTIPLE = 32+MAX_ASPECT_RATIO = 200+MAX_PIXELS_BY_REFS = {+ 1: 1024 * 1024,+ 2: 724 * 724,+ **dict.fromkeys(range(3, MAX_REFERENCES + 1), 512 * 512),+}+JPEG_QUALITY = 92+# Header guard on decoded uploads, diffusion.decode_b64_image's default: a larger reference fails the generate anyway.+_MAX_UPLOAD_SIDE = 4096++# ── server ──────────────────────────────────────────────────────────────────────────────────────────────────────────+# Holds the system prompt (~1k tokens), up to ten references (256-1024 image tokens each) and the 3584-token caption.+# The model's own 262k default would allocate a KV cache many times the weights.+_CONTEXT_TOKENS = 16384+_SERVER_START_TIMEOUT_SECONDS = 180.0+_REQUEST_TIMEOUT_SECONDS = 600.0+_NUM_LAYERS = 32+# KV cache (8 full-attention layers at 16k), recurrent state and compute buffers on top of the weights.+_GPU_OVERHEAD_MIB = 2560+# Fewer layers than this on the GPU is not worth the split.+_MIN_GPU_LAYERS = 4+# After a call, keep the server resident only when at least this much accelerator memory stays free; else unload it+# before the DiT denoises.+_TIGHT_FREE_MIB = 6144+_TIGHT_FREE_FRACTION = 0.20+_CACHE_ENTRIES = 128++STATUS_APPLIED = "applied"+STATUS_OFF = "off"+STATUS_UNAVAILABLE = "unavailable"+STATUS_FAILED = "failed"+++class UpsamplerUnavailableError(RuntimeError):+ """The upsampler cannot run here: no llama-server, or the GGUF is not downloaded."""+++class UpsamplerNotDownloadedError(UpsamplerUnavailableError):+ pass+++class UpsamplerCancelledError(RuntimeError):+ pass+++# ── family defaults ─────────────────────────────────────────────────────────────────────────────────────────────────+def supports_family(family_name: Optional[str]) -> bool:+ return family_name in FLUX3_UPSAMPLER_FAMILIES+++def variant_for_family(family_name: Optional[str]) -> str:+ """ "base" for a base family, else "distilled" (the frontend applies the same rule)."""+ return "base" if "base" in (family_name or "") else "distilled"+++def resolve_enhance(family_name: Optional[str], enhance_prompt: Optional[bool]) -> bool:+ """Whether a generate call upsamples: None is the family default (on for every FLUX.3 Klein family), and every+ other family ignores the flag."""+ if not supports_family(family_name):+ return False+ return True if enhance_prompt is None else bool(enhance_prompt)+++def family_capabilities(family_name: Optional[str]) -> dict:+ """The two fields the loaded-model status carries."""+ supported = supports_family(family_name)+ return {"supports_prompt_upsampler": supported, "prompt_upsampler_default": supported}+++# ── request building ────────────────────────────────────────────────────────────────────────────────────────────────+def get_reference_size(width: int, height: int, num_references: int) -> tuple[int, int]:+ """Sides rounded to a multiple of 32, under the pixel budget for this many references."""+ if max(width, height) / max(1, min(width, height)) > MAX_ASPECT_RATIO:+ raise ValueError(+ f"The aspect ratio of a {width}x{height} reference image exceeds {MAX_ASPECT_RATIO}:1."+ )++ def floor(value):+ return max(+ REFERENCE_MULTIPLE, int(math.floor(value / REFERENCE_MULTIPLE)) * REFERENCE_MULTIPLE+ )++ budget = MAX_PIXELS_BY_REFS[min(max(1, num_references), MAX_REFERENCES)]+ w = max(REFERENCE_MULTIPLE, round(width / REFERENCE_MULTIPLE) * REFERENCE_MULTIPLE)+ h = max(REFERENCE_MULTIPLE, round(height / REFERENCE_MULTIPLE) * REFERENCE_MULTIPLE)+ if w * h > budget:+ scale = math.sqrt(width * height / budget)+ w, h = floor(width / scale), floor(height / scale)+ if w * h > budget:+ if w == REFERENCE_MULTIPLE:+ h = floor(budget / w)+ else:+ w = floor(budget / h)+ return w, h+++def prepare_reference(image: Any, num_references: int) -> bytes:+ """One reference as the upsampler saw them in training, as JPEG bytes: EXIF-upright, RGB over white, LANCZOS to+ the size budget, JPEG quality 92."""+ from PIL import Image, ImageOps++ image = ImageOps.exif_transpose(image)+ if image.mode in ("RGBA", "LA", "PA") or (image.mode == "P" and "transparency" in image.info):+ rgba = image.convert("RGBA")+ image = Image.alpha_composite(Image.new("RGBA", rgba.size, (255, 255, 255, 255)), rgba)+ image = image.convert("RGB")+ size = get_reference_size(image.width, image.height, num_references)+ if image.size != size:+ image = image.resize(size, Image.Resampling.LANCZOS)+ buffer = io.BytesIO()+ image.save(buffer, format = "JPEG", quality = JPEG_QUALITY)+ return buffer.getvalue()+++def decode_reference(data: str) -> Any:+ """A base64 / data-URL upload as a PIL image, untouched (prepare_reference does the rest)."""+ import base64+ import binascii++ from PIL import Image++ raw = (data or "").strip()+ if raw.startswith("data:"):+ _, _, raw = raw.partition(",")+ try:+ blob = base64.b64decode(raw, validate = False)+ except (binascii.Error, ValueError) as exc:+ raise ValueError(f"Invalid base64 image data: {exc}") from exc+ try:+ image = Image.open(io.BytesIO(blob))+ w, h = image.size+ if w > _MAX_UPLOAD_SIDE or h > _MAX_UPLOAD_SIDE:+ raise ValueError(f"Image is too large ({w}x{h}).")+ image.load()+ except ValueError:+ raise+ except Exception as exc: # noqa: BLE001 -- surfaced as a 400 / an unenhanced generate+ raise ValueError(f"Could not decode image: {exc}") from exc+ return image+++# Decoded uploads are large (an 8192x8192 RGBA is 256 MB): one at a time per request, two requests at once.+_DECODE_SLOTS = threading.BoundedSemaphore(2)+++def prepare_references(images: Optional[list]) -> list[bytes]:+ """Data URLs or PIL images as the upsampler's JPEG references (at most ten, in order)."""+ items = list(images or [])[:MAX_REFERENCES]+ prepared = []+ with _DECODE_SLOTS:+ for item in items:+ image = decode_reference(item) if isinstance(item, str) else item+ prepared.append(prepare_reference(image, len(items)))+ del image+ return prepared+++def references_hash(references: list[bytes]) -> str:+ digest = hashlib.sha256()+ for ref in references:+ digest.update(hashlib.sha256(ref).digest())+ return digest.hexdigest()+++def build_chat_payload(+ prompt: str,+ variant: str = "distilled",+ references: Optional[list[bytes]] = None,+ *,+ edit: Optional[bool] = None,+) -> dict:+ """The /v1/chat/completions body. ``references`` are prepared JPEGs; ``edit`` forces the edit prompt without+ images (a missing mmproj runs an edit text-only)."""+ if variant not in VARIANTS:+ raise ValueError(f"variant must be one of {', '.join(VARIANTS)}")+ references = list(references or [])+ edit = bool(references) if edit is None else bool(edit)+ mode = "i2i" if edit else "t2i"+ if edit:+ import base64+ content: list[dict] = [+ {+ "type": "image_url",+ "image_url": {+ "url": "data:image/jpeg;base64," + base64.b64encode(ref).decode("ascii")+ },+ }+ for ref in references+ ]+ content.append({"type": "text", "text": I2I_SCAFFOLD.format(instruction = prompt)})+ else:+ content = [{"type": "text", "text": prompt}]+ repeat_penalty, presence_penalty = PENALTIES[(mode, variant)]+ return {+ "messages": [+ {"role": "system", "content": SYSTEM_PROMPTS[mode]},+ {"role": "user", "content": content},+ ],+ **_BASE_SAMPLING,+ "repeat_penalty": repeat_penalty,+ "presence_penalty": presence_penalty,+ "max_tokens": MAX_NEW_TOKENS,+ "chat_template_kwargs": {"enable_thinking": False},+ "stream": False,+ }+++_THINK_BLOCK = re.compile(r"<think>.*?</think>", re.DOTALL)+++def clean_caption(text: Optional[str]) -> str:+ """The caption without a stray (empty) think block or surrounding quotes and whitespace."""+ text = _THINK_BLOCK.sub("", text or "").strip()+ if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'":+ text = text[1:-1].strip()+ return text+++# ── weights on disk ─────────────────────────────────────────────────────────────────────────────────────────────────+@dataclass(frozen = True)+class _UpsamplerSpec:+ repo: str+ model_file: str+ mmproj_file: str+ label: str+++UPSAMPLER_SPEC = _UpsamplerSpec(+ repo = UPSAMPLER_REPO,+ model_file = UPSAMPLER_MODEL_FILE,+ mmproj_file = UPSAMPLER_MMPROJ_FILE,+ label = "FLUX.3 prompt upsampler",+)+++def _env_path(name: str) -> Optional[str]:+ value = (os.environ.get(name) or "").strip()+ if not value:+ return None+ path = Path(value).expanduser()+ return str(path) if path.is_file() else None+++def _cached_file(+ filename: str,+ *,+ hub_cache: Optional[Path] = None,+ revision: Optional[str] = None,+) -> Optional[str]:+ try:+ from huggingface_hub import hf_hub_download++ from core.inference.stt_sidecar import _active_hf_hub_cache++ root = hub_cache if hub_cache is not None else _active_hf_hub_cache()+ return hf_hub_download(+ repo_id = UPSAMPLER_REPO,+ filename = filename,+ revision = revision,+ local_files_only = True,+ cache_dir = str(root),+ )+ except Exception:+ return None+++def _candidate_revisions(hub_cache: Optional[Path] = None) -> list:+ from core.inference.stt_sidecar import _fallback_revisions, _read_revision_record+ recorded = _read_revision_record(UPSAMPLER_REPO)+ return ([recorded] if recorded else []) + [+ rev for rev in _fallback_revisions(UPSAMPLER_REPO, hub_cache = hub_cache) if rev != recorded+ ]+++def _cached_at_any_revision(filename: str, hub_cache: Optional[Path] = None) -> Optional[str]:+ for revision in _candidate_revisions(hub_cache):+ path = _cached_file(filename, hub_cache = hub_cache, revision = revision)+ if path is not None:+ return path+ return None+++def _cached_pair(hub_cache: Optional[Path] = None) -> tuple[Optional[str], Optional[str]]:+ """The model and projector from ONE revision (the recorded one first), as the downloader pins them; a model+ with no projector at its revision when no revision holds both."""+ model_only = None+ for revision in _candidate_revisions(hub_cache):+ model = _cached_file(UPSAMPLER_MODEL_FILE, hub_cache = hub_cache, revision = revision)+ if model is None:+ continue+ mmproj = _cached_file(UPSAMPLER_MMPROJ_FILE, hub_cache = hub_cache, revision = revision)+ if mmproj is not None:+ return model, mmproj+ model_only = model_only or model+ return model_only, None+++def _cached_download_paths(+ model_id: str,+ *,+ hub_cache: Optional[Path] = None,+ revision: Optional[str] = None,+) -> Optional[tuple[str, str]]:+ """Both repo files at one revision, or None (the download state's completion check)."""+ if revision is None:+ model, mmproj = _cached_pair(hub_cache)+ else:+ model = _cached_file(UPSAMPLER_MODEL_FILE, hub_cache = hub_cache, revision = revision)+ mmproj = _cached_file(UPSAMPLER_MMPROJ_FILE, hub_cache = hub_cache, revision = revision)+ if model is None or mmproj is None:+ return None+ return model, mmproj+++def model_path() -> Optional[str]:+ """The upsampler GGUF: the env override, else the downloaded repo file."""+ if (os.environ.get(ENV_MODEL_OVERRIDE) or "").strip():+ # An override that does not exist is a typo, not a reason to read the cache instead.+ return _env_path(ENV_MODEL_OVERRIDE)+ return _cached_pair()[0]+++def mmproj_path() -> Optional[str]:+ if (os.environ.get(ENV_MMPROJ_OVERRIDE) or "").strip():+ return _env_path(ENV_MMPROJ_OVERRIDE)+ if _overridden():+ # Beside a local model file there is no revision to match.+ return _cached_at_any_revision(UPSAMPLER_MMPROJ_FILE)+ return _cached_pair()[1]+++def _overridden() -> bool:+ return any(+ (os.environ.get(name) or "").strip() for name in (ENV_MODEL_OVERRIDE, ENV_MMPROJ_OVERRIDE)+ )+++def find_llama_server_binary() -> Optional[str]:+ try:+ from core.inference.stt_mtmd_sidecar import find_llama_server_binary as find+ return find()+ except Exception: # noqa: BLE001 -- no llama.cpp is "not supported", never a crash+ return None+++def _resolve_upsampler_id(model_id: Optional[str]) -> str:+ if (model_id or UPSAMPLER_ID) != UPSAMPLER_ID:+ raise ValueError(f"Unknown prompt upsampler '{model_id}'.")+ return UPSAMPLER_ID+++_download_state_lock = threading.Lock()+_download_state = None+++def _get_download_state():+ global _download_state+ with _download_state_lock:+ if _download_state is None:+ from core.inference.stt_mtmd_sidecar import _MtmdDownloadState+ _download_state = _MtmdDownloadState(+ specs = {UPSAMPLER_ID: UPSAMPLER_SPEC},+ cached_paths = _cached_download_paths,+ resolve = _resolve_upsampler_id,+ label = "FLUX.3 prompt upsampler",+ )+ return _download_state+++def start_download(hf_token: Optional[str] = None) -> str:+ return _get_download_state().start(UPSAMPLER_ID, hf_token)+++def download_status() -> dict:+ return _get_download_state().status()+++def cancel_download(download_id: Optional[str] = None) -> bool:+ return _get_download_state().cancel(UPSAMPLER_ID, download_id)+++def status() -> dict:+ """The prompt-upsampler status route body."""+ model = model_path()+ mmproj = mmproj_path()+ model_name = (+ Path(model).name if model and os.environ.get(ENV_MODEL_OVERRIDE) else UPSAMPLER_MODEL_FILE+ )+ size = None+ if model:+ with suppress(OSError):+ size = os.path.getsize(model)+ return {+ "supported": find_llama_server_binary() is not None,+ "downloaded": model is not None,+ "loaded": get_sidecar().loaded,+ "repo": UPSAMPLER_REPO,+ "filename": model_name,+ "size_bytes": size or UPSAMPLER_FILE_SIZES.get(UPSAMPLER_MODEL_FILE),+ "mmproj_filename": UPSAMPLER_MMPROJ_FILE,+ "mmproj_size_bytes": UPSAMPLER_FILE_SIZES.get(UPSAMPLER_MMPROJ_FILE),+ "mmproj_downloaded": mmproj is not None,+ # Local files from the env overrides: the repo download is not offered beside them.+ "override": _overridden(),+ "download": download_status(),+ }+++# ── accelerator memory ──────────────────────────────────────────────────────────────────────────────────────────────+def _resident_cuda_ordinal() -> Optional[int]:+ """The CUDA card the resident image model sits on, else None."""+ try:+ from core.inference.diffusion_engine_router import get_active_diffusion_engine+ ordinal = getattr(+ getattr(get_active_diffusion_engine(), "_state", None), "placed_ordinal", None+ )+ return None if ordinal is None else int(ordinal)+ except Exception: # noqa: BLE001+ return None+++def _child_device_mask(ordinal: Optional[int]) -> dict:+ """Pin llama-server to the image model's card: left alone it splits layers over every visible GPU, onto a card the+ memory probe never read."""+ if ordinal is None:+ return {}+ try:+ import torch++ if torch.cuda.device_count() <= 1:+ return {}+ var = "CUDA_VISIBLE_DEVICES"+ if getattr(torch.version, "hip", None) and os.environ.get(var) is None:+ var = "HIP_VISIBLE_DEVICES"+ parent = os.environ.get(var)+ if parent is None:+ return {var: str(ordinal)}+ # Torch indexes inside the parent's mask, so the child keeps that entry (an index or a UUID).+ entries = [entry.strip() for entry in parent.split(",") if entry.strip()]+ return {var: entries[ordinal]} if ordinal < len(entries) else {}+ except Exception: # noqa: BLE001 -- no mask: llama.cpp's own device choice+ return {}+++def _accelerator_memory_mib(+ ordinal: Optional[int] = None,+) -> tuple[Optional[int], Optional[int], str]:+ """(free, total, device) of the accelerator llama.cpp would offload to (CUDA card ``ordinal`` when given), through+ the diffusion backend's helper. ``device`` is cuda / mps / cpu; free is None when unknown."""+ try:+ import torch+ from types import SimpleNamespace++ from core.inference.diffusion_memory import snapshot_device_memory++ if torch.cuda.is_available():+ device = "cuda"+ elif (+ getattr(getattr(torch, "backends", None), "mps", None) is not None+ and torch.backends.mps.is_available()+ ):+ device = "mps"+ else:+ return None, None, "cpu"+ from core.inference.diffusion_device import diffusion_device_scope++ with diffusion_device_scope(ordinal if device == "cuda" else None):+ snap = snapshot_device_memory(SimpleNamespace(device = device, backend = device))+ return snap.free_mib, snap.total_mib, device+ except Exception: # noqa: BLE001 -- unknown memory leaves the choice to the defaults+ return None, None, "unknown"+++def plan_gpu_layers(+ free_mib: Optional[int],+ model_bytes: int,+ mmproj_bytes: int = 0,+ device: str = "cuda",+ training: bool = False,+) -> int:+ """How many layers to offload: all when free memory holds weights + overhead, a share of them when it holds some,+ else none. Unknown memory offloads all (llama.cpp's own default); a CPU-only host and training get none."""+ if training or device == "cpu":+ return 0+ if device == "mps" or free_mib is None:+ return 99+ model_mib = model_bytes / (1024 * 1024)+ budget = free_mib - _GPU_OVERHEAD_MIB - mmproj_bytes / (1024 * 1024)+ if budget >= model_mib:+ return 99+ layers = int(_NUM_LAYERS * max(0.0, budget) / max(1.0, model_mib))+ return layers if layers >= _MIN_GPU_LAYERS else 0+++def vram_is_tight(free_mib: Optional[int], total_mib: Optional[int]) -> bool:+ if free_mib is None:+ return False+ floor = _TIGHT_FREE_MIB+ if total_mib:+ floor = max(floor, int(total_mib * _TIGHT_FREE_FRACTION))+ return free_mib < floor+++# ── the sidecar ─────────────────────────────────────────────────────────────────────────────────────────────────────+class PromptUpsamplerSidecar:+ """One llama-server serving the upsampler GGUF, one request at a time."""++ def __init__(self, keep_alive_seconds: Optional[float] = None) -> None:+ if keep_alive_seconds is None:+ from core.inference.stt_sidecar import STT_KEEP_ALIVE_SECONDS+ keep_alive_seconds = STT_KEEP_ALIVE_SECONDS+ self._keep_alive_seconds = float(keep_alive_seconds)+ self._lock = threading.RLock()+ # One request at a time (--parallel 1); also serialises load against use.+ self._request_lock = threading.Lock()+ self._process: Optional[subprocess.Popen] = None+ self._port: Optional[int] = None+ self._api_key: Optional[str] = None+ self._key: Optional[tuple] = None+ self._gpu_layers: Optional[int] = None+ self._penalize_prompt_supported = False+ self._loading = False+ self._starting_process: Optional[subprocess.Popen] = None+ self._load_cancel = threading.Event()+ self._update_in_progress = False+ self._idle_timer: Optional[threading.Timer] = None+ self._generation = 0+ self._active_requests = 0+ # A blocking unload bumps the epoch and holds ``_stopping``: requests queued before it never start a server.+ self._stop_epoch = 0+ self._stopping = 0+ self._gpu_ordinal: Optional[int] = None++ @property+ def loaded(self) -> bool:+ process = self._process+ return process is not None and process.poll() is None++ @property+ def loaded_model(self) -> Optional[str]:+ return UPSAMPLER_ID if self.loaded else None++ @property+ def gpu_layers(self) -> Optional[int]:+ return self._gpu_layers if self.loaded else None++ def is_loading(self) -> bool:+ return self._loading++ def holds_vram(self) -> bool:+ return self.loaded and bool(self._gpu_layers)++ # keep-alive+ def _cancel_idle_locked(self) -> None:+ if self._idle_timer is not None:+ self._idle_timer.cancel()+ self._idle_timer = None++ def _schedule_idle_locked(self) -> None:+ self._cancel_idle_locked()+ if self._keep_alive_seconds <= 0 or self._active_requests:+ return+ generation = self._generation+ timer = threading.Timer(self._keep_alive_seconds, self._idle_unload, args = (generation,))+ timer.daemon = True+ self._idle_timer = timer+ timer.start()++ def _idle_unload(self, generation: int) -> None:+ with self._lock:+ if generation != self._generation or self._active_requests:+ return+ self._release_locked()++ def _release_locked(self) -> None:+ from core.inference.stt_mtmd_sidecar import _reap++ self._cancel_idle_locked()+ self._generation += 1+ process = self._process+ try:+ _reap(process)+ finally:+ self._process = None+ self._port = None+ self._api_key = None+ self._key = None+ self._gpu_layers = None+ self._penalize_prompt_supported = False++ def unload(self, wait: bool = True) -> None:+ """Stop the server. ``wait=False`` leaves a busy one alone."""+ if not wait and (self._loading or self._active_requests):+ return+ if wait:+ with self._lock:+ self._stop_epoch += 1+ self._stopping += 1+ try:+ self.cancel_pending_load()+ if wait:+ deadline = time.monotonic() + 30.0+ while self._active_requests and time.monotonic() < deadline:+ time.sleep(0.1)+ with self._lock:+ if not wait and self._active_requests:+ return+ self._release_locked()+ finally:+ if wait:+ with self._lock:+ self._stopping -= 1++ def cancel_pending_load(self) -> bool:+ if not self._loading:+ return False+ self._load_cancel.set()+ process = self._starting_process+ if process is not None and process.poll() is None:+ with suppress(Exception):+ process.terminate()+ return True++ @contextmanager+ def update_maintenance(self) -> Iterator[bool]:+ """Refuse loads and stop the server while the llama.cpp tree it runs from is replaced."""+ self._update_in_progress = True+ try:+ self.cancel_pending_load()+ with self._request_lock, self._lock:+ was_active = self.loaded+ self._release_locked()+ yield was_active+ finally:+ self._update_in_progress = False++ def _launch(self, binary: str, model: str, mmproj: Optional[str]) -> None:+ """Start llama-server for (model, mmproj). Called under ``_request_lock``."""+ # Published before the training probe: training that starts after it finds a load to cancel.+ self._load_cancel.clear()+ with self._lock:+ self._loading = True+ try:+ self._spawn_server(binary, model, mmproj)+ finally:+ with self._lock:+ self._loading = False+ self._starting_process = None++ def _spawn_server(self, binary: str, model: str, mmproj: Optional[str]) -> None:+ from core.inference.stt_mtmd_sidecar import (+ MtmdSttSidecar,+ _llama_server_child_env,+ _reap,+ )+ from core.inference.stt_sidecar import _training_active+ from utils.process_lifetime import (+ adopt_pid,+ child_popen_kwargs,+ is_process_shutting_down,+ spawn_on_lifetime_thread,+ )++ model_bytes = os.path.getsize(model)+ mmproj_bytes = os.path.getsize(mmproj) if mmproj else 0+ ordinal = _resident_cuda_ordinal()+ free_mib, _total, device = _accelerator_memory_mib(ordinal)+ training = _training_active()+ # Nobody holds the GPU (a CPU image load, no model): offloaded layers would sit outside every eviction.+ cpu_only = training or _gpu_unowned()+ layers = plan_gpu_layers(free_mib, model_bytes, mmproj_bytes, device, cpu_only)+ sock, port = MtmdSttSidecar._reserve_free_port()+ cmd = [+ binary,+ "-m",+ model,+ "--host",+ "127.0.0.1",+ "--port",+ str(port),+ "--parallel",+ "1",+ "-c",+ str(_CONTEXT_TOKENS),+ "-ngl",+ str(layers),+ "--jinja",+ ]+ if mmproj:+ cmd += ["--mmproj", mmproj]+ if layers == 0:+ cmd.append("--no-mmproj-offload")+ sock.close()+ if is_process_shutting_down():+ raise UpsamplerUnavailableError("Unsloth is shutting down.")+ from core.inference.llama_cpp import (+ LlamaCppBackend,+ _llama_server_api_key_enabled,+ _llama_server_key_launch,+ )++ api_key = key_file = None+ key_env: dict = {}+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(+ binary+ ).get("supports_api_key_file", True):+ api_key = secrets.token_urlsafe(32)+ key_argv, key_env, key_file = _llama_server_key_launch(api_key)+ cmd[1:1] = key_argv+ logger.info(+ "flux3.prompt_upsampler: starting llama-server (%s, -ngl %s, free %s MiB on %s%s)",+ Path(model).name,+ layers,+ free_mib,+ device,+ ", vision" if mmproj else "",+ )+ if self._load_cancel.is_set():+ raise UpsamplerCancelledError("The prompt upsampler load was cancelled.")+ # Undrained pipes would block llama-server once its log fills them, so its output goes nowhere unless a log+ # file is asked for.+ log_path = (os.environ.get(ENV_SERVER_LOG) or "").strip()+ log_file = open(log_path, "ab") if log_path else None+ try:+ env = {**_llama_server_child_env(binary), **key_env, **_child_device_mask(ordinal)}+ # Forked from the process-lifetime thread: PDEATHSIG fires when the forking THREAD exits, and requests+ # run on executor workers, so a server spawned there died with its worker and the keep-alive never held.+ process = spawn_on_lifetime_thread(+ lambda: subprocess.Popen(+ cmd,+ stdout = log_file or subprocess.DEVNULL,+ stderr = subprocess.STDOUT if log_file else subprocess.DEVNULL,+ stdin = subprocess.DEVNULL,+ env = env,+ **child_popen_kwargs(),+ )+ )+ with self._lock:+ self._starting_process = process+ adopt_pid(process.pid)+ if is_process_shutting_down():+ _reap(process)+ raise UpsamplerUnavailableError("Unsloth is shutting down.")+ try:+ ready = _wait_for_server(process, port, self._load_cancel)+ finally:+ if key_file is not None:+ with suppress(OSError):+ key_file.unlink()+ if not ready:+ _reap(process)+ if self._load_cancel.is_set():+ raise UpsamplerCancelledError("The prompt upsampler load was cancelled.")+ raise RuntimeError("llama-server did not become ready for the prompt upsampler.")+ penalize_prompt_supported = _server_has_penalize_prompt(port, api_key)+ with self._lock:+ self._process = process+ self._port = port+ self._api_key = api_key+ self._penalize_prompt_supported = penalize_prompt_supported+ self._key = (binary, model, mmproj)+ self._gpu_layers = layers+ self._gpu_ordinal = ordinal+ self._generation += 1+ finally:+ if log_file is not None:+ with suppress(OSError):+ log_file.close()++ def generate(+ self,+ payload: dict,+ *,+ binary: str,+ model: str,+ mmproj: Optional[str],+ cancel_event: Optional[threading.Event] = None,+ ) -> str:+ """Run one chat completion and return the raw text, (re)starting the server for these files."""+ if self._update_in_progress:+ raise UpsamplerUnavailableError("llama.cpp is being updated.")+ epoch = self._stop_epoch+ with self._request_lock:+ if cancel_event is not None and cancel_event.is_set():+ raise UpsamplerCancelledError("Cancelled.")+ with self._lock:+ self._cancel_idle_locked()+ current = self._key if self.loaded else None+ # A server with the vision projector also serves text-only requests, so a t2i after an edit does+ # not relaunch.+ if current is not None and not (+ current[:2] == (binary, model) and (mmproj is None or current[2] == mmproj)+ ):+ self._release_locked()+ if self._stopping or epoch != self._stop_epoch:+ # Queued behind a request a training start or GPU handover drained: starting now would sit beside+ # the new owner.+ raise UpsamplerUnavailableError(+ "The prompt upsampler was stopped for another GPU user."+ )+ # Claimed under the same lock: an idle callback already waiting on it then sees the request.+ self._active_requests += 1+ try:+ if not self.loaded:+ if not _image_model_owns_gpu():+ # Chat or video took the GPU after this request read the FLUX.3 status: starting now would+ # sit beside the new owner for the keep-alive.+ raise UpsamplerUnavailableError("The image model no longer holds the GPU.")+ with _forward_cancel(cancel_event, self.cancel_pending_load):+ self._launch(binary, model, mmproj)+ with self._lock:+ port, api_key = self._port, self._api_key+ generated_only = self._penalize_prompt_supported+ if generated_only:+ payload = {**payload, **GENERATED_ONLY_PENALTIES}+ return _post_chat(port, payload, api_key = api_key, cancel_event = cancel_event)+ finally:+ with self._lock:+ self._active_requests -= 1+ self._schedule_idle_locked()++ def release_if_tight(self) -> bool:+ """Unload now when the server holds VRAM and the accelerator is short, so the DiT and text encoder do not+ denoise next to it. True when it unloaded."""+ if not self.holds_vram():+ return False+ free_mib, total_mib, _device = _accelerator_memory_mib(self._gpu_ordinal)+ if not vram_is_tight(free_mib, total_mib):+ return False+ logger.info(+ "flux3.prompt_upsampler: unloading after the call (free %s of %s MiB)",+ free_mib,+ total_mib,+ )+ self.unload(wait = False)+ return True+++def _gpu_unowned() -> bool:+ try:+ from core.inference import gpu_arbiter+ return gpu_arbiter.current_owner() is None+ except Exception: # noqa: BLE001+ return False+++def _image_model_owns_gpu() -> bool:+ """The GPU arbiter's owner is the image model (or nobody yet, as on a fresh start). Fail-open."""+ try:+ from core.inference import gpu_arbiter+ return gpu_arbiter.current_owner() in (None, gpu_arbiter.DIFFUSION)+ except Exception: # noqa: BLE001+ return True+++@contextmanager+def _forward_cancel(cancel_event: Optional[threading.Event], cancel: Any) -> Iterator[None]:+ """Stop pressed while llama-server starts cancels the start, not just the request after it."""+ if cancel_event is None:+ yield+ return+ done = threading.Event()++ def _watch() -> None:+ # Retried until it lands: a Stop just before the start flips `_loading` would otherwise be dropped.+ while not done.wait(0.2):+ if cancel_event.is_set() and cancel():+ return++ watcher = threading.Thread(target = _watch, name = "flux3-upsampler-cancel", daemon = True)+ watcher.start()+ try:+ yield+ finally:+ done.set()+ watcher.join(timeout = 1)+++def _wait_for_server(process, port: int, cancel_event: threading.Event) -> bool:+ from core.inference.stt_mtmd_sidecar import MtmdSttSidecar+ return MtmdSttSidecar._wait_for_server(process, port, cancel_event)+++def _server_has_penalize_prompt(port: int, api_key: Optional[str] = None) -> bool:+ """Whether this llama-server takes the `penalize_prompt` request field: builds that have it echo it among the+ default generation settings in /props. Any failure reads as an older server."""+ connection = http.client.HTTPConnection("127.0.0.1", port, timeout = 10)+ try:+ connection.request(+ "GET", "/props", headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}+ )+ response = connection.getresponse()+ if response.status != 200:+ return False+ settings = (json.loads(response.read()) or {}).get("default_generation_settings") or {}+ return "penalize_prompt" in settings or "penalize_prompt" in (settings.get("params") or {})+ except (OSError, ValueError, AttributeError, http.client.HTTPException):+ return False+ finally:+ connection.close()+++def _post_chat(+ port: int,+ payload: dict,+ *,+ api_key: Optional[str] = None,+ cancel_event: Optional[threading.Event] = None,+) -> str:+ from core.inference.stt_sidecar import _close_connection_on_cancel++ connection = http.client.HTTPConnection("127.0.0.1", port, timeout = _REQUEST_TIMEOUT_SECONDS)+ done = threading.Event()+ if cancel_event is not None:+ threading.Thread(+ target = _close_connection_on_cancel,+ args = (connection, cancel_event, done),+ daemon = True,+ ).start()+ try:+ connection.request(+ "POST",+ "/v1/chat/completions",+ body = json.dumps(payload).encode("utf-8"),+ headers = {+ "Content-Type": "application/json",+ **({"Authorization": f"Bearer {api_key}"} if api_key else {}),+ },+ )+ with connection.getresponse() as response:+ raw = response.read()+ if not 200 <= response.status < 300:+ raise RuntimeError(f"The prompt upsampler returned HTTP {response.status}.")+ body = json.loads(raw.decode("utf-8"))+ except Exception:+ if cancel_event is not None and cancel_event.is_set():+ raise UpsamplerCancelledError("Cancelled.")+ raise+ finally:+ done.set()+ connection.close()+ choices = body.get("choices") or []+ return (choices[0].get("message", {}).get("content") or "") if choices else ""+++_sidecar: Optional[PromptUpsamplerSidecar] = None+_sidecar_lock = threading.Lock()+++def get_sidecar() -> PromptUpsamplerSidecar:+ global _sidecar+ with _sidecar_lock:+ if _sidecar is None:+ _sidecar = PromptUpsamplerSidecar()+ return _sidecar+++def stop_for_handover() -> None:+ """The image model the server serves is leaving the GPU: stop every run still enhancing and the server, blocking,+ so nothing of it outlives the DIFFUSION claim."""+ cancel_active()+ unload_sidecar(wait = True)+++def unload_sidecar(wait: bool = False) -> None:+ """Stop the server if one was ever started (the FLUX.3 model it serves was unloaded). Never creates one.+ ``wait`` also stops a busy one, for a GPU handover."""+ sidecar = _sidecar+ if sidecar is not None:+ sidecar.unload(wait = wait)+++# ── upsampling with cache, progress and fallback ────────────────────────────────────────────────────────────────────+_cache_lock = threading.Lock()+_cache: "collections.OrderedDict[tuple, str]" = collections.OrderedDict()+++def _cache_get(key: tuple) -> Optional[str]:+ with _cache_lock:+ value = _cache.get(key)+ if value is not None:+ _cache.move_to_end(key)+ return value+++def _cache_put(key: tuple, value: str) -> None:+ with _cache_lock:+ _cache[key] = value+ _cache.move_to_end(key)+ while len(_cache) > _CACHE_ENTRIES:+ _cache.popitem(last = False)+++def clear_cache() -> None:+ with _cache_lock:+ _cache.clear()+++# Runs in flight, oldest (the one holding llama-server) first, for generate-progress and the Stop button.+_progress_lock = threading.Lock()+_runs: list = []+++def progress() -> Optional[dict]:+ """``{"done", "total", "message"}`` of the run being enhanced, else None."""+ with _progress_lock:+ return dict(_runs[0][0]) if _runs else None+++def cancel_active() -> bool:+ """Stop every run still enhancing (a queued one waits on the same server)."""+ with _progress_lock:+ events = [event for _state, event in _runs if not event.is_set()]+ for event in events:+ event.set()+ return bool(events)+++@contextmanager+def _tracking(total: int, cancel_event: threading.Event) -> Iterator[dict]:+ run = ({"done": 0, "total": total, "message": "Enhancing prompt..."}, cancel_event)+ with _progress_lock:+ _runs.append(run)+ try:+ yield run[0]+ finally:+ with _progress_lock:+ _runs.remove(run)+++def _advance(state: dict) -> None:+ with _progress_lock:+ state["done"] += 1+++def upsample(+ prompt: str,+ *,+ variant: str = "distilled",+ reference_images: Optional[list] = None,+ cancel_event: Optional[threading.Event] = None,+ sidecar: Optional[PromptUpsamplerSidecar] = None,+) -> str:+ """The caption for ``prompt`` (cached). Raises UpsamplerUnavailableError when nothing can run, a cancel as+ UpsamplerCancelledError, and anything else the server does. An empty caption returns ``prompt``."""+ binary = find_llama_server_binary()+ if binary is None:+ raise UpsamplerUnavailableError("llama.cpp (llama-server) is not installed.")+ model = model_path()+ if model is None:+ raise UpsamplerNotDownloadedError("The FLUX.3 prompt upsampler is not downloaded.")+ edit = bool(reference_images)+ mmproj = mmproj_path() if edit else None+ references: list[bytes] = []+ if edit:+ if mmproj is None:+ logger.warning(+ "flux3.prompt_upsampler: %s is not downloaded; enhancing the edit text-only",+ UPSAMPLER_MMPROJ_FILE,+ )+ else:+ references = prepare_references(reference_images)+ payload = build_chat_payload(prompt, variant, references, edit = edit)+ key = (prompt, variant, edit, references_hash(references), model)+ cached = _cache_get(key)+ if cached is not None:+ # A Stop while the references decoded never reaches the sidecar, which is where it is normally enforced.+ if cancel_event is not None and cancel_event.is_set():+ raise UpsamplerCancelledError("Cancelled.")+ return cached+ sidecar = sidecar or get_sidecar()+ text = clean_caption(+ sidecar.generate(+ payload, binary = binary, model = model, mmproj = mmproj, cancel_event = cancel_event+ )+ )+ if not text:+ logger.warning("flux3.prompt_upsampler: empty caption; using the prompt as typed")+ return prompt+ _cache_put(key, text)+ return text+++@dataclass+class EnhanceOutcome:+ """Per-prompt captions for one generate call. ``status`` is None for families without an upsampler."""++ status: Optional[str]+ captions: dict = field(default_factory = dict)+ statuses: dict = field(default_factory = dict)+ # The resident model the captions were written for, set by the generate route.+ load: Any = None++ def caption_for(self, prompt: str) -> Optional[str]:+ return self.captions.get(prompt)++ def status_for(self, prompt: str) -> Optional[str]:+ return self.statuses.get(prompt, self.status)++ def encoded(self, prompt: str) -> str:+ return self.captions.get(prompt) or prompt+++def enhance_for_generation(+ prompts: list[str],+ *,+ family_name: Optional[str],+ enhance_prompt: Optional[bool],+ reference_images: Optional[list] = None,+ cancel_event: Optional[threading.Event] = None,+) -> EnhanceOutcome:+ """Captions for each distinct prompt of a generate call. Never raises except on cancel: anything that stops the+ upsampler leaves the prompt as typed and says why in the status."""+ if not supports_family(family_name):+ return EnhanceOutcome(status = None)+ if not resolve_enhance(family_name, enhance_prompt):+ return EnhanceOutcome(status = STATUS_OFF)+ if find_llama_server_binary() is None or model_path() is None:+ logger.warning(+ "flux3.prompt_upsampler: unavailable (%s); generating with the prompt as typed",+ "llama-server not found"+ if find_llama_server_binary() is None+ else "GGUF not downloaded",+ )+ return EnhanceOutcome(status = STATUS_UNAVAILABLE)+ variant = variant_for_family(family_name)+ distinct = list(dict.fromkeys(p for p in prompts if isinstance(p, str)))+ cancel_event = cancel_event or threading.Event()+ outcome = EnhanceOutcome(status = STATUS_APPLIED)+ sidecar = get_sidecar()+ # On every exit, a Stop included: a retry with enhancement off must not denoise beside a tight server.+ try:+ with _tracking(len(distinct), cancel_event) as tracked:+ for index, prompt in enumerate(distinct):+ if cancel_event.is_set():+ raise UpsamplerCancelledError("Cancelled.")+ systemic = None+ try:+ caption = upsample(+ prompt,+ variant = variant,+ reference_images = reference_images,+ cancel_event = cancel_event,+ sidecar = sidecar,+ )+ except UpsamplerCancelledError:+ raise+ except UpsamplerUnavailableError as exc:+ logger.warning("flux3.prompt_upsampler: unavailable (%s); prompt as typed", exc)+ systemic = STATUS_UNAVAILABLE+ except Exception as exc: # noqa: BLE001 -- an upsampler failure never fails the image+ if cancel_event.is_set():+ raise UpsamplerCancelledError("Cancelled.") from exc+ logger.warning("flux3.prompt_upsampler: failed (%s); prompt as typed", exc)+ systemic = STATUS_FAILED+ else:+ if caption == prompt:+ outcome.statuses[prompt] = STATUS_FAILED+ else:+ outcome.captions[prompt] = caption+ outcome.statuses[prompt] = STATUS_APPLIED+ if systemic is not None:+ # The runtime itself failed (a start that timed out, a dead server): the rest of the batch would wait+ # out the same timeout each, so they run as typed now.+ for rest in distinct[index:]:+ outcome.statuses[rest] = systemic+ break+ _advance(tracked)+ finally:+ with suppress(Exception):+ sidecar.release_if_tight()+ if outcome.statuses and not outcome.captions:+ # Nothing applied: the batch-level status is the first reason.+ outcome.status = next(iter(outcome.statuses.values()))+ return outcomediff --git a/studio/backend/core/inference/gpu_arbiter.py b/studio/backend/core/inference/gpu_arbiter.pyindex 238db506e90..6fc7d184af2 100644--- a/studio/backend/core/inference/gpu_arbiter.py+++ b/studio/backend/core/inference/gpu_arbiter.py@@ -11,6 +11,7 @@ under the lock, so a transfer is atomic vs other acquires.from __future__ import annotations+import sysimport threadingfrom typing import Any, Callable, Optional@@ -79,6 +80,19 @@ def _evict_diffusion() -> None:# Unload whichever engine the router has active (diffusers or native sd.cpp).from core.inference.diffusion_engine_router import get_active_diffusion_engineget_active_diffusion_engine().unload()+ _stop_prompt_upsampler()+++def _stop_prompt_upsampler() -> None:+ """The FLUX.3 prompt upsampler serves the image model, so it leaves with every DIFFUSION claim: an evicted, a+ released or a replaced one. Only if Images ever started it."""+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")+ if module is None:+ return+ try:+ module.stop_for_handover()+ except Exception as exc: # noqa: BLE001 - never block an acquire on this+ logger.warning("gpu_arbiter: could not stop the FLUX.3 prompt upsampler: %s", exc)def _release_idle_video_servers(reason: str) -> None:@@ -252,6 +266,8 @@ def release(owner: str) -> None:global _owner, _owner_epoch, _owner_account, _prior_accountwith _lock:if _owner == owner:+ if owner == DIFFUSION:+ _stop_prompt_upsampler()_owner = None_owner_account = None_prior_account = None@@ -269,6 +285,8 @@ def release_if(owner: str, predicate: Callable[[], bool]) -> bool:with _lock:if _owner != owner or not predicate():return False+ if owner == DIFFUSION:+ _stop_prompt_upsampler()_owner = None_owner_account = None_prior_account = Nonediff --git a/studio/backend/core/inference/image_gallery.py b/studio/backend/core/inference/image_gallery.pyindex 03f4f627496..aa01968ef6a 100644--- a/studio/backend/core/inference/image_gallery.py+++ b/studio/backend/core/inference/image_gallery.py@@ -50,7 +50,9 @@ _IMG2IMG_WORKFLOWS = ("img2img", "inpaint", "outpaint", "edit", "upscale")def _params_text(meta: dict[str, Any]) -> str:"""Automatic1111-style ``parameters`` string for cross-tool interop."""- lines = [str(meta.get("prompt", ""))]+ # The caption the model encoded (FLUX.3 prompt upsampler): other tools reproduce the image from this line. The+ # typed prompt stays in the Unsloth chunk.+ lines = [str(meta.get("enhanced_prompt") or meta.get("prompt", ""))]negative = meta.get("negative_prompt")if negative:lines.append(f"Negative prompt: {negative}")diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.pyindex 9dbe23f5ed8..ab6e113a839 100644--- a/studio/backend/core/inference/stt_mtmd_sidecar.py+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py@@ -314,9 +314,23 @@ def is_model_downloaded(model_id: str) -> bool:class _MtmdDownloadState:- """Tracks the two-file download of one mtmd dictation model."""+ """Tracks the two-file download of one mtmd dictation model.- def __init__(self) -> None:+ ``specs`` / ``cached_paths`` / ``resolve`` let another llama.cpp sidecar (the FLUX.3 prompt upsampler) reuse the+ same pinned two-file download; None keeps the dictation catalogue, looked up at call time.+ """++ def __init__(+ self,+ specs: Optional[dict] = None,+ cached_paths = None,+ resolve = None,+ label: str = "mtmd STT",+ ) -> None:+ self._specs = specs+ self._cached_paths_fn = cached_paths+ self._resolve_fn = resolve+ self._label = labelself._lock = threading.Lock()self._thread: Optional[threading.Thread] = Noneself._process: Optional[subprocess.Popen] = None@@ -330,6 +344,15 @@ class _MtmdDownloadState:self._hub_cache: Optional[Path] = Noneself._cancelled = False+ def _spec(self, model_id: str):+ return (self._specs if self._specs is not None else MTMD_STT_MODELS)[model_id]++ def _resolve(self, model_id: Optional[str]) -> str:+ return (self._resolve_fn or resolve_mtmd_model_id)(model_id)++ def _cached_paths(self, model_id: str, **kwargs):+ return (self._cached_paths_fn or _cached_model_paths)(model_id, **kwargs)+def status(self) -> dict:with self._lock:downloading = self._thread is not None and self._thread.is_alive()@@ -403,7 +426,7 @@ class _MtmdDownloadState:total = total if total is not None else self._total_bytesif not model_id or hub_cache is None or not selected_files:return None- repo = MTMD_STT_MODELS[model_id].repo+ repo = self._spec(model_id).repodone = sum(_downloaded_file_bytes(hub_cache = hub_cache,@@ -424,7 +447,7 @@ class _MtmdDownloadState:model_id: str,hf_token: Optional[str] = None,) -> str:- model_id = resolve_mtmd_model_id(model_id)+ model_id = self._resolve(model_id)hub_cache = _capture_stt_hub_cache()with self._lock:if self._thread is not None and self._thread.is_alive():@@ -465,7 +488,7 @@ class _MtmdDownloadState:if hub_cache is None:from core.inference.stt_sidecar import _active_hf_hub_cachehub_cache = self._hub_cache or _active_hf_hub_cache()- spec = MTMD_STT_MODELS[model_id]+ spec = self._spec(model_id)registry = Noneowner = Nonetry:@@ -527,7 +550,7 @@ class _MtmdDownloadState:cancelled = self._cancelledif process.returncode == 0 and not cancelled:if (- _cached_model_paths(+ self._cached_paths(model_id,hub_cache = hub_cache,revision = revision,@@ -548,13 +571,13 @@ class _MtmdDownloadState:self._cancelled = Truereturndetail = stderr.decode("utf-8", "replace").strip()- logger.warning("mtmd STT download failed for %s: %s", model_id, detail)+ logger.warning("%s download failed for %s: %s", self._label, model_id, detail)with self._lock:self._error = modelscope_missing(detail) or f"Download failed for '{model_id}'."except Exception as exc:with self._lock:if not self._cancelled:- logger.warning("mtmd STT download failed for %s: %s", model_id, exc)+ logger.warning("%s download failed for %s: %s", self._label, model_id, exc)self._error = modelscope_missing(exc) or f"Download failed for '{model_id}'."finally:# Release first: it is the half that would wedge the repository.diff --git a/studio/backend/core/inference/stt_sidecar.py b/studio/backend/core/inference/stt_sidecar.pyindex 8699ae1c751..f87efa24c52 100644--- a/studio/backend/core/inference/stt_sidecar.py+++ b/studio/backend/core/inference/stt_sidecar.py@@ -168,6 +168,17 @@ def _downloaded_file_bytes(if blob_key:blobs = repo_cache / "blobs"candidates.extend((blobs / f"{blob_key}.incomplete", blobs / blob_key))+ # huggingface_hub >= 1.18 writes a process-unique `<etag>.<8 hex>.incomplete` partial instead.+ from hub.utils.hf_cache_state import incomplete_blob_hash++ try:+ candidates.extend(+ partial+ for partial in blobs.glob(f"{blob_key}.*.incomplete")+ if incomplete_blob_hash(partial.name) == blob_key+ )+ except OSError:+ passif revision:candidates.append(repo_cache / "snapshots" / revision / filename)sizes: list[int] = []diff --git a/studio/backend/loggers/handlers.py b/studio/backend/loggers/handlers.pyindex 5331f2fd3cc..5f32f469e54 100644--- a/studio/backend/loggers/handlers.py+++ b/studio/backend/loggers/handlers.py@@ -120,6 +120,8 @@ _QUIET_SUCCESS_PATHS = {"/api/inference/video/load-progress","/api/inference/images/generate-progress","/api/inference/video/generate-progress",+ # Polled by the Images page while the FLUX.3 prompt upsampler downloads.+ "/api/inference/images/prompt-upsampler/status","/api/llama/update-status",# Engine status, re-read every 5s by the Hub while the vLLM GPU probe runs and by the open Settings engine list.# The UI shows each answer, install jobs log their own progress, and a failed read (non-2xx) still logs.diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 0e06a1c273d..189cd85b209 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -4843,6 +4843,12 @@ class DiffusionGenerateRequest(BaseModel):"forward pass (plain text-to-image only). Overrides `prompt` for the images; ""`prompt` is still required as the fallback/display value.",)+ enhance_prompt: Optional[bool] = Field(+ None,+ description = "Rewrite the prompt with the FLUX.3 prompt upsampler before encoding. Null = the "+ "family default (on for every FLUX.3 Klein family); ignored by every other family. An "+ "unavailable or failing upsampler leaves the prompt as typed.",+ )seeds: Optional[list[int]] = Field(None,min_length = 1,@@ -5040,6 +5046,16 @@ class GalleryImage(BaseModel):id: str = Field(..., description = "Stable id (the on-disk filename stem)")url: str = Field(..., description = "Relative URL to fetch the PNG bytes")prompt: str = Field(..., description = "Prompt used")+ enhanced_prompt: Optional[str] = Field(+ None,+ description = "The caption the prompt upsampler wrote and the model encoded; null when "+ "not enhanced. Replaying it with enhance_prompt=false reproduces the image.",+ )+ prompt_enhancement: Optional[Literal["applied", "off", "unavailable", "failed"]] = Field(+ None,+ description = "Prompt upsampling outcome for families that have one: applied | off | "+ "unavailable (not downloaded or no llama-server) | failed; null otherwise.",+ )negative_prompt: Optional[str] = Field(None, description = "Negative prompt, if any")width: int = Field(..., description = "Image width")height: int = Field(..., description = "Image height")@@ -5204,6 +5220,44 @@ class GalleryListResponse(BaseModel):has_more: bool = Field(False, description = "Whether older images remain past this page")+class PromptUpsamplerDownloadRequest(BaseModel):+ """Body of POST /images/prompt-upsampler/download (optional)."""++ hf_token: Optional[str] = Field(None, description = "Hugging Face token for the download")+++class PromptUpsamplerPreviewRequest(BaseModel):+ """Enhance one prompt with the FLUX.3 prompt upsampler without generating."""++ prompt: str = Field(..., min_length = 1, description = "The prompt as typed")+ reference_images: Optional[list[str]] = Field(+ None,+ max_length = 10,+ description = "Reference images (base64/data-URL, as /images/generate) for an edit, in order",+ )+ variant: Optional[Literal["distilled", "base"]] = Field(+ None,+ description = "Sampling variant; null = the loaded FLUX.3 family's (distilled when none is loaded)",+ )++ @field_validator("reference_images")+ @classmethod+ def _bounded_reference_items(cls, value: Optional[list[str]]) -> Optional[list[str]]:+ if value is not None:+ for item in value:+ if len(item) > 32 * 1024 * 1024:+ raise ValueError("each reference image must be at most 32 MiB (base64)")+ if sum(len(item) for item in value) > _MAX_CONDITION_PAYLOAD:+ raise ValueError(+ "the input images together must be at most 128 MiB (base64); use smaller images"+ )+ return value+++class PromptUpsamplerPreviewResponse(BaseModel):+ enhanced_prompt: str = Field(..., description = "The caption the upsampler wrote")++class DiffusionGenerateProgressResponse(BaseModel):"""Live per-step progress for an in-flight generation."""@@ -5214,12 +5268,17 @@ class DiffusionGenerateProgressResponse(BaseModel):eta_seconds: Optional[float] = Field(None, description = "Estimated seconds remaining")phase: Optional[str] = Field(None,- description = "encode | denoise | decode; null from engines that report no phase (sd.cpp)",+ description = "enhance (FLUX.3 prompt upsampling, before the run) | encode | denoise | decode; null "+ "from engines that report no phase (sd.cpp)",)preview: Optional[str] = Field(None, description = "Live latent preview of the image being denoised, as a JPEG data URL")preview_seq: int = Field(0, description = "Moves each time a new preview is published")+ message: Optional[str] = Field(+ None,+ description = "Status text for a phase with no steps (phase 'enhance': 'Enhancing prompt...')",+ )target: Optional[Literal["connection"]] = Field(None,description = "'connection' when the run is on a saved image connection, so a page that "@@ -5352,6 +5411,13 @@ class DiffusionStatusResponse(BaseModel):modular_families: List[str] = Field(default_factory = list, description = "Loadable families shipped as Modular Diffusers pipelines")+ supports_prompt_upsampler: bool = Field(+ False,+ description = "Whether the loaded family can enhance prompts with the FLUX.3 prompt upsampler",+ )+ prompt_upsampler_default: bool = Field(+ False, description = "Whether enhance_prompt defaults to on for the loaded family"+ )base_repo: Optional[str] = Field(None, description = "Companion diffusers base repo")device: Optional[str] = Field(None, description = "Device the pipeline is on")dtype: Optional[str] = Field(None, description = "Compute dtype")diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 659e0879fb8..1b2ee886b3b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3779,6 +3779,9 @@ from models.inference import (DiffusionLoadRequest,DiffusionGenerateRequest,DiffusionGenerateResponse,+ PromptUpsamplerDownloadRequest,+ PromptUpsamplerPreviewRequest,+ PromptUpsamplerPreviewResponse,DiffusionGenerateProgressResponse,DiffusionStatusResponse,DiffusionDownloadPlanResponse,@@ -50206,6 +50209,10 @@ async def load_diffusion_model_gated(# worker is handed the credential.for _ref in _media_repos_to_record:_note_load_fetched_with_a_request_token(_ref, request.hf_token)+ # Past admission: the upsampler serves the model being replaced, and a same-owner swap runs no eviction.+ from core.inference.flux3_prompt_upsampler import stop_for_handover++ stop_for_handover()# Kicks the slow load onto a background thread and returns at once (the client polls images/load-progress).return engine.begin_load(request.model_path,@@ -50552,6 +50559,56 @@ async def _external_image_generation(_stopped_external_image_runs.discard(task)+async def _enhance_generate_prompts(backend, request: DiffusionGenerateRequest):+ """FLUX.3 prompt upsampling for one generate call (``EnhanceOutcome``). Every other family, and FLUX.3 with+ ``enhance_prompt=false``, passes its prompts through untouched. Never fails the generation: only a Stop during+ the upsampling surfaces, as the cancelled 409."""+ from core.inference import flux3_prompt_upsampler as pup+ from core.inference.diffusion_families import DIFFUSION_CANCELLED_MSG++ try:+ status = await asyncio.to_thread(backend.status)+ family = status.get("family") if status.get("loaded") else None+ except Exception as exc: # noqa: BLE001 -- no status, no upsampling+ logger.debug("flux3.prompt_upsampler: status probe failed: %s", exc)+ status, family = {}, None+ if not pup.supports_family(family):+ return pup.EnhanceOutcome(status = None)+ # A workflow the family refuses fails in the run anyway: no upsampler start (or vision load) ahead of that 400.+ workflows = status.get("workflows") or []+ if request.workflow and workflows and request.workflow not in workflows:+ return pup.EnhanceOutcome(status = None)+ if not pup.resolve_enhance(family, request.enhance_prompt):+ return pup.EnhanceOutcome(status = pup.STATUS_OFF)+ if account_access.managed_account():+ # The run below refuses a resident model this account may not use; refuse before the sidecar works for it.+ await asyncio.to_thread(account_access.require_media_generation_access, status, "diffusion")+ references = ([request.init_image] if request.init_image else []) + list(+ request.reference_images or []+ )+ # The sidecar takes GPU memory of its own: never beside another account's chat, image or video run.+ try:+ account_access.require_idle_other_accounts()+ except (HTTPException, account_access.GpuBusyForAnotherAccountError):+ return pup.EnhanceOutcome(status = pup.STATUS_UNAVAILABLE)+ try:+ with account_access.media_generation("diffusion"):+ outcome = await asyncio.to_thread(+ pup.enhance_for_generation,+ list(request.prompts or [request.prompt]),+ family_name = family,+ enhance_prompt = request.enhance_prompt,+ reference_images = references or None,+ )+ except pup.UpsamplerCancelledError:+ raise HTTPException(status_code = 409, detail = DIFFUSION_CANCELLED_MSG)+ if outcome.status not in (None, pup.STATUS_OFF):+ # Every attempted enhancement, a fallback too: a model swapped in meanwhile retries with its own default.+ from core.inference.diffusion_families import load_identity+ outcome.load = load_identity(status.get("repo_id"), status.get("base_repo"), family)+ return outcome++@studio_router.post("/images/generate", response_model = DiffusionGenerateResponse)@account_access.gpu_busy_routeasync def generate_diffusion_image(@@ -50590,6 +50647,8 @@ async def generate_diffusion_image(if request.localized_edit is not Noneelse None)+ # FLUX.3 prompt upsampling runs ahead of the run, so the sidecar can leave the VRAM before the DiT denoises.+ enhancement = await _enhance_generate_prompts(backend, request)result = Nonefor attempt in range(2):expected_load = None@@ -50602,6 +50661,9 @@ async def generate_diffusion_image(expected_load = load_identity(status.get("repo_id"), status.get("base_repo"), status.get("family"))+ if enhancement.load is not None:+ # Captions are written for one family: a model swapped in since then retries with fresh ones.+ expected_load = enhancement.load# Ahead of the run: milestones key off the previous poll, so a resumed range logs nothing.reset_media_generation_progress("image")try:@@ -50609,7 +50671,7 @@ async def generate_diffusion_image(result = await asyncio.to_thread(backend.generate,expected_load = expected_load,- prompt = request.prompt,+ prompt = enhancement.encoded(request.prompt),negative_prompt = request.negative_prompt,width = None if size_omitted else request.width,height = None if size_omitted else request.height,@@ -50617,7 +50679,11 @@ async def generate_diffusion_image(guidance = request.guidance,seed = request.seed,batch_size = request.batch_size,- prompts = request.prompts,+ prompts = (+ [enhancement.encoded(p) for p in request.prompts]+ if request.prompts+ else request.prompts+ ),seeds = request.seeds,init_image = request.init_image,mask_image = request.mask_image,@@ -50660,6 +50726,8 @@ async def generate_diffusion_image(except DiffusionModelReplacedError as exc:if attempt > 0:raise HTTPException(status_code = 409, detail = str(exc))+ # The caption was written for the replaced model's family.+ enhancement = await _enhance_generate_prompts(backend, request)continueexcept RuntimeError as exc:# Match these two EXACT client-state messages (409); other RuntimeErrors are failures.@@ -50686,16 +50754,21 @@ async def generate_diffusion_image(if per_image_seeds and index < len(per_image_seeds)else result["seed"])+ # A prompts-list batch records each image's OWN prompt so its recipe replays exactly.+ typed_prompt = (+ request.prompts[index]+ if request.prompts and index < len(request.prompts)+ else request.prompt+ )records.append(image_gallery.save(image,{- # A prompts-list batch records each image's OWN prompt so its recipe replays exactly.- "prompt": (- request.prompts[index]- if request.prompts and index < len(request.prompts)- else request.prompt- ),+ "prompt": typed_prompt,+ # The caption actually encoded (FLUX.3 prompt upsampling); replaying it with+ # enhance_prompt=false reproduces the image.+ "enhanced_prompt": enhancement.caption_for(typed_prompt),+ "prompt_enhancement": enhancement.status_for(typed_prompt),"negative_prompt": result.get("negative_prompt"),# Persist the ACTUAL output size, not the request sliders: the conditioned workflows derive it from the upload."width": getattr(image, "width", None) or request.width,@@ -51451,6 +51524,12 @@ async def unload_diffusion_model(if account_access.account_scope() is not None:unload = functools.partial(unload, expected_account = current_account_id())status_dict = await asyncio.to_thread(unload)+ # The FLUX.3 prompt upsampler serves the image model: it leaves with it instead of holding VRAM to its keep-alive.+ from core.inference import flux3_prompt_upsampler as pup++ # Blocking: DIFFUSION ownership is dropped below, so a sidecar still starting or answering would sit beside the+ # next owner without an eviction to stop it.+ await asyncio.to_thread(pup.stop_for_handover)# Drop DIFFUSION ownership only if nothing is resident AND no load is in flight, or a later chat load skips eviction and# OOMs the new pipeline. An in-flight load reads is_loaded False, so gate on loading_repo_ids() and use release_if.engine = get_active_diffusion_engine()@@ -51556,6 +51635,16 @@ async def diffusion_generate_progress(current_subject: str = Depends(get_currentprogress = get_active_diffusion_engine().generate_progress()log_media_generation_progress("image", progress)+ if not progress["active"]:+ from core.inference.flux3_prompt_upsampler import progress as upsampler_progress+ enhancing = upsampler_progress()+ if enhancing is not None:+ progress = {+ **progress,+ "active": True,+ "phase": "enhance",+ "message": enhancing.get("message"),+ }if external and not progress["active"]:progress = {**progress, "active": True, "target": "connection"}# A finished generation still persisting its gallery record counts as active, so a reload probe keeps polling.@@ -51597,13 +51686,116 @@ async def cancel_diffusion_generation():return {"cancelled": False}+ # A run still upsampling its prompt has not reached the engine yet.+ from core.inference.flux3_prompt_upsampler import cancel_active as cancel_upsampling++ # Both: another tab's request may be enhancing while this one denoises.+ upsampling = cancel_upsampling()# The slot can change hands before the executor callback, so the engine rechecks under the lock.expected = account_access.tracked_generation_account()cancel = get_active_diffusion_engine().cancel_generateif expected is not None:cancel = functools.partial(cancel, expected_account = expected)cancelled = await asyncio.get_running_loop().run_in_executor(_CANCEL_EXECUTOR, cancel)- return {"cancelled": cancelled}+ return {"cancelled": bool(cancelled or upsampling)}+++# ── FLUX.3 prompt upsampler (llama.cpp sidecar) ──────────────────────────────────────────────────────────+++@studio_router.get("/images/prompt-upsampler/status")+async def prompt_upsampler_status(current_subject: str = Depends(get_current_subject)):+ """Whether the FLUX.3 prompt upsampler can run (llama-server present), is downloaded, is loaded, and the+ download progress (the dictation download's shape)."""+ from core.inference import flux3_prompt_upsampler+ return JSONResponse(content = await asyncio.to_thread(flux3_prompt_upsampler.status))+++@studio_router.post("/images/prompt-upsampler/download")+async def prompt_upsampler_download(+ payload: Optional[PromptUpsamplerDownloadRequest] = None,+ current_subject: str = Depends(get_current_subject),+ hf_token: HfTokenArg = Depends(get_request_hf_token),+):+ """Start the background download of the upsampler GGUF and its vision projector; progress on /status."""+ from core.inference import flux3_prompt_upsampler+ from core.inference.stt_sidecar import SttDownloadCacheError, SttModelIdError++ account_access.require_installation_owner()+ if flux3_prompt_upsampler.status().get("override"):+ raise HTTPException(+ status_code = 409,+ detail = "The prompt upsampler files come from UNSLOTH_FLUX3_UPSAMPLER_GGUF / _MMPROJ; set both instead.",+ )+ token = account_access.account_hf_token(+ (payload.hf_token if payload is not None and payload.hf_token else None) or hf_token+ )+ try:+ download_id = await asyncio.to_thread(flux3_prompt_upsampler.start_download, token)+ except (SttModelIdError, SttDownloadCacheError) as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ return JSONResponse(+ content = {**flux3_prompt_upsampler.download_status(), "download_id": download_id}+ )+++@studio_router.post("/images/prompt-upsampler/cancel")+async def prompt_upsampler_cancel(current_subject: str = Depends(get_current_subject)):+ """Stop the upsampler download; partial files stay cached, so the next download resumes."""+ from core.inference import flux3_prompt_upsampler++ account_access.require_installation_owner()+ cancelled = await asyncio.to_thread(flux3_prompt_upsampler.cancel_download)+ return JSONResponse(+ content = {**flux3_prompt_upsampler.download_status(), "cancelled": cancelled}+ )+++@studio_router.post(+ "/images/prompt-upsampler/preview", response_model = PromptUpsamplerPreviewResponse+)+@account_access.gpu_busy_route+async def prompt_upsampler_preview(+ request: PromptUpsamplerPreviewRequest, current_subject: str = Depends(get_current_subject)+):+ """The caption the upsampler writes for ``prompt`` (and references), so the page can show or edit it before+ generating. 409 when not downloaded, 501 without llama-server."""+ from core.inference import flux3_prompt_upsampler as pup++ variant = request.variant+ if variant is None:+ try:+ from core.inference.diffusion_engine_router import get_active_diffusion_engine+ status = await asyncio.to_thread(get_active_diffusion_engine().status)+ variant = pup.variant_for_family(status.get("family") if status.get("loaded") else None)+ except Exception: # noqa: BLE001 -- nothing loaded reads as distilled+ variant = "distilled"+ account_access.require_idle_other_accounts()+ try:+ with account_access.media_generation("diffusion"):+ caption = await asyncio.to_thread(+ pup.upsample,+ request.prompt,+ variant = variant,+ reference_images = request.reference_images or None,+ )+ except pup.UpsamplerNotDownloadedError as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ except pup.UpsamplerUnavailableError as exc:+ raise HTTPException(status_code = 501, detail = str(exc))+ except pup.UpsamplerCancelledError as exc:+ raise HTTPException(status_code = 409, detail = str(exc))+ except ValueError as exc:+ raise HTTPException(status_code = 400, detail = str(exc))+ except Exception as exc: # noqa: BLE001+ logger.error("flux3.prompt_upsampler.preview_failed: %s", exc, exc_info = True)+ raise HTTPException(status_code = 500, detail = "The prompt upsampler failed.")+ finally:+ try:+ await asyncio.to_thread(pup.get_sidecar().release_if_tight)+ except Exception: # noqa: BLE001 -- the keep-alive reclaims it anyway+ pass+ return PromptUpsamplerPreviewResponse(enhanced_prompt = caption)# ──────────────────────────────────────────────────────────────────────────diff --git a/studio/backend/routes/training_vram.py b/studio/backend/routes/training_vram.pyindex 9b0d532cd41..90f08dbdd2b 100644--- a/studio/backend/routes/training_vram.py+++ b/studio/backend/routes/training_vram.py@@ -5,6 +5,7 @@models when they fit. STT is evicted before chat when training needs memory."""+import sysfrom typing import Any, Callable, Dict, List, Optional, Tuplefrom hub.utils.hf_tokens import HfTokenArg, normalize_token@@ -668,10 +669,35 @@ def free_stt_model_for_training(reason: str) -> List[str]:return freed+def free_prompt_upsampler_for_training(reason: str) -> List[str]:+ """Stop the FLUX.3 prompt upsampler's llama-server, only if Images ever started it. Never raises."""+ upsampler_module = sys.modules.get("core.inference.flux3_prompt_upsampler")+ if upsampler_module is None:+ return []+ try:+ # A run still decoding its references has not reached the sidecar: stopped here, it never starts one.+ upsampler_module.cancel_active()+ upsampler = upsampler_module.get_sidecar()+ # Loaded, not only on the GPU: a -ngl 0 server still holds ~6 GB of host RAM and the CPU through training.+ if upsampler.loaded or upsampler.is_loading():+ logger.info("Unloading the FLUX.3 prompt upsampler for training (%s)", reason)+ upsampler.unload()+ return ["image:prompt-upsampler"]+ except Exception as e:+ logger.warning("Could not unload the FLUX.3 prompt upsampler: %s", e)+ return []++def coordinate_models_for_training(can_keep: Callable[[], Tuple[bool, Dict[str, Any]]],) -> List[str]:"""Keep resident models when they fit, evicting STT before chat."""+ # The upsampler restarts on demand, so it never holds memory through training.+ freed = free_prompt_upsampler_for_training(reason = "training start")+ return freed + _coordinate_resident_models(can_keep)+++def _coordinate_resident_models(can_keep: Callable[[], Tuple[bool, Dict[str, Any]]]) -> List[str]:resident_chat = summarize_resident_chat()resident_stt = summarize_resident_stt()if not resident_chat["any"] and not resident_stt["any"]:diff --git a/studio/backend/tests/test_flux3_prompt_upsampler.py b/studio/backend/tests/test_flux3_prompt_upsampler.pynew file mode 100644index 00000000000..c139c5be465--- /dev/null+++ b/studio/backend/tests/test_flux3_prompt_upsampler.py@@ -0,0 +1,1202 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 prompt upsampling: request building, reference preparation, family defaults, fallbacks, the gallery+record and the routes. The llama-server is mocked; no GPU and no network."""++from __future__ import annotations++import base64+import hashlib+import io+import threading+import time++import pytest++from core.inference import flux3_prompt_upsampler as pup++[email protected](autouse = True)+def _clean(monkeypatch):+ pup.clear_cache()+ monkeypatch.delenv(pup.ENV_MODEL_OVERRIDE, raising = False)+ monkeypatch.delenv(pup.ENV_MMPROJ_OVERRIDE, raising = False)+ yield+ pup.clear_cache()+++# ── prompts and sampling ────────────────────────────────────────────────────────────────────────────────────────────+# sha256 of BFL's system_prompts.json entries (black-forest-labs/flux-3-image-klein-pup @ 8b069751), equal to+# diffusers SYSTEM_PROMPTS at d42ebf9c6e.+_T2I_SHA = "8b235a421bce2213d11ada01f5a1244443dd72303ffbb4cc9e5a0d8b0de5ab6f"+_I2I_SHA = "95cec384d5b6d341caa9fb3f2c7ca45ee80c2364b3a95d4044aebf976f6dd3bf"+++def test_system_prompts_are_bfls_verbatim():+ assert hashlib.sha256(pup.SYSTEM_PROMPTS["t2i"].encode()).hexdigest() == _T2I_SHA+ assert hashlib.sha256(pup.SYSTEM_PROMPTS["i2i"].encode()).hexdigest() == _I2I_SHA+++ "variant, repeat, presence",+ [("distilled", 1.05, 1.5), ("base", 1.0, 1.5)],+)+def test_t2i_payload(variant, repeat, presence):+ body = pup.build_chat_payload("a gymnast doing a backflip", variant)+ assert body["messages"] == [+ {"role": "system", "content": pup.SYSTEM_PROMPTS["t2i"]},+ {"role": "user", "content": [{"type": "text", "text": "a gymnast doing a backflip"}]},+ ]+ assert body["top_k"] == 1 and body["temperature"] == 1.0 and body["top_p"] == 0.95+ assert body["min_p"] == 0.0 and body["seed"] == 0+ assert body["repeat_penalty"] == repeat and body["presence_penalty"] == presence+ assert body["max_tokens"] == 3584+ assert body["chat_template_kwargs"] == {"enable_thinking": False}++[email protected]("variant, repeat", [("distilled", 1.05), ("base", 1.0)])+def test_edit_payload_images_then_scaffold(variant, repeat):+ refs = [b"jpeg-one", b"jpeg-two"]+ body = pup.build_chat_payload("make it red", variant, refs)+ system, user = body["messages"]+ assert system == {"role": "system", "content": pup.SYSTEM_PROMPTS["i2i"]}+ parts = user["content"]+ assert [p["type"] for p in parts] == ["image_url", "image_url", "text"]+ assert (+ parts[0]["image_url"]["url"]+ == "data:image/jpeg;base64," + base64.b64encode(b"jpeg-one").decode()+ )+ assert parts[2]["text"] == (+ "Reference images follow in order (the first image, the second image, ...).\n"+ "User request: make it red\n"+ "Output the caption."+ )+ assert body["repeat_penalty"] == repeat and body["presence_penalty"] == 0.0+++def test_edit_without_images_uses_the_edit_prompt_text_only():+ body = pup.build_chat_payload("make it red", "distilled", [], edit = True)+ system, user = body["messages"]+ assert system["content"] == pup.SYSTEM_PROMPTS["i2i"]+ assert [p["type"] for p in user["content"]] == ["text"]+ assert "User request: make it red" in user["content"][0]["text"]+++def test_unknown_variant_is_refused():+ with pytest.raises(ValueError):+ pup.build_chat_payload("x", "turbo")+++def test_clean_caption():+ assert pup.clean_caption("<think>\n\n</think>\n\n A caption. ") == "A caption."+ assert pup.clean_caption('"quoted"') == "quoted"+ assert pup.clean_caption(None) == ""+++# ── reference preparation ───────────────────────────────────────────────────────────────────────────────────────────+ "size, count, expected",+ [+ ((1024, 1024), 1, (1024, 1024)),+ ((4000, 3000), 1, (1152, 864)),+ ((4000, 3000), 2, (832, 608)),+ ((4000, 3000), 3, (576, 416)),+ ((4000, 3000), 10, (576, 416)),+ ((100, 50), 1, (96, 64)),+ ((1000, 31), 1, (992, 32)),+ ],+)+def test_reference_size(size, count, expected):+ w, h = pup.get_reference_size(*size, count)+ assert (w, h) == expected+ assert w % 32 == 0 and h % 32 == 0+ assert w * h <= pup.MAX_PIXELS_BY_REFS[count]+++def test_reference_size_refuses_extreme_aspect():+ with pytest.raises(ValueError):+ pup.get_reference_size(10000, 10, 1)+++def test_prepare_reference_rgba_over_white_and_jpeg():+ from PIL import Image++ image = Image.new("RGBA", (300, 200), (0, 0, 0, 0))+ out = Image.open(io.BytesIO(pup.prepare_reference(image, 1)))+ assert out.format == "JPEG" and out.mode == "RGB"+ assert out.size == (288, 192)+ r, g, b = out.getpixel((10, 10))+ assert min(r, g, b) > 245 # transparent became white, not black+++def test_prepare_references_caps_at_ten_and_decodes_data_urls():+ from PIL import Image++ buf = io.BytesIO()+ Image.new("RGB", (800, 600), (10, 20, 30)).save(buf, format = "PNG")+ url = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()+ refs = pup.prepare_references([url] * 12)+ assert len(refs) == 10+ assert Image.open(io.BytesIO(refs[0])).size == pup.get_reference_size(800, 600, 10)+++# ── family defaults ─────────────────────────────────────────────────────────────────────────────────────────────────+ "family, flag, expected",+ [+ ("flux.3-klein", None, True),+ ("flux.3-klein-base", None, True),+ ("flux.3-klein", False, False),+ ("flux.3-klein-base", True, True),+ ("flux.2-klein", None, False),+ ("flux.2-klein", True, False),+ ("z-image", True, False),+ (None, True, False),+ ],+)+def test_resolve_enhance(family, flag, expected):+ assert pup.resolve_enhance(family, flag) is expected+++def test_variant_and_capabilities():+ assert pup.variant_for_family("flux.3-klein") == "distilled"+ assert pup.variant_for_family("flux.3-klein-base") == "base"+ assert pup.family_capabilities("flux.3-klein-base") == {+ "supports_prompt_upsampler": True,+ "prompt_upsampler_default": True,+ }+ assert pup.family_capabilities("qwen-image") == {+ "supports_prompt_upsampler": False,+ "prompt_upsampler_default": False,+ }+++def test_status_annotation_carries_the_capabilities(monkeypatch):+ from core.inference import diffusion_engine_router as router+ from models.inference import DiffusionStatusResponse++ monkeypatch.setattr(router, "_supported_family_capabilities", lambda: ("flux.3-klein",))+ out = router.annotate_status({"loaded": True, "repo_id": "x", "family": "flux.3-klein"})+ assert out["supports_prompt_upsampler"] is True and out["prompt_upsampler_default"] is True+ assert DiffusionStatusResponse(**out).supports_prompt_upsampler is True+ out = router.annotate_status({"loaded": False, "repo_id": None, "family": None})+ assert out["supports_prompt_upsampler"] is False+++# ── memory planning ─────────────────────────────────────────────────────────────────────────────────────────────────+GIB = 1024**3+++def test_plan_gpu_layers():+ model = int(5.24 * GIB)+ assert pup.plan_gpu_layers(40_000, model, 900 * 1024 * 1024) == 99+ assert 4 <= pup.plan_gpu_layers(6_000, model) < 32+ assert pup.plan_gpu_layers(2_000, model) == 0+ assert pup.plan_gpu_layers(40_000, model, training = True) == 0+ assert pup.plan_gpu_layers(None, model, device = "cpu") == 0+ assert pup.plan_gpu_layers(None, model) == 99+ assert pup.plan_gpu_layers(1_000, model, device = "mps") == 99+++def test_vram_is_tight():+ assert pup.vram_is_tight(4_000, 24_000)+ assert not pup.vram_is_tight(20_000, 24_000)+ assert pup.vram_is_tight(15_000, 80_000) # under 20% of the card+ assert not pup.vram_is_tight(None, None)+++# ── paths and fallbacks ─────────────────────────────────────────────────────────────────────────────────────────────+def test_env_override_paths(monkeypatch, tmp_path):+ model = tmp_path / "pup.gguf"+ model.write_bytes(b"x" * 10)+ monkeypatch.setenv(pup.ENV_MODEL_OVERRIDE, str(model))+ monkeypatch.setenv(pup.ENV_MMPROJ_OVERRIDE, str(tmp_path / "missing.gguf"))+ monkeypatch.setattr(pup, "_cached_at_any_revision", lambda *a, **k: "/cache/should-not-be-used")+ assert pup.model_path() == str(model)+ # A set but missing override is a typo, not a reason to read the cache.+ assert pup.mmproj_path() is None+++class _FakeSidecar:+ def __init__(+ self,+ reply = "A long caption.",+ error = None,+ ):+ self.reply = reply+ self.error = error+ self.calls: list[dict] = []++ def generate(+ self,+ payload,+ *,+ binary,+ model,+ mmproj,+ cancel_event = None,+ ):+ self.calls.append({"payload": payload, "model": model, "mmproj": mmproj})+ if self.error is not None:+ raise self.error+ return self.reply(payload) if callable(self.reply) else self.reply++ def release_if_tight(self):+ return False+++def runnable(monkeypatch):+ sidecar = _FakeSidecar(+ reply = lambda payload: "Caption for " + payload["messages"][1]["content"][-1]["text"]+ )+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")+ monkeypatch.setattr(pup, "mmproj_path", lambda: "/m/mmproj.gguf")+ monkeypatch.setattr(pup, "get_sidecar", lambda: sidecar)+ return sidecar+++def test_one_upsample_per_distinct_prompt_and_cache(runnable):+ out = pup.enhance_for_generation(+ ["a cat", "a dog", "a cat"], family_name = "flux.3-klein", enhance_prompt = None+ )+ assert out.status == "applied"+ assert len(runnable.calls) == 2+ assert out.encoded("a cat") == "Caption for a cat"+ assert out.status_for("a dog") == "applied"+ # Same prompt + variant + refs: the cache answers, so a re-run encodes the same caption.+ again = pup.enhance_for_generation(["a cat"], family_name = "flux.3-klein", enhance_prompt = True)+ assert len(runnable.calls) == 2 and again.encoded("a cat") == "Caption for a cat"+ # A different variant is a different caption request.+ pup.enhance_for_generation(["a cat"], family_name = "flux.3-klein-base", enhance_prompt = True)+ assert len(runnable.calls) == 3+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.0+++def test_off_and_other_families_never_call(runnable):+ assert (+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = False).status+ == "off"+ )+ other = pup.enhance_for_generation(["a"], family_name = "flux.2-klein", enhance_prompt = True)+ assert other.status is None and other.encoded("a") == "a"+ assert runnable.calls == []+++def test_unavailable_without_weights_or_binary(monkeypatch):+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")+ monkeypatch.setattr(pup, "model_path", lambda: None)+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)+ assert out.status == "unavailable" and out.encoded("a") == "a" and out.caption_for("a") is None+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)+ monkeypatch.setattr(pup, "model_path", lambda: "/m/pup.gguf")+ assert (+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None).status+ == "unavailable"+ )+++def test_server_failure_falls_back_to_the_typed_prompt(runnable):+ runnable.error = RuntimeError("boom")+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)+ assert out.status == "failed" and out.encoded("a") == "a" and out.caption_for("a") is None+++def test_empty_caption_falls_back(runnable):+ runnable.reply = " "+ assert pup.upsample("a", variant = "distilled") == "a"+ out = pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)+ assert out.status == "failed" and out.encoded("a") == "a"+++def test_a_stopped_enhancement_still_releases_a_tight_server(runnable, monkeypatch):+ released = []+ monkeypatch.setattr(runnable, "release_if_tight", lambda: released.append(True) or True)+ runnable.error = pup.UpsamplerCancelledError("Cancelled.")+ with pytest.raises(pup.UpsamplerCancelledError):+ pup.enhance_for_generation(["a"], family_name = "flux.3-klein", enhance_prompt = None)+ assert released == [True]+++def test_cancel_propagates(runnable):+ event = threading.Event()+ event.set()+ with pytest.raises(pup.UpsamplerCancelledError):+ pup.enhance_for_generation(+ ["a"], family_name = "flux.3-klein", enhance_prompt = None, cancel_event = event+ )+ assert pup.progress() is None+++def _png_url(size = (640, 480)):+ from PIL import Image++ buf = io.BytesIO()+ Image.new("RGB", size, (200, 10, 10)).save(buf, format = "PNG")+ return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()+++def test_edit_sends_references_with_mmproj(runnable):+ caption = pup.upsample("make it blue", variant = "distilled", reference_images = [_png_url()])+ call = runnable.calls[-1]+ assert call["mmproj"] == "/m/mmproj.gguf"+ parts = call["payload"]["messages"][1]["content"]+ assert [p["type"] for p in parts] == ["image_url", "text"]+ assert caption.startswith("Caption for Reference images follow in order")+++def test_edit_without_mmproj_runs_text_only(runnable, monkeypatch):+ monkeypatch.setattr(pup, "mmproj_path", lambda: None)+ pup.upsample("make it blue", variant = "distilled", reference_images = [_png_url()])+ call = runnable.calls[-1]+ assert call["mmproj"] is None+ system, user = call["payload"]["messages"]+ assert system["content"] == pup.SYSTEM_PROMPTS["i2i"]+ assert [p["type"] for p in user["content"]] == ["text"]+++def test_references_change_the_cache_key(runnable):+ pup.upsample("edit", reference_images = [_png_url((640, 480))])+ pup.upsample("edit", reference_images = [_png_url((640, 480))])+ pup.upsample("edit", reference_images = [_png_url((480, 640))])+ assert len(runnable.calls) == 2+++def test_upsample_raises_when_not_downloaded(monkeypatch):+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")+ monkeypatch.setattr(pup, "model_path", lambda: None)+ with pytest.raises(pup.UpsamplerNotDownloadedError):+ pup.upsample("a")+++# ── the server request (mocked HTTP) ────────────────────────────────────────────────────────────────────────────────+def test_post_chat_parses_the_reply(monkeypatch):+ import json++ sent = {}++ class _Response:+ status = 200++ def read(self):+ return json.dumps({"choices": [{"message": {"content": "The caption."}}]}).encode()++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ class _Connection:+ def __init__(+ self,+ host,+ port,+ timeout = None,+ ):+ sent["target"] = (host, port)+ self.sock = None++ def request(+ self,+ method,+ path,+ body = None,+ headers = None,+ ):+ sent.update(method = method, path = path, body = json.loads(body), headers = headers)++ def getresponse(self):+ return _Response()++ def close(self):+ pass++ monkeypatch.setattr(pup.http.client, "HTTPConnection", _Connection)+ payload = pup.build_chat_payload("a", "distilled")+ assert pup._post_chat(4321, payload, api_key = "k") == "The caption."+ assert sent["target"] == ("127.0.0.1", 4321)+ assert sent["path"] == "/v1/chat/completions" and sent["body"] == payload+ assert sent["headers"]["Authorization"] == "Bearer k"+++ "status, props, expected",+ [+ (200, {"default_generation_settings": {"params": {"penalize_prompt": True}}}, True),+ (200, {"default_generation_settings": {"penalize_prompt": True}}, True),+ (200, {"default_generation_settings": {"params": {"presence_penalty": 0.0}}}, False),+ (200, {}, False),+ (404, {}, False),+ (200, None, False),+ ],+)+def test_penalize_prompt_probe(monkeypatch, status, props, expected):+ import json++ class _Response:+ def __init__(self):+ self.status = status++ def read(self):+ return b"not json" if props is None else json.dumps(props).encode()++ class _Connection:+ def __init__(+ self,+ host,+ port,+ timeout = None,+ ):+ pass++ def request(+ self,+ method,+ path,+ headers = None,+ ):+ assert (method, path) == ("GET", "/props")++ def getresponse(self):+ return _Response()++ def close(self):+ pass++ monkeypatch.setattr(pup.http.client, "HTTPConnection", _Connection)+ assert pup._server_has_penalize_prompt(4321) is expected++[email protected]("supported", [True, False])+def test_generated_only_penalties_follow_the_server(monkeypatch, supported):+ sent = {}+ sidecar = pup.PromptUpsamplerSidecar()++ def _fake_launch(binary, model, mmproj):+ sidecar._process = type("P", (), {"poll": lambda self: None})()+ sidecar._port, sidecar._key = 4321, (binary, model, mmproj)+ sidecar._penalize_prompt_supported = supported++ monkeypatch.setattr(sidecar, "_launch", _fake_launch)+ monkeypatch.setattr(sidecar, "_schedule_idle_locked", lambda: None)+ monkeypatch.setattr(+ pup, "_post_chat", lambda port, payload, **kw: sent.update(payload) or "caption"+ )+ payload = pup.build_chat_payload("a", "base")+ assert sidecar.generate(payload, binary = "b", model = "m", mmproj = None) == "caption"+ if supported:+ assert sent["penalize_prompt"] is False and sent["repeat_last_n"] == 4096+ else:+ assert "penalize_prompt" not in sent and "repeat_last_n" not in sent+ assert sent["presence_penalty"] == 1.5 and "penalize_prompt" not in payload+++# ── the download state reuses the dictation worker ──────────────────────────────────────────────────────────────────+def test_download_state_targets_the_unsloth_repo(monkeypatch):+ state = pup._get_download_state()+ assert state._spec(pup.UPSAMPLER_ID).repo == "unsloth/flux-3-image-klein-pup-GGUF"+ assert state._spec(pup.UPSAMPLER_ID).model_file == "flux-3-image-klein-pup-Q4_K_M.gguf"+ assert state._spec(pup.UPSAMPLER_ID).mmproj_file == "mmproj-F16.gguf"+ with pytest.raises(ValueError):+ state._resolve("black-forest-labs/flux-3-image-klein-pup")+ snapshot = state.status()+ assert {"downloading", "download_id", "error", "cancelled", "bytes_total", "bytes_done"} <= set(+ snapshot+ )+++def test_status_shape(monkeypatch):+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)+ monkeypatch.setattr(pup, "model_path", lambda: None)+ monkeypatch.setattr(pup, "mmproj_path", lambda: None)+ out = pup.status()+ assert out["supported"] is False and out["downloaded"] is False and out["loaded"] is False+ assert out["mmproj_downloaded"] is False+ assert out["repo"] == pup.UPSAMPLER_REPO and out["filename"] == pup.UPSAMPLER_MODEL_FILE+ assert out["size_bytes"] == pup.UPSAMPLER_FILE_SIZES[pup.UPSAMPLER_MODEL_FILE]+ assert isinstance(out["download"], dict)+++# ── gallery record ──────────────────────────────────────────────────────────────────────────────────────────────────+def test_gallery_record_round_trip(monkeypatch, tmp_path):+ from PIL import Image++ from core.inference import image_gallery+ from models.inference import GalleryImage++ monkeypatch.setattr(image_gallery, "gallery_dir", lambda: tmp_path)+ meta = {+ "prompt": "a gymnast doing a backflip",+ "enhanced_prompt": "Full-color indoor photograph of a gymnast ...",+ "prompt_enhancement": "applied",+ "width": 64,+ "height": 64,+ "steps": 4,+ "guidance": 1.0,+ "seed": 7,+ "created_at": 1.0,+ }+ record = image_gallery.save(Image.new("RGB", (64, 64)), meta)+ listed = [r for r in image_gallery.list_images(10, 0) if r["id"] == record["id"]][0]+ image = GalleryImage(**listed)+ assert image.prompt == "a gymnast doing a backflip"+ assert image.enhanced_prompt == meta["enhanced_prompt"]+ assert image.prompt_enhancement == "applied"+ # Records written before this read as unset.+ old = {k: v for k, v in listed.items() if k not in ("enhanced_prompt", "prompt_enhancement")}+ image = GalleryImage(**old)+ assert image.enhanced_prompt is None and image.prompt_enhancement is None+++# ── routes ──────────────────────────────────────────────────────────────────────────────────────────────────────────+class _FluxBackend:+ def __init__(self, family = "flux.3-klein"):+ self.family = family+ self.calls: list[dict] = []++ def status(self):+ return {"loaded": True, "repo_id": "unsloth/flux", "family": self.family}++ def generate(+ self,+ *,+ seed = None,+ prompts = None,+ batch_size = 1,+ **kwargs,+ ):+ self.calls.append({"prompts": prompts, "seed": seed, **kwargs})+ count = len(prompts) if prompts else batch_size+ return {"images": [object()] * count, "seed": 5, "seeds": [5 + i for i in range(count)]}++ def generate_progress(self):+ return {"active": False, "step": 0, "total_steps": 0, "fraction": 0.0, "eta_seconds": None}++ def cancel_generate(self, expected_account = None):+ return False+++def routes(monkeypatch):+ from fastapi import FastAPI+ from fastapi.testclient import TestClient++ import core.inference.diffusion_engine_router as engine_router+ import core.inference.image_gallery as gallery_module+ from auth.authentication import authenticated_via_api_key, get_current_subject+ from routes.inference import studio_router++ backend = _FluxBackend()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: backend)+ saved: list[dict] = []++ def _save(image, meta):+ record = {**meta, "id": f"img{len(saved)}", "url": "/x"}+ saved.append(record)+ return record++ monkeypatch.setattr(gallery_module, "save", _save)+ app = FastAPI()+ app.include_router(studio_router, prefix = "/api/inference")+ app.dependency_overrides[get_current_subject] = lambda: "test-user"+ app.dependency_overrides[authenticated_via_api_key] = lambda: False+ return TestClient(app), backend, saved+++def test_generate_encodes_the_caption_and_records_both(routes, runnable):+ client, backend, saved = routes+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ assert backend.calls[-1]["prompt"] == "Caption for a cat"+ image = r.json()["images"][0]+ assert image["prompt"] == "a cat"+ assert image["enhanced_prompt"] == "Caption for a cat"+ assert image["prompt_enhancement"] == "applied"+++def test_generate_restore_with_enhance_off_is_untouched(routes, runnable):+ client, backend, saved = routes+ r = client.post(+ "/api/inference/images/generate",+ json = {"prompt": "Caption for a cat", "enhance_prompt": False, "steps": 4},+ )+ assert r.status_code == 200, r.text+ assert backend.calls[-1]["prompt"] == "Caption for a cat"+ assert runnable.calls == []+ image = r.json()["images"][0]+ assert image["enhanced_prompt"] is None and image["prompt_enhancement"] == "off"+++def test_generate_prompts_list_one_upsample_each(routes, runnable):+ client, backend, saved = routes+ r = client.post(+ "/api/inference/images/generate",+ json = {"prompt": "a", "prompts": ["a", "b", "a"], "steps": 4},+ )+ assert r.status_code == 200, r.text+ assert backend.calls[-1]["prompts"] == ["Caption for a", "Caption for b", "Caption for a"]+ assert len(runnable.calls) == 2+ assert [i["enhanced_prompt"] for i in r.json()["images"]] == [+ "Caption for a",+ "Caption for b",+ "Caption for a",+ ]+++def test_generate_other_family_passes_through(routes, runnable):+ client, backend, saved = routes+ backend.family = "flux.2-klein"+ r = client.post(+ "/api/inference/images/generate",+ json = {"prompt": "a cat", "enhance_prompt": True, "steps": 4},+ )+ assert r.status_code == 200, r.text+ assert backend.calls[-1]["prompt"] == "a cat" and runnable.calls == []+ image = r.json()["images"][0]+ assert image["enhanced_prompt"] is None and image["prompt_enhancement"] is None+++def test_generate_skips_the_upsampler_for_a_workflow_the_family_refuses(+ routes, runnable, monkeypatch+):+ client, backend, saved = routes+ monkeypatch.setattr(+ backend,+ "status",+ lambda: {**_FluxBackend.status(backend), "workflows": ["txt2img", "reference"]},+ )+ client.post(+ "/api/inference/images/generate",+ json = {+ "prompt": "make it night time",+ "workflow": "edit",+ "init_image": "data:,",+ "steps": 4,+ },+ )+ assert runnable.calls == []+ client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert len(runnable.calls) == 1+++def test_unload_sidecar_never_creates_one(monkeypatch):+ monkeypatch.setattr(pup, "_sidecar", None)+ pup.unload_sidecar()+ assert pup._sidecar is None+ released = []+ sidecar = pup.PromptUpsamplerSidecar()+ monkeypatch.setattr(sidecar, "unload", lambda wait = True: released.append(wait))+ monkeypatch.setattr(pup, "_sidecar", sidecar)+ pup.unload_sidecar()+ assert released == [False]+++def test_generate_upsampler_failure_never_fails_the_image(routes, runnable):+ client, backend, saved = routes+ runnable.error = RuntimeError("server died")+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ assert backend.calls[-1]["prompt"] == "a cat"+ assert r.json()["images"][0]["prompt_enhancement"] == "failed"+++def test_generate_edit_sends_init_and_references(routes, runnable):+ client, backend, saved = routes+ r = client.post(+ "/api/inference/images/generate",+ json = {+ "prompt": "swap the hat",+ "init_image": _png_url(),+ "reference_images": [_png_url((320, 320))],+ "workflow": "edit",+ "steps": 4,+ },+ )+ assert r.status_code == 200, r.text+ parts = runnable.calls[-1]["payload"]["messages"][1]["content"]+ assert [p["type"] for p in parts] == ["image_url", "image_url", "text"]+++def test_upsampler_routes(routes, runnable, monkeypatch):+ client, backend, saved = routes+ monkeypatch.setattr(pup, "status", lambda: {"supported": True, "downloaded": True})+ assert client.get("/api/inference/images/prompt-upsampler/status").json() == {+ "supported": True,+ "downloaded": True,+ }+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a gymnast"})+ assert r.status_code == 200, r.text+ assert r.json() == {"enhanced_prompt": "Caption for a gymnast"}+ backend.family = "flux.3-klein-base"+ client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "b"})+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.0+ client.post(+ "/api/inference/images/prompt-upsampler/preview",+ json = {"prompt": "c", "variant": "distilled"},+ )+ assert runnable.calls[-1]["payload"]["repeat_penalty"] == 1.05++ started = {}+ monkeypatch.setattr(+ pup, "start_download", lambda token: started.setdefault("token", token) and "dl-1"+ )+ monkeypatch.setattr(pup, "download_status", lambda: {"downloading": True})+ r = client.post("/api/inference/images/prompt-upsampler/download", json = {"hf_token": "hf_x"})+ assert r.status_code == 200 and r.json()["download_id"] == "dl-1" and started["token"] == "hf_x"+ r = client.post("/api/inference/images/prompt-upsampler/download")+ assert r.status_code == 200+ monkeypatch.setattr(pup, "cancel_download", lambda: True)+ r = client.post("/api/inference/images/prompt-upsampler/cancel")+ assert r.status_code == 200 and r.json()["cancelled"] is True+++def test_preview_errors(routes, monkeypatch):+ client, backend, saved = routes+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: "/bin/llama-server")+ monkeypatch.setattr(pup, "model_path", lambda: None)+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a"})+ assert r.status_code == 409+ monkeypatch.setattr(pup, "find_llama_server_binary", lambda: None)+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a"})+ assert r.status_code == 501+++def test_images_status_route_carries_the_capabilities(routes, monkeypatch):+ client, backend, saved = routes+ from core.inference import diffusion_engine_router as engine_router++ monkeypatch.setattr(engine_router, "_supported_family_capabilities", lambda: ("flux.3-klein",))+ body = client.get("/api/inference/images/status").json()+ assert body["supports_prompt_upsampler"] is True and body["prompt_upsampler_default"] is True+ backend.family = "z-image"+ body = client.get("/api/inference/images/status").json()+ assert body["supports_prompt_upsampler"] is False and body["prompt_upsampler_default"] is False+++def test_download_status_keys_match_the_frontend():+ assert {"downloading", "bytes_total", "bytes_done", "error", "cancelled"} <= set(+ pup.download_status()+ )+++def test_progress_reports_the_enhance_phase(routes, monkeypatch):+ client, backend, saved = routes+ monkeypatch.setattr(+ pup, "progress", lambda: {"done": 0, "total": 1, "message": "Enhancing prompt..."}+ )+ body = client.get("/api/inference/images/generate-progress").json()+ assert body["active"] is True and body["phase"] == "enhance"+ assert body["message"] == "Enhancing prompt..."+ monkeypatch.setattr(pup, "cancel_active", lambda: True)+ stopped = []+ monkeypatch.setattr(+ backend, "cancel_generate", lambda **_: stopped.append(True) or False, raising = False+ )+ assert client.post("/api/inference/images/generate/cancel").json() == {"cancelled": True}+ # Another tab's denoise is stopped too, not only the enhancement.+ assert stopped == [True]+++def test_generate_retry_recomputes_the_caption_for_the_new_model(routes, runnable, monkeypatch):+ from core.inference.diffusion_families import DiffusionModelReplacedError, load_identity++ client, backend, saved = routes+ original = backend.generate++ def replaced_once(**kwargs):+ if not backend.calls:+ backend.calls.append({"replaced": True})+ backend.family = "flux.2-klein"+ raise DiffusionModelReplacedError(+ load_identity("unsloth/flux", None, "flux.3-klein"),+ load_identity("unsloth/flux", None, "flux.2-klein"),+ )+ return original(**kwargs)++ monkeypatch.setattr(backend, "generate", replaced_once)+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ # FLUX.2 has no upsampler: the retry encodes the typed prompt, not the FLUX.3 caption.+ assert backend.calls[-1]["prompt"] == "a cat"+ assert r.json()["images"][0]["prompt_enhancement"] is None+++def test_single_user_generate_binds_the_caption_to_its_model(routes, runnable):+ from core.inference.diffusion_families import load_identity++ client, backend, saved = routes+ status = backend.status()+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ # Bound on single-user installs too: a model swapped in after the caption raises and retries.+ assert backend.calls[-1]["expected_load"] == load_identity(+ status.get("repo_id"), status.get("base_repo"), "flux.3-klein"+ )+++def test_a_cached_caption_still_honors_a_stop(runnable):+ assert pup.upsample("a cat", variant = "distilled") == "Caption for a cat"+ event = threading.Event()+ event.set()+ with pytest.raises(pup.UpsamplerCancelledError):+ pup.upsample("a cat", variant = "distilled", cancel_event = event)+++def test_launch_publishes_loading_before_the_training_probe(monkeypatch):+ sidecar = pup.PromptUpsamplerSidecar()+ seen = []+ monkeypatch.setattr(sidecar, "_spawn_server", lambda *a: seen.append(sidecar.is_loading()))+ sidecar._launch("b", "m", None)+ assert seen == [True] and not sidecar.is_loading()+++def test_an_idle_callback_waiting_on_the_lock_sees_the_request(monkeypatch):+ sidecar = pup.PromptUpsamplerSidecar()+ reaped = []+ monkeypatch.setattr(pup, "_image_model_owns_gpu", lambda: True)+ monkeypatch.setattr(sidecar, "_release_locked", lambda: reaped.append(True))++ def launch(binary, model, mmproj):+ # The keep-alive timer fired just before this request: its callback runs while the server starts.+ sidecar._idle_unload(sidecar._generation)++ monkeypatch.setattr(sidecar, "_launch", launch)+ monkeypatch.setattr(pup, "_post_chat", lambda *a, **k: "caption")+ assert sidecar.generate({}, binary = "b", model = "m", mmproj = None) == "caption"+ assert reaped == [] and sidecar._active_requests == 0+++def test_unload_route_stops_the_upsampler_blocking(routes, monkeypatch):+ client, backend, saved = routes+ waits, cancels = [], []+ monkeypatch.setattr(pup, "unload_sidecar", lambda wait = False: waits.append(wait))+ monkeypatch.setattr(pup, "cancel_active", lambda: cancels.append(True))+ monkeypatch.setattr(backend, "unload", lambda **_: {"loaded": False}, raising = False)+ r = client.post("/api/inference/images/unload", json = {})+ assert r.status_code == 200, r.text+ assert waits == [True] and cancels == [True]+++def test_a_request_queued_before_a_blocking_unload_never_starts(monkeypatch):+ sidecar = pup.PromptUpsamplerSidecar()+ launched = []+ monkeypatch.setattr(pup, "_image_model_owns_gpu", lambda: True)+ monkeypatch.setattr(sidecar, "_launch", lambda *a: launched.append(a))+ monkeypatch.setattr(pup, "_post_chat", lambda *a, **k: "caption")+ sidecar._request_lock.acquire()+ errors = []++ def queued():+ try:+ sidecar.generate({}, binary = "b", model = "m", mmproj = None)+ except pup.UpsamplerUnavailableError as exc:+ errors.append(exc)++ worker = threading.Thread(target = queued)+ worker.start()+ time.sleep(0.2)+ # Training starts while that request waits on the lock: the drain finds nothing active and returns.+ sidecar.unload(wait = True)+ sidecar._request_lock.release()+ worker.join(5)+ assert errors and launched == []+ # A request after the handover is a new one.+ assert sidecar.generate({}, binary = "b", model = "m", mmproj = None) == "caption"+++def test_an_unowned_gpu_starts_the_upsampler_on_the_cpu(monkeypatch):+ from core.inference import gpu_arbiter++ monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: None)+ assert pup._gpu_unowned()+ monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: gpu_arbiter.DIFFUSION)+ assert not pup._gpu_unowned()+++def test_the_child_is_pinned_to_the_image_models_card(monkeypatch):+ import torch++ monkeypatch.setattr(torch.cuda, "device_count", lambda: 4)+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,5,GPU-abc")+ assert pup._child_device_mask(2) == {"CUDA_VISIBLE_DEVICES": "GPU-abc"}+ assert pup._child_device_mask(1) == {"CUDA_VISIBLE_DEVICES": "5"}+ assert pup._child_device_mask(None) == {}+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES")+ assert pup._child_device_mask(1) == {"CUDA_VISIBLE_DEVICES": "1"}+ monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ assert pup._child_device_mask(0) == {}+++def test_references_follow_the_generate_decoders_side_limit():+ with pytest.raises(ValueError):+ pup.prepare_references([_png_url((4100, 64))])+++def test_generate_skips_the_upsampler_while_another_account_runs(routes, runnable, monkeypatch):+ from hub.services.models import account_access++ client, backend, saved = routes++ def busy(path = None):+ # The combined guard: another account's image or video job, not only chat.+ raise account_access.GpuBusyForAnotherAccountError("other", 1)++ monkeypatch.setattr(account_access, "require_idle_other_accounts", busy)+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ assert runnable.calls == [] and backend.calls[-1]["prompt"] == "a cat"+ assert r.json()["images"][0]["prompt_enhancement"] == "unavailable"+++def test_preview_is_refused_while_another_account_runs(routes, runnable, monkeypatch):+ from hub.services.models import account_access++ client, backend, saved = routes++ def busy(path = None):+ raise account_access.GpuBusyForAnotherAccountError("other", 1)++ monkeypatch.setattr(account_access, "require_idle_other_accounts", busy)+ r = client.post("/api/inference/images/prompt-upsampler/preview", json = {"prompt": "a cat"})+ assert r.status_code == 409+ assert runnable.calls == []+++def test_references_are_decoded_one_at_a_time(monkeypatch):+ live = []+ peak = []+ real_decode, real_prepare = pup.decode_reference, pup.prepare_reference++ def decode(data):+ live.append(1)+ peak.append(len(live))+ return real_decode(data)++ def prepare(image, count):+ out = real_prepare(image, count)+ live.pop()+ return out++ monkeypatch.setattr(pup, "decode_reference", decode)+ monkeypatch.setattr(pup, "prepare_reference", prepare)+ refs = pup.prepare_references([_png_url((256, 256))] * 4)+ assert len(refs) == 4 and max(peak) == 1+++def test_stop_during_server_start_cancels_the_start(monkeypatch):+ sidecar = pup.PromptUpsamplerSidecar()+ started, cancelled = threading.Event(), threading.Event()++ def launch(binary, model, mmproj):+ sidecar._loading = True+ started.set()+ try:+ assert sidecar._load_cancel.wait(5), "Stop never reached the starting server"+ cancelled.set()+ raise pup.UpsamplerCancelledError("The prompt upsampler load was cancelled.")+ finally:+ sidecar._loading = False++ monkeypatch.setattr(sidecar, "_launch", launch)+ event = threading.Event()+ threading.Thread(target = lambda: (started.wait(5), event.set()), daemon = True).start()+ with pytest.raises(pup.UpsamplerCancelledError):+ sidecar.generate({}, binary = "b", model = "m", mmproj = None, cancel_event = event)+ assert cancelled.is_set()+++def test_training_stops_an_enhancement_that_has_not_reached_the_sidecar(monkeypatch):+ import sys++ from routes import training_vram++ stopped = []+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)+ monkeypatch.setattr(pup, "cancel_active", lambda: stopped.append(True) or True)+ monkeypatch.setattr(pup, "get_sidecar", lambda: pup.PromptUpsamplerSidecar())+ assert training_vram.free_prompt_upsampler_for_training("test") == []+ assert stopped == [True]+++class _ResidentUpsampler:+ loaded = True++ def __init__(self):+ self.unloaded = []++ def holds_vram(self):+ return True++ def is_loading(self):+ return False++ def unload(self, wait = True):+ self.unloaded.append(wait)+++def test_training_stops_the_upsampler_even_when_it_is_the_only_resident(monkeypatch):+ import sys++ from routes import training_vram++ resident = _ResidentUpsampler()+ monkeypatch.setattr(pup, "get_sidecar", lambda: resident)+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)+ idle = {"model": None, "device": None, "loading": False, "any": False}+ monkeypatch.setattr(training_vram, "summarize_resident_chat", lambda: dict(idle))+ monkeypatch.setattr(training_vram, "summarize_resident_stt", lambda: dict(idle))+ freed = training_vram.coordinate_models_for_training(lambda: (True, {}))+ assert freed == ["image:prompt-upsampler"] and resident.unloaded+++def test_arbiter_diffusion_eviction_stops_the_upsampler(monkeypatch):+ import core.inference.diffusion_engine_router as engine_router+ from core.inference import gpu_arbiter++ class _Engine:+ def unload(self):+ pass++ sidecar = _ResidentUpsampler()+ monkeypatch.setattr(engine_router, "get_active_diffusion_engine", lambda: _Engine())+ monkeypatch.setattr(pup, "_sidecar", sidecar)+ import sys++ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)+ event = threading.Event()+ with pup._tracking(1, event):+ gpu_arbiter._evict_diffusion()+ # The run answering is stopped first, so the handover does not wait out its caption.+ assert event.is_set()+ assert sidecar.unloaded == [True]+++def test_every_drop_of_the_diffusion_claim_stops_the_upsampler(monkeypatch):+ import sys++ from core.inference import gpu_arbiter++ stops = []+ monkeypatch.setitem(sys.modules, "core.inference.flux3_prompt_upsampler", pup)+ monkeypatch.setattr(pup, "stop_for_handover", lambda: stops.append(gpu_arbiter.current_owner()))+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)+ # A switch to a CPU-only image model releases the claim without an eviction.+ gpu_arbiter.release(gpu_arbiter.DIFFUSION)+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.DIFFUSION)+ assert gpu_arbiter.release_if(gpu_arbiter.DIFFUSION, lambda: True)+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ gpu_arbiter.release(gpu_arbiter.CHAT)+ # Stopped while DIFFUSION still held the claim, so no other owner can start beside it.+ assert stops == [gpu_arbiter.DIFFUSION, gpu_arbiter.DIFFUSION]+++def test_a_failed_enhancement_still_binds_the_generation_to_its_model(routes, runnable):+ from core.inference.diffusion_families import load_identity++ client, backend, saved = routes+ runnable.error = RuntimeError("boom")+ status = backend.status()+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 200, r.text+ assert r.json()["images"][0]["prompt_enhancement"] == "failed"+ assert backend.calls[-1]["expected_load"] == load_identity(+ status.get("repo_id"), status.get("base_repo"), "flux.3-klein"+ )+++def test_a_queued_enhancement_keeps_the_active_one_tracked_and_stop_reaches_both():+ first, second = threading.Event(), threading.Event()+ with pup._tracking(1, first) as active:+ with pup._tracking(2, second):+ # The run holding llama-server is reported, not the one queued behind it.+ assert pup.progress()["total"] == 1+ pup._advance(active)+ assert pup.progress()["done"] == 1+ # The queued run leaving does not clear the active one.+ assert pup.progress()["total"] == 1+ with pup._tracking(2, second):+ assert pup.cancel_active() is True+ assert first.is_set() and second.is_set()+ assert pup.progress() is None and pup.cancel_active() is False+++def test_model_and_projector_come_from_one_revision(monkeypatch):+ files = {+ ("rev_new", pup.UPSAMPLER_MODEL_FILE): "/c/new/model",+ ("rev_old", pup.UPSAMPLER_MODEL_FILE): "/c/old/model",+ ("rev_old", pup.UPSAMPLER_MMPROJ_FILE): "/c/old/mmproj",+ }+ monkeypatch.delenv(pup.ENV_MODEL_OVERRIDE, raising = False)+ monkeypatch.delenv(pup.ENV_MMPROJ_OVERRIDE, raising = False)+ monkeypatch.setattr(pup, "_candidate_revisions", lambda hub_cache = None: ["rev_new", "rev_old"])+ monkeypatch.setattr(+ pup, "_cached_file", lambda name, hub_cache = None, revision = None: files.get((revision, name))+ )+ # rev_new has no projector, so the pair is rev_old's, never rev_new's model with rev_old's projector.+ assert (pup.model_path(), pup.mmproj_path()) == ("/c/old/model", "/c/old/mmproj")+ del files[("rev_old", pup.UPSAMPLER_MMPROJ_FILE)]+ files[("rev_new", pup.UPSAMPLER_MMPROJ_FILE)] = None+ assert (pup.model_path(), pup.mmproj_path()) == ("/c/new/model", None)+ assert pup._cached_download_paths(pup.UPSAMPLER_ID) is None+++def test_no_repo_download_beside_an_override(routes, monkeypatch):+ client, backend, saved = routes+ started = []+ monkeypatch.setattr(pup, "status", lambda: {"override": True})+ monkeypatch.setattr(pup, "start_download", lambda token: started.append(token) or "dl")+ r = client.post("/api/inference/images/prompt-upsampler/download")+ assert r.status_code == 409 and "UNSLOTH_FLUX3_UPSAMPLER_GGUF" in r.text and started == []+++def test_a_systemic_failure_runs_the_rest_of_the_batch_as_typed(runnable):+ runnable.error = RuntimeError("llama-server did not become ready for the prompt upsampler.")+ out = pup.enhance_for_generation(+ ["a", "b", "c"], family_name = "flux.3-klein", enhance_prompt = None+ )+ # One start attempt, not one per prompt; every prompt falls back.+ assert len(runnable.calls) == 1+ assert out.statuses == {"a": "failed", "b": "failed", "c": "failed"} and out.captions == {}+++def test_no_sidecar_start_after_the_gpu_moved_to_another_owner(monkeypatch):+ from core.inference import gpu_arbiter++ sidecar = pup.PromptUpsamplerSidecar()+ launched = []+ monkeypatch.setattr(sidecar, "_launch", lambda *a: launched.append(a))+ monkeypatch.setattr(gpu_arbiter, "_owner", gpu_arbiter.CHAT)+ with pytest.raises(pup.UpsamplerUnavailableError, match = "no longer holds the GPU"):+ sidecar.generate({}, binary = "b", model = "m", mmproj = None)+ assert launched == []+++def test_the_preview_caps_the_aggregate_image_payload():+ from pydantic import ValidationError++ from models.inference import PromptUpsamplerPreviewRequest++ big = "x" * (30 * 1024 * 1024)+ PromptUpsamplerPreviewRequest(prompt = "p", reference_images = [big] * 4)+ with pytest.raises(ValidationError, match = "at most 128 MiB"):+ PromptUpsamplerPreviewRequest(prompt = "p", reference_images = [big] * 5)+++def test_a_managed_account_is_authorized_before_the_upsampler_runs(routes, runnable, monkeypatch):+ from fastapi import HTTPException++ from hub.services.models import account_access++ client, backend, saved = routes+ monkeypatch.setattr(account_access, "managed_account", lambda: True)++ def deny(status, modality = None):+ raise HTTPException(status_code = 404, detail = "Model not found")++ monkeypatch.setattr(account_access, "require_media_generation_access", deny)+ r = client.post("/api/inference/images/generate", json = {"prompt": "a cat", "steps": 4})+ assert r.status_code == 404 and runnable.calls == []diff --git a/studio/backend/tests/test_image_gallery.py b/studio/backend/tests/test_image_gallery.pyindex d1a913dad41..10f3895870f 100644--- a/studio/backend/tests/test_image_gallery.py+++ b/studio/backend/tests/test_image_gallery.py@@ -706,3 +706,10 @@ def test_connection_images_record_no_local_engine(monkeypatch):remote = gallery.save(_img(), {**base, "model_kind": "connection"})assert local["engine"] == "diffusers"assert remote["engine"] is None+++def test_parameters_chunk_carries_the_encoded_caption():+ meta = {**_meta(), "prompt": "a cat", "enhanced_prompt": "A tabby cat on a sunlit sill."}+ text = gallery._params_text(meta)+ assert text.splitlines()[0] == "A tabby cat on a sunlit sill."+ assert gallery._params_text({**_meta(), "prompt": "a cat"}).splitlines()[0] == "a cat"diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex df0063cad8c..b28be02b76c 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -3871,6 +3871,7 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():"core/inference/audio_cpp_server.py","core/inference/diffusion_vae_prebuild.py","core/inference/engine_install.py",+ "core/inference/flux3_prompt_upsampler.py","core/inference/lemonade_server.py","core/inference/llama_cpp.py","core/inference/managed_engine.py",diff --git a/studio/backend/tests/test_stt_http_downloads.py b/studio/backend/tests/test_stt_http_downloads.pyindex 6d8b5636ff2..0991a8a5bc1 100644--- a/studio/backend/tests/test_stt_http_downloads.py+++ b/studio/backend/tests/test_stt_http_downloads.py@@ -272,6 +272,25 @@ def test_selected_file_progress_covers_partial_final_and_windows_snapshot(tmp_paassert stt_sidecar._downloaded_file_bytes(**kwargs) == 100+def test_selected_file_progress_counts_the_process_unique_partial(tmp_path):+ # huggingface_hub >= 1.18 names the partial `<etag>.<8 hex>.incomplete`.+ repo = "Org/Repo"+ blob_key = "c" * 64+ blobs = _repo_dir(tmp_path, repo) / "blobs"+ blobs.mkdir(parents = True)+ (blobs / f"{blob_key}.ca4ea4fe.incomplete").write_bytes(b"x" * 30)+ (blobs / f"{'d' * 64}.0badf00d.incomplete").write_bytes(b"x" * 90)+ kwargs = {+ "hub_cache": tmp_path,+ "repo": repo,+ "filename": "model.gguf",+ "size": 100,+ "blob_key": blob_key,+ "revision": None,+ }+ assert stt_sidecar._downloaded_file_bytes(**kwargs) == 30++def test_mtmd_progress_counts_only_its_two_exact_files(tmp_path):model_id = "qwen3-asr-0.6b"spec = stt_mtmd_sidecar.MTMD_STT_MODELS[model_id]diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.pyindex 2a99e717d3e..902bc2b2515 100644--- a/studio/backend/utils/llama_cpp_update.py+++ b/studio/backend/utils/llama_cpp_update.py@@ -777,6 +777,7 @@ def _run_llama_phase(# The mtmd dictation sidecar serves Qwen3-ASR from this same llama-server out of this same tree, so a live one locks the exe on Windows and a concurrent load would start against a half-swapped install.model_was_active = _block_mtmd_sidecar(mtmd_guard) or model_was_active+ model_was_active = _block_prompt_upsampler(mtmd_guard) or model_was_activecmd = [sys.executable,@@ -961,6 +962,19 @@ def _block_mtmd_sidecar(stack: ExitStack) -> bool:return False+def _block_prompt_upsampler(stack: ExitStack) -> bool:+ """The FLUX.3 prompt upsampler runs the same llama-server; hold its maintenance guard for the install, only+ if Images ever loaded the module. Not fail-closed, like the mtmd guard."""+ module = sys.modules.get("core.inference.flux3_prompt_upsampler")+ if module is None:+ return False+ try:+ return stack.enter_context(module.get_sidecar().update_maintenance())+ except Exception as exc: # noqa: BLE001 - the update proceeds without it+ logger.debug("llama update: prompt upsampler coordination failed", error = str(exc))+ return False++# Combined-job progress split when every phase runs (the llama bundle dwarfs the others); normalized to 0..1 over the phases that run._LLAMA_PHASE_WEIGHT = 0.7_WHISPER_PHASE_WEIGHT = 0.3diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.tsindex cd6a6f4da10..abe4ba05efb 100644--- a/studio/frontend/src/features/images/api.ts+++ b/studio/frontend/src/features/images/api.ts@@ -9,6 +9,13 @@ import {MEMORY_REFUSAL_HEADER,MemoryEstimateRefusalError,} from "./lib/memory-refusal";+import {+ parsePromptUpsamplerStatus,+ PROMPT_UPSAMPLER_ROUTES,+ type PromptEnhancement,+ type PromptUpsamplerPreviewRequest,+ type PromptUpsamplerStatus,+} from "./lib/prompt-enhance";// One Advanced control's resolved value and provenance, for the Advanced-panel badges. `value` is the engaged// value (null when off), `requested` is what the caller asked for (null = left to the backend), `source` is "auto"@@ -83,6 +90,10 @@ export interface DiffusionStatus {resolved?: Record<string, DiffusionResolvedControl> | null;// Default steps / guidance the backend renders the loaded model with (file header before base repo).generation_defaults?: { steps?: number; guidance?: number } | null;+ // Whether the loaded family can run the prompt upsampler (FLUX.3), and whether it is on by default. Absent on+ // an older backend: no Enhance prompt toggle.+ supports_prompt_upsampler?: boolean;+ prompt_upsampler_default?: boolean;}export interface DiffusionSamplerOption {@@ -113,8 +124,10 @@ export interface DiffusionGenerateProgress {total_steps: number;fraction: number;eta_seconds: number | null;- // Absent (sd.cpp engine) means "denoise". "encode" runs before the denoise loop starts.- phase?: "encode" | "denoise" | "decode" | null;+ // Absent (sd.cpp engine) means "denoise". "encode" runs before the denoise loop starts; "enhance" is the FLUX.3+ // prompt upsampler writing the caption before the run (with `message`).+ phase?: "enhance" | "encode" | "denoise" | "decode" | null;+ message?: string | null;// Live latent preview of the image being denoised (small JPEG data URL), and a counter that moves with each one.preview?: string | null;preview_seq?: number;@@ -215,6 +228,9 @@ export interface DiffusionGenerateRequest {controlnet?: ControlNetSpecInput;sampler?: string;schedule_type?: string;+ // Rewrite the prompt with the family's prompt upsampler first. null/absent = the family default. Sent only for a+ // family that supports it.+ enhance_prompt?: boolean | null;}// One LoRA selection sent with a generation.@@ -303,6 +319,9 @@ export interface GalleryImage {// null = the model's shipped scheduler; schedule_type is the resolved schedule.sampler?: string | null;schedule_type?: string | null;+ // The caption the prompt upsampler wrote and the model actually ran; `prompt` stays the user's text.+ enhanced_prompt?: string | null;+ prompt_enhancement?: PromptEnhancement | null;created_at: number;// Library state, not recipe: stored beside the PNG, absent on records written before this existed.pinned?: boolean;@@ -494,6 +513,43 @@ export async function unloadDiffusionModel(): Promise<DiffusionStatus> {return parseJson(await authFetch("/api/inference/images/unload", { method: "POST" }));}+export async function getPromptUpsamplerStatus(signal?: AbortSignal): Promise<PromptUpsamplerStatus> {+ return parsePromptUpsamplerStatus(+ await parseJson<unknown>(await authFetch(PROMPT_UPSAMPLER_ROUTES.status, { signal })),+ );+}++/** Start the background download of the prompt upsampler. Progress is read back from the status route. */+export async function startPromptUpsamplerDownload(hfToken?: string): Promise<void> {+ await parseJson<unknown>(+ await authFetch(PROMPT_UPSAMPLER_ROUTES.download, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(hfToken ? { hf_token: hfToken } : {}),+ }),+ );+}++/** Stop the download. Partial files stay cached, so a later download resumes. */+export async function cancelPromptUpsamplerDownload(): Promise<void> {+ await parseJson<unknown>(+ await authFetch(PROMPT_UPSAMPLER_ROUTES.cancel, { method: "POST" }),+ );+}++/** Rewrite a prompt without generating, so the caption can be read or edited first. */+export async function previewEnhancedPrompt(+ body: PromptUpsamplerPreviewRequest,+): Promise<{ enhanced_prompt: string }> {+ return parseJson(+ await authFetch(PROMPT_UPSAMPLER_ROUTES.preview, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(body),+ }),+ );+}+/** List diffusion LoRA adapters, optionally filtered to a model family. */export async function listDiffusionLoras(family?: string): Promise<DiffusionLoraInfo[]> {const qs = family ? `?family=${encodeURIComponent(family)}` : "";diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsxindex 8e91a080c21..7520888b6d4 100644--- a/studio/frontend/src/features/images/images-page.tsx+++ b/studio/frontend/src/features/images/images-page.tsx@@ -320,6 +320,20 @@ import {SCHEDULE_LABELS,withSamplerChoice,} from "./lib/sampler-choice";+import {+ enhancePromptRequestField,+ enhanceRestoreFor,+ enhancementNotice,+ hasEnhancedPrompt,+ ENHANCE_UNSUPPORTED_FAMILY,+ overrideAfterFamilyChange,+ promptUpsamplerCapability,+ readEnhanceChoices,+ resolveEnhancePrompt,+ upsamplerVariantFor,+ withEnhanceChoice,+} from "./lib/prompt-enhance";+import { EnhancedPromptRecipeRow, EnhancePromptControls } from "./prompt-enhance-panel";import { useNavigate, useSearch } from "@tanstack/react-router";import { useStagedDownload, type StagedDownloadEntry } from "@/features/hub/download-manager";import { DiffusionTrainPanel } from "./train/diffusion-train-panel";@@ -1146,7 +1160,7 @@ function RecipePopover({active,}: {image: GalleryImage;- onRestore: (image: GalleryImage) => void;+ onRestore: (image: GalleryImage, exactCaption?: boolean) => void;active: boolean;}) {// Controlled + force-closed off-tab: PopoverContent portals to body, so the inert page wrapper cannot contain it.@@ -1177,6 +1191,9 @@ function RecipePopover({</div><div className="flex min-h-0 flex-col gap-2 overflow-y-auto overscroll-contain px-4 py-3 text-xs"><RecipeRow label="Prompt" value={image.prompt} wrap />+ {hasEnhancedPrompt(image) ? (+ <EnhancedPromptRecipeRow caption={(image.enhanced_prompt ?? "").trim()} />+ ) : null}{image.negative_prompt ? (<RecipeRow label="Negative" value={image.negative_prompt} wrap />) : null}@@ -1267,6 +1284,17 @@ function RecipePopover({<HugeiconsIcon icon={Refresh01Icon} className="size-4" />Restore these settings</Button>+ {/* The caption that actually ran, with Enhance prompt off, so the run reproduces without the upsampler. */}+ {image.prompt_enhancement === "applied" && hasEnhancedPrompt(image) ? (+ <Button+ size="sm"+ variant="ghost"+ className="mt-1 w-full gap-1.5"+ onClick={() => onRestore(image, true)}+ >+ Restore with the enhanced prompt+ </Button>+ ) : null}</div></PopoverContent></Popover>@@ -1495,6 +1523,9 @@ export function ImagesPage({const [steps, setSteps] = useState(DEFAULT_GEN.steps);const [guidance, setGuidance] = useState(DEFAULT_GEN.guidance);const [samplerChoices, setSamplerChoices] = useState(readSamplerChoices);+ const [enhanceChoices, setEnhanceChoices] = useState(readEnhanceChoices);+ // The family it was set under ("" = before a model loaded, which applies to whichever family loads next).+ const [enhanceOverrideState, setEnhanceOverrideState] = useState<{ family: string; on: boolean } | null>(null);// A recipe restored while unloaded: Generate recalls the model, uses this, and stores it for that model.const [restoredSampler, setRestoredSampler] = useState<Partial<SamplerChoice> | null>(null);// Whether the user has taken the recipe since the pick still waiting for its status: a preset@@ -1975,6 +2006,41 @@ export function ImagesPage({},[samplerChoice, samplerModelKey, samplerOptions],);+ // Enhance prompt (FLUX.3's prompt upsampler): stored per family, defaulting to the family's own default. A+ // restore or "Use as prompt" sets a one-off override that the next toggle or family change drops.+ const upsamplerCapability = promptUpsamplerCapability(status, Boolean(external));+ const enhanceFamilyKey = upsamplerCapability.supported ? (status?.family ?? "") : "";+ // A family change consumes the override (overrideAfterFamilyChange), a switch to a model or connection without an+ // upsampler included.+ const enhanceTrackKey =+ enhanceFamilyKey || (external || status?.loaded ? ENHANCE_UNSUPPORTED_FAMILY : "");+ useEffect(() => {+ if (enhanceTrackKey) setEnhanceOverrideState((prev) => overrideAfterFamilyChange(prev, enhanceTrackKey));+ }, [enhanceTrackKey]);+ const enhanceOverride =+ enhanceOverrideState &&+ (enhanceOverrideState.family === "" || enhanceOverrideState.family === enhanceFamilyKey)+ ? enhanceOverrideState.on+ : null;+ const setEnhanceOverride = useCallback(+ (on: boolean | null) =>+ setEnhanceOverrideState(on === null ? null : { family: enhanceFamilyKey, on }),+ [enhanceFamilyKey],+ );+ const enhanceOn = resolveEnhancePrompt({+ capability: upsamplerCapability,+ stored: enhanceChoices[enhanceFamilyKey],+ override: enhanceOverride,+ });+ const changeEnhancePrompt = useCallback(+ (on: boolean) => {+ setEnhanceOverride(null);+ setEnhanceChoices((all) =>+ withEnhanceChoice(all, enhanceFamilyKey, on, upsamplerCapability.defaultOn),+ );+ },+ [enhanceFamilyKey, setEnhanceOverride, upsamplerCapability.defaultOn],+ );useEffect(() => {if (!controlnetSupported) {setAvailableControlNets([]);@@ -2469,7 +2535,7 @@ export function ImagesPage({[dropFromStrip],);- const restoreSettings = useCallback((image: GalleryImage) => {+ const restoreSettings = useCallback((image: GalleryImage, exactCaption = false) => {const viaConnection = image.model_kind === "connection";// Negative prompt only applies when guidance>0; do not restore a hidden value.const restoredNegative = image.guidance > 0 ? (image.negative_prompt ?? "") : "";@@ -2519,7 +2585,11 @@ export function ImagesPage({const reopened: WorkflowId =image.workflow === "edit" ? "edit" : image.workflow === "reference" ? "reference" : "create";setWorkflow(reopened);- setPromptFor(reopened, image.prompt);+ // An enhanced run restores the user's prompt with Enhance prompt on, or (exactCaption) the caption it ran with+ // the toggle off. Records without an outcome leave the toggle alone.+ const enhanceRestore = enhanceRestoreFor(image, exactCaption);+ setPromptFor(reopened, enhanceRestore.prompt);+ if (enhanceRestore.enhance !== null) setEnhanceOverride(enhanceRestore.enhance);setInitImage(null);setMaskImage(null);setReferenceImages(@@ -2549,7 +2619,7 @@ export function ImagesPage({} else {toast.success("Settings restored to inputs", rescaled);}- }, [changeSamplerChoice, samplerOptions.length, setPromptFor, setWorkflow, sizeLimits]);+ }, [changeSamplerChoice, samplerOptions.length, setEnhanceOverride, setPromptFor, setWorkflow, sizeLimits]);// Like dropping an image on ComfyUI: Unsloth's recipe first, else AUTOMATIC1111 / ComfyUI metadata.const importFileRef = useRef<HTMLInputElement>(null);@@ -4601,6 +4671,7 @@ export function ImagesPage({allow_oversized: allowOversizedSent ? true : undefined,live_preview: livePreview,...samplerRequestFields(samplerOptions, samplerChoice),+ ...enhancePromptRequestField(upsamplerCapability, enhanceOn),...condFields,// Drop empty and zero-weight rows and trim hand-typed repo ids, so the recipe records only// adapters that applied. Gated on loraCapable, since a restore can leave adapters in state.@@ -4647,6 +4718,11 @@ export function ImagesPage({setImages((prev) => mergeGenerated(prev, res.images));res.images.forEach((image) => knownIds.add(image.id));if (res.images[0]) setSelectedId(res.images[0].id);+ // Asked for enhancement and did not get it: say so once, without blocking the result.+ if (enhanceOn && upsamplerCapability.supported) {+ const notice = enhancementNotice(res.images[0]?.prompt_enhancement);+ if (notice) toast.info(notice.title, { id: "prompt-enhancement-notice", description: notice.description });+ }res.images.forEach((image) => void ensureSrc(image));setGenDone(i + 1);}@@ -4688,7 +4764,7 @@ export function ImagesPage({setGenStep(null);setStopping(false);}- }, [external, allowOversized, livePreview, prompt, negativePrompt, negativeCapable, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);+ }, [external, allowOversized, livePreview, enhanceOn, upsamplerCapability, prompt, negativePrompt, negativeCapable, restoredSampler, samplerModelKey, samplerOptions, samplerChoice, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, loras, loraCapable, controlnetCapable, controlnetId, controlImage, controlType, controlStrength, ensureSrc, loadGallery, refreshStatus, unifiedEdit, localizedMode, localizedLayer, maxExtras, referenceResolution, conditioning, editSize, editSizing, sizeLimits]);// Stop the in-flight generation. Latch FIRST, so a multi-run request stops even if the POST// races the run that is already finishing.@@ -5644,6 +5720,29 @@ export function ImagesPage({onChange={(e) => setPrompt(e.target.value)}/></Field>+ {upsamplerCapability.supported && (+ <EnhancePromptControls+ enabled={enhanceOn}+ onEnabledChange={changeEnhancePrompt}+ prompt={prompt}+ previewRequest={() => {+ // Edit and Reference condition on their images, so the caption describes them too.+ const conditioned = workflow === "edit" || workflow === "reference";+ const images = conditioned+ ? [initImage, ...referenceImages].filter((src): src is string => Boolean(src))+ : [];+ return {+ prompt: prompt.trim(),+ variant: upsamplerVariantFor(status?.family),+ ...(images.length ? { reference_images: images } : {}),+ };+ }}+ onUseCaption={(caption) => {+ setPrompt(caption);+ setEnhanceOverride(false);+ }}+ />+ )}{negativeCapable && (<NegativePromptFieldvalue={negativePrompt}diff --git a/studio/frontend/src/features/images/lib/image-metadata.ts b/studio/frontend/src/features/images/lib/image-metadata.tsindex 755f03ba2b4..e466d1c0c8b 100644--- a/studio/frontend/src/features/images/lib/image-metadata.ts+++ b/studio/frontend/src/features/images/lib/image-metadata.ts@@ -619,6 +619,9 @@ export interface RestorableRecipe {model_kind?: string;sampler?: string;schedule_type?: string;+ // The prompt upsampler's caption and outcome; restore decides the Enhance prompt toggle from them.+ enhanced_prompt?: string | null;+ prompt_enhancement?: "applied" | "off" | "unavailable" | "failed" | null;}export const APPLIED_RECIPE_KEYS = [@@ -640,6 +643,8 @@ export const APPLIED_RECIPE_KEYS = ["localized_edit","sampler","schedule_type",+ "enhanced_prompt",+ "prompt_enhancement",] as const;/** Known but not applied: loading a model or a ControlNet image stays the user's call. */@@ -767,6 +772,14 @@ export function recipeSettings(model_kind: r.model_kind === "connection" ? "connection" : undefined,sampler: str(r.sampler),schedule_type: str(r.schedule_type),+ enhanced_prompt: str(r.enhanced_prompt) ?? null,+ prompt_enhancement:+ r.prompt_enhancement === "applied" ||+ r.prompt_enhancement === "off" ||+ r.prompt_enhancement === "unavailable" ||+ r.prompt_enhancement === "failed"+ ? r.prompt_enhancement+ : null,};}diff --git a/studio/frontend/src/features/images/lib/prompt-enhance.ts b/studio/frontend/src/features/images/lib/prompt-enhance.tsnew file mode 100644index 00000000000..cd5c819ac7a--- /dev/null+++ b/studio/frontend/src/features/images/lib/prompt-enhance.ts@@ -0,0 +1,294 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// "Enhance prompt": the FLUX.3 prompt upsampler rewrites a short request into the detailed caption the model was+// trained on. Pure logic only (no "@/" imports) so node:test can load it directly.++/** Every upsampler route, in one place. */+export const PROMPT_UPSAMPLER_ROUTES = {+ status: "/api/inference/images/prompt-upsampler/status",+ download: "/api/inference/images/prompt-upsampler/download",+ cancel: "/api/inference/images/prompt-upsampler/cancel",+ preview: "/api/inference/images/prompt-upsampler/preview",+} as const;++export const ENHANCE_PROMPT_LABEL = "Enhance prompt";+export const ENHANCE_PROMPT_HINT =+ "Rewrites your prompt into a detailed caption with FLUX.3's prompt upsampler before generating. Fixes most broken poses and hands. Turn off to use your prompt exactly as typed.";++export type PromptEnhancement = "applied" | "off" | "unavailable" | "failed";++/** Same shape as the speech-to-text model download status (SttDownloadStatus). */+export interface PromptUpsamplerDownload {+ downloading: boolean;+ model: string | null;+ download_id?: string | null;+ completed_download_ids?: string[];+ variant?: string | null;+ error: string | null;+ cancelled?: boolean;+ cancelled_model?: string | null;+ bytes_total: number | null;+ bytes_done: number | null;+}++export interface PromptUpsamplerStatus {+ supported: boolean;+ downloaded: boolean;+ loaded: boolean;+ repo: string;+ filename: string;+ size_bytes: number;+ mmproj_size_bytes: number;+ mmproj_downloaded: boolean;+ // Local files from the env overrides: no repo download is offered beside them.+ override: boolean;+ download: PromptUpsamplerDownload;+}++export interface PromptUpsamplerPreviewRequest {+ prompt: string;+ reference_images?: string[];+ variant?: "distilled" | "base";+}++const IDLE_DOWNLOAD: PromptUpsamplerDownload = {+ downloading: false,+ model: null,+ error: null,+ bytes_total: null,+ bytes_done: null,+};++const num = (v: unknown): number | null =>+ typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;+const str = (v: unknown): string | null => (typeof v === "string" ? v : null);++/** Tolerant parse of the status body: a missing or malformed field reads as "not there", never a throw. */+export function parsePromptUpsamplerStatus(raw: unknown): PromptUpsamplerStatus {+ const body = raw && typeof raw === "object" ? (raw as Record<string, unknown>) : {};+ const d =+ body.download && typeof body.download === "object"+ ? (body.download as Record<string, unknown>)+ : {};+ const download: PromptUpsamplerDownload = {+ ...IDLE_DOWNLOAD,+ downloading: d.downloading === true,+ model: str(d.model),+ error: str(d.error),+ bytes_total: num(d.bytes_total),+ bytes_done: num(d.bytes_done),+ };+ if (typeof d.download_id === "string") download.download_id = d.download_id;+ if (typeof d.cancelled === "boolean") download.cancelled = d.cancelled;+ return {+ supported: body.supported === true,+ downloaded: body.downloaded === true,+ loaded: body.loaded === true,+ repo: str(body.repo) ?? "",+ filename: str(body.filename) ?? "",+ size_bytes: num(body.size_bytes) ?? 0,+ mmproj_size_bytes: num(body.mmproj_size_bytes) ?? 0,+ mmproj_downloaded: body.mmproj_downloaded === true,+ override: body.override === true,+ download,+ };+}++/** The model or its vision projector (edits) is missing, so the Download control stays offered. */+export function upsamplerNeedsDownload(status: PromptUpsamplerStatus): boolean {+ return status.supported && !status.override && (!status.downloaded || !status.mmproj_downloaded);+}++/** Bytes the Download button fetches: whichever of the model and the vision projector is still missing. */+export function upsamplerDownloadBytes(status: PromptUpsamplerStatus): number {+ return (+ (status.downloaded ? 0 : status.size_bytes) ++ (status.mmproj_downloaded ? 0 : status.mmproj_size_bytes)+ );+}++/** 0..1 download fraction, or null when the total is not known yet. */+export function upsamplerDownloadFraction(download: PromptUpsamplerDownload): number | null {+ const total = download.bytes_total ?? 0;+ if (!(total > 0)) return null;+ return Math.min(1, Math.max(0, (download.bytes_done ?? 0) / total));+}++/** What the loaded model advertises. Absent on an older backend: unsupported, default off. */+export interface PromptUpsamplerCapability {+ supported: boolean;+ defaultOn: boolean;+}++export function promptUpsamplerCapability(+ status:+ | {+ loaded?: boolean;+ supports_prompt_upsampler?: boolean;+ prompt_upsampler_default?: boolean;+ }+ | null+ | undefined,+ external = false,+): PromptUpsamplerCapability {+ const supported = !external && status?.loaded === true && status.supports_prompt_upsampler === true;+ return { supported, defaultOn: supported && status?.prompt_upsampler_default === true };+}++/** The toggle's value: a one-off restore override, else the stored per-family choice, else the family default. */+export function resolveEnhancePrompt(args: {+ capability: PromptUpsamplerCapability;+ stored: boolean | undefined;+ override?: boolean | null;+}): boolean {+ if (!args.capability.supported) return false;+ if (typeof args.override === "boolean") return args.override;+ if (typeof args.stored === "boolean") return args.stored;+ return args.capability.defaultOn;+}++/** The request field. Only a supporting family sends it, so every other request stays byte-identical. */+export function enhancePromptRequestField(+ capability: PromptUpsamplerCapability,+ on: boolean,+): { enhance_prompt?: boolean } {+ return capability.supported ? { enhance_prompt: on } : {};+}++/** The preview variant for the loaded family: base checkpoints get the base caption style. */+export function upsamplerVariantFor(family: string | null | undefined): "distilled" | "base" {+ return (family ?? "").toLowerCase().includes("base") ? "base" : "distilled";+}++const KEY = "unsloth:images:enhance-prompt-by-family";++export function readEnhanceChoices(): Record<string, boolean> {+ try {+ const value = JSON.parse(localStorage.getItem(KEY) ?? "{}");+ if (!value || typeof value !== "object" || Array.isArray(value)) return {};+ const out: Record<string, boolean> = {};+ for (const [k, v] of Object.entries(value)) if (typeof v === "boolean") out[k] = v;+ return out;+ } catch {+ return {};+ }+}++/** Store the choice for one family. A choice equal to the family default is dropped, so a changed default reaches+ * users who never touched the toggle. */+export function withEnhanceChoice(+ choices: Record<string, boolean>,+ familyKey: string,+ on: boolean,+ defaultOn: boolean,+): Record<string, boolean> {+ if (!familyKey) return choices;+ const next = { ...choices };+ if (on === defaultOn) delete next[familyKey];+ else next[familyKey] = on;+ try {+ localStorage.setItem(KEY, JSON.stringify(next));+ } catch {+ // storage can be unavailable in private browsing+ }+ return next;+}++/** What a recipe restore puts back. `exact` reuses the caption that actually ran, with enhancement off. */+export function enhanceRestoreFor(+ image: { prompt: string; enhanced_prompt?: string | null; prompt_enhancement?: string | null },+ exact = false,+): { prompt: string; enhance: boolean | null } {+ const applied = image.prompt_enhancement === "applied";+ const caption = (image.enhanced_prompt ?? "").trim();+ if (exact && applied && caption) return { prompt: caption, enhance: false };+ if (applied) return { prompt: image.prompt, enhance: true };+ // "off" ran the prompt as typed; unavailable / failed did too, but the user had asked for it, so keep asking.+ if (image.prompt_enhancement === "off") return { prompt: image.prompt, enhance: false };+ if (image.prompt_enhancement === "unavailable" || image.prompt_enhancement === "failed") {+ return { prompt: image.prompt, enhance: true };+ }+ // Records from before the upsampler, or another family: leave the toggle alone.+ return { prompt: image.prompt, enhance: null };+}++/** Whether a record carries an enhanced caption worth showing next to the user's prompt. */+export function hasEnhancedPrompt(image: {+ prompt: string;+ enhanced_prompt?: string | null;+ prompt_enhancement?: string | null;+}): boolean {+ const caption = (image.enhanced_prompt ?? "").trim();+ return Boolean(caption) && caption !== image.prompt.trim();+}++/** Two 32-bit hashes (FNV-1a, djb2) over the whole data URL: the same-format headers and trailers of two different+ * images often match, so the ends alone do not tell them apart. */+export function referenceDigest(src: string): string {+ let fnv = 0x811c9dc5;+ let djb = 5381;+ for (let i = 0; i < src.length; i++) {+ const code = src.charCodeAt(i);+ fnv = Math.imul(fnv ^ code, 0x01000193);+ djb = (Math.imul(djb, 33) + code) | 0;+ }+ return `${src.length}:${(fnv >>> 0).toString(16)}:${(djb >>> 0).toString(16)}`;+}++// The stale check runs on every render: digests of the images the last call named, and only those, so a removed+// image is not kept alive by the cache.+let referenceDigests = new Map<string, string>();++/** Drop the cached digests (and with them the uploads they are keyed on) once no preview is shown. */+export function clearPreviewDigests(): void {+ referenceDigests = new Map();+}++/** What a preview caption was written for: the prompt, the variant and the reference images. */+export function previewSignature(body: PromptUpsamplerPreviewRequest): string {+ const next = new Map<string, string>();+ const refs = (body.reference_images ?? []).map((src) => {+ const digest = next.get(src) ?? referenceDigests.get(src) ?? referenceDigest(src);+ next.set(src, digest);+ return digest;+ });+ referenceDigests = next;+ return JSON.stringify([body.prompt.trim(), body.variant ?? null, refs]);+}++/** Non-blocking notice after a run that asked for enhancement and did not get it, else null. */+export function enhancementNotice(+ outcome: string | null | undefined,+): { title: string; description: string } | null {+ if (outcome === "unavailable") {+ return {+ title: "Prompt used as typed",+ description:+ "The prompt upsampler could not run (not downloaded yet, or llama.cpp is unavailable), so the prompt ran unchanged.",+ };+ }+ if (outcome === "failed") {+ return {+ title: "Prompt used as typed",+ description: "The prompt upsampler could not rewrite this prompt, so it ran unchanged.",+ };+ }+ return null;+}++/** The family key of a loaded model or connection with no upsampler. */+export const ENHANCE_UNSUPPORTED_FAMILY = "\u0000unsupported";++/** The one-off Enhance override after the loaded family becomes ``familyKey`` (non-empty): one set before any model+ * loaded ("") binds to the first FLUX.3 family and survives an unsupported model, one set under another family is+ * dropped, so it cannot come back on a later switch. */+export function overrideAfterFamilyChange(+ prev: { family: string; on: boolean } | null,+ familyKey: string,+): { family: string; on: boolean } | null {+ if (!prev || !familyKey || prev.family === familyKey) return prev;+ if (prev.family === "") return familyKey === ENHANCE_UNSUPPORTED_FAMILY ? prev : { ...prev, family: familyKey };+ return null;+}+diff --git a/studio/frontend/src/features/images/prompt-enhance-panel.tsx b/studio/frontend/src/features/images/prompt-enhance-panel.tsxnew file mode 100644index 00000000000..b41b3256487--- /dev/null+++ b/studio/frontend/src/features/images/prompt-enhance-panel.tsx@@ -0,0 +1,346 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import { Cancel01Icon, Copy01Icon, Download04Icon, SparklesIcon } from "@hugeicons/core-free-icons";+import { HugeiconsIcon } from "@hugeicons/react";+import { ChevronDown } from "lucide-react";+import { useCallback, useEffect, useRef, useState } from "react";++import { Button } from "@/components/ui/button";+import { InfoHint } from "@/components/ui/info-hint";+import { Progress } from "@/components/ui/progress";+import { Spinner } from "@/components/ui/spinner";+import { Switch } from "@/components/ui/switch";+import { useIsAccountOwner } from "@/features/auth/account-session";+import { formatBytes, getHfToken, hfApiToken } from "@/features/hub";+import { copyToClipboard } from "@/lib/copy-to-clipboard";+import { toast } from "@/lib/toast";+import { cn } from "@/lib/utils";++import {+ cancelPromptUpsamplerDownload,+ getPromptUpsamplerStatus,+ previewEnhancedPrompt,+ startPromptUpsamplerDownload,+} from "./api";+import {+ ENHANCE_PROMPT_HINT,+ ENHANCE_PROMPT_LABEL,+ type PromptUpsamplerPreviewRequest,+ type PromptUpsamplerStatus,+ upsamplerDownloadBytes,+ upsamplerNeedsDownload,+ clearPreviewDigests,+ previewSignature,+ upsamplerDownloadFraction,+} from "./lib/prompt-enhance";++/** Upsampler status, polled every second while a download runs and once otherwise. */+function usePromptUpsamplerStatus(active: boolean) {+ const [status, setStatus] = useState<PromptUpsamplerStatus | null>(null);+ const [tick, setTick] = useState(0);+ const refresh = useCallback(() => setTick((t) => t + 1), []);+ const downloading = status?.download.downloading === true;+ useEffect(() => {+ if (!active) return;+ const controller = new AbortController();+ getPromptUpsamplerStatus(controller.signal)+ .then(setStatus)+ .catch(() => {});+ return () => controller.abort();+ }, [active, tick]);+ useEffect(() => {+ if (!active || !downloading) return;+ const id = window.setInterval(refresh, 1000);+ return () => window.clearInterval(id);+ }, [active, downloading, refresh]);+ return { status, refresh };+}++function CopyButton({ text, label }: { text: string; label: string }) {+ return (+ <Button+ type="button"+ size="sm"+ variant="ghost"+ className="h-7 gap-1 px-2 text-xs"+ aria-label={label}+ onClick={() => {+ void copyToClipboard(text).then((ok) =>+ ok ? toast.success("Copied") : toast.error("Could not copy"),+ );+ }}+ >+ <HugeiconsIcon icon={Copy01Icon} className="size-3.5" />+ Copy+ </Button>+ );+}++function UpsamplerDownloadNotice({+ status,+ onChanged,+}: {+ status: PromptUpsamplerStatus;+ onChanged: () => void;+}) {+ const [busy, setBusy] = useState(false);+ // The download and cancel routes are owner-only; a managed account is told who can fetch it instead.+ const owner = useIsAccountOwner();+ const { download } = status;+ const fraction = upsamplerDownloadFraction(download);+ const downloadBytes = upsamplerDownloadBytes(status);+ const size = downloadBytes > 0 ? formatBytes(downloadBytes) : null;+ const start = async () => {+ setBusy(true);+ try {+ await startPromptUpsamplerDownload(hfApiToken(getHfToken()));+ } catch (err) {+ toast.error("Could not download the prompt upsampler", {+ description: err instanceof Error ? err.message : undefined,+ });+ } finally {+ setBusy(false);+ onChanged();+ }+ };+ const cancel = async () => {+ try {+ await cancelPromptUpsamplerDownload();+ } catch (err) {+ toast.error(err instanceof Error ? err.message : "Could not cancel the download");+ } finally {+ onChanged();+ }+ };+ return (+ <div+ data-testid="prompt-upsampler-download"+ className="flex flex-col gap-1.5 rounded-md border border-border/60 bg-muted/30 px-2.5 py-2 text-ui-11"+ >+ {download.downloading ? (+ <>+ <div className="flex items-center justify-between gap-2">+ <span className="text-muted-foreground">+ {download.bytes_total+ ? `Downloading prompt upsampler: ${formatBytes(download.bytes_done ?? 0)} of ${formatBytes(download.bytes_total)}`+ : "Downloading prompt upsampler..."}+ </span>+ <Button+ type="button"+ size="sm"+ variant="ghost"+ className={cn("h-6 gap-1 px-1.5 text-xs", !owner && "hidden")}+ onClick={() => void cancel()}+ >+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />+ Cancel+ </Button>+ </div>+ <Progress+ className="h-1.5"+ value={fraction === null ? undefined : fraction * 100}+ indeterminate={fraction === null}+ />+ </>+ ) : (+ <>+ <span className="text-muted-foreground">+ {status.downloaded+ ? `The prompt upsampler's vision projector${size ? ` (${size})` : ""} is not downloaded. Until it is, edits are enhanced from the text alone.`+ : `The prompt upsampler${size ? ` (${size})` : ""} is not downloaded. Until it is, prompts run as typed.`}+ </span>+ {download.error && !download.cancelled ? (+ <span className="text-destructive">{download.error}</span>+ ) : null}+ {owner ? (+ <Button+ type="button"+ size="sm"+ variant="secondary"+ className="h-7 gap-1.5 self-start text-xs"+ disabled={busy}+ onClick={() => void start()}+ >+ {busy ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={Download04Icon} className="size-3.5" />}+ {size ? `Download (${size})` : "Download"}+ </Button>+ ) : (+ <span className="text-muted-foreground">Ask the owner of this Unsloth install to download it.</span>+ )}+ </>+ )}+ </div>+ );+}++/** The Enhance prompt toggle, the download notice, and the preview panel. Rendered only for a supporting family. */+export function EnhancePromptControls({+ enabled,+ onEnabledChange,+ prompt,+ previewRequest,+ onUseCaption,+}: {+ enabled: boolean;+ onEnabledChange: (on: boolean) => void;+ prompt: string;+ /** The preview body for the current form (prompt, reference images, variant). */+ previewRequest: () => PromptUpsamplerPreviewRequest;+ /** Put the caption in the prompt box and turn enhancement off. */+ onUseCaption: (caption: string) => void;+}) {+ const { status, refresh } = usePromptUpsamplerStatus(true);+ // Re-read once a download ends, so the notice clears without a reload.+ const wasDownloading = useRef(false);+ useEffect(() => {+ const now = status?.download.downloading === true;+ if (wasDownloading.current && !now) refresh();+ wasDownloading.current = now;+ }, [status?.download.downloading, refresh]);++ const [preview, setPreview] = useState<{ signature: string; caption: string } | null>(null);+ // The stale check stops running without a preview, so its cache would otherwise keep the last uploads alive.+ useEffect(() => {+ if (preview === null) clearPreviewDigests();+ }, [preview]);+ useEffect(() => clearPreviewDigests, []);+ const [previewOpen, setPreviewOpen] = useState(true);+ const [previewing, setPreviewing] = useState(false);+ const previewSeq = useRef(0);+ const runPreview = async () => {+ const body = previewRequest();+ if (!body.prompt.trim()) {+ toast.error("Prompt is empty");+ return;+ }+ const seq = ++previewSeq.current;+ setPreviewing(true);+ try {+ const res = await previewEnhancedPrompt(body);+ if (seq !== previewSeq.current) return;+ setPreview({ signature: previewSignature(body), caption: res.enhanced_prompt });+ setPreviewOpen(true);+ } catch (err) {+ if (seq !== previewSeq.current) return;+ toast.error("Could not enhance the prompt", {+ description: err instanceof Error ? err.message : undefined,+ });+ } finally {+ if (seq === previewSeq.current) setPreviewing(false);+ }+ };++ const needsDownload = enabled && status !== null && upsamplerNeedsDownload(status);+ const canPreview = status === null || status.downloaded;+ // A caption for an older prompt, other images or the other variant stays readable but is marked stale.+ const stale = preview !== null && preview.signature !== previewSignature(previewRequest());++ return (+ <div className="flex flex-col gap-1.5" data-testid="enhance-prompt-controls">+ <div className="flex items-center justify-between gap-2">+ <span className="flex items-center gap-1 text-xs font-medium text-muted-foreground">+ {ENHANCE_PROMPT_LABEL}+ <InfoHint>{ENHANCE_PROMPT_HINT}</InfoHint>+ </span>+ <div className="flex items-center gap-1.5">+ <Button+ type="button"+ size="sm"+ variant="ghost"+ className="h-7 gap-1 px-2 text-xs"+ disabled={previewing || !canPreview || !prompt.trim()}+ title={canPreview ? "Rewrite the prompt now and show the caption" : "Download the prompt upsampler first"}+ onClick={() => void runPreview()}+ >+ {previewing ? <Spinner className="size-3.5" /> : <HugeiconsIcon icon={SparklesIcon} className="size-3.5" />}+ {previewing ? "Enhancing..." : "Preview"}+ </Button>+ <Switch checked={enabled} onCheckedChange={onEnabledChange} aria-label={ENHANCE_PROMPT_LABEL} />+ </div>+ </div>+ {needsDownload && status ? <UpsamplerDownloadNotice status={status} onChanged={refresh} /> : null}+ {preview ? (+ <div+ data-testid="enhanced-prompt-preview"+ className="flex flex-col gap-1 rounded-md border border-border/60 px-2.5 py-2 text-xs"+ >+ <div className="flex items-center justify-between gap-2">+ <button+ type="button"+ aria-expanded={previewOpen}+ onClick={() => setPreviewOpen((o) => !o)}+ className="flex items-center gap-1 font-medium text-muted-foreground transition-colors hover:text-foreground"+ >+ Enhanced prompt{stale ? " (for an earlier request)" : ""}+ <ChevronDown className={cn("size-3 transition-transform", previewOpen && "rotate-180")} />+ </button>+ <button+ type="button"+ aria-label="Dismiss enhanced prompt"+ className="text-muted-foreground hover:text-foreground"+ onClick={() => setPreview(null)}+ >+ <HugeiconsIcon icon={Cancel01Icon} className="size-3.5" />+ </button>+ </div>+ {previewOpen ? (+ <>+ <p className="max-h-48 overflow-y-auto whitespace-pre-wrap break-words text-foreground">+ {preview.caption}+ </p>+ <div className="flex items-center justify-end gap-1">+ <CopyButton text={preview.caption} label="Copy enhanced prompt" />+ <Button+ type="button"+ size="sm"+ variant="secondary"+ className="h-7 px-2 text-xs"+ onClick={() => {+ onUseCaption(preview.caption);+ setPreview(null);+ }}+ >+ Use as prompt+ </Button>+ </div>+ </>+ ) : null}+ </div>+ ) : null}+ </div>+ );+}++/** The recipe's "Enhanced prompt" row: collapsible, with copy. */+export function EnhancedPromptRecipeRow({ caption }: { caption: string }) {+ const [open, setOpen] = useState(false);+ return (+ <div className="grid grid-cols-[72px_1fr] items-start gap-2">+ <button+ type="button"+ aria-expanded={open}+ onClick={() => setOpen((o) => !o)}+ className="flex items-start gap-0.5 text-left text-muted-foreground hover:text-foreground"+ >+ Enhanced prompt+ <ChevronDown className={cn("mt-0.5 size-3 shrink-0 transition-transform", open && "rotate-180")} />+ </button>+ <div className="flex min-w-0 flex-col gap-1">+ <span+ className={cn(+ "min-w-0 text-foreground",+ open ? "whitespace-pre-wrap break-words" : "truncate",+ )}+ >+ {caption}+ </span>+ {open ? (+ <div className="flex justify-end">+ <CopyButton text={caption} label="Copy enhanced prompt" />+ </div>+ ) : null}+ </div>+ </div>+ );+}diff --git a/studio/frontend/src/lib/media-generate-phase.ts b/studio/frontend/src/lib/media-generate-phase.tsindex abf691f76de..68d3f48f652 100644--- a/studio/frontend/src/lib/media-generate-phase.ts+++ b/studio/frontend/src/lib/media-generate-phase.ts@@ -2,7 +2,8 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0export interface MediaGeneratePhaseProgress {- // encode | denoise | decode | export (video); absent or null (sd.cpp) reads as denoise.+ // enhance (FLUX.3 prompt upsampler) | encode | denoise | decode | export (video); absent or null (sd.cpp) reads as+ // denoise.phase?: string | null;step: number;total: number;@@ -14,6 +15,7 @@ export function generatePhaseLabel(p: MediaGeneratePhaseProgress,opts: { hasAudio?: boolean; formatEta?: (seconds: number) => string } = {},): string {+ if (p.phase === "enhance") return "Enhancing prompt…";if (p.phase === "encode") return "Encoding prompt…";if (p.phase === "decode") return opts.hasAudio ? "Decoding video and audio…" : "Decoding…";if (p.phase === "export") return "Encoding video…";diff --git a/studio/frontend/tests/image-prompt-enhance.test.ts b/studio/frontend/tests/image-prompt-enhance.test.tsnew file mode 100644index 00000000000..a36abeb8f69--- /dev/null+++ b/studio/frontend/tests/image-prompt-enhance.test.ts@@ -0,0 +1,244 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import assert from "node:assert/strict";+import test from "node:test";++import { installLocalStorageFake, readSrc } from "./helpers/kit.ts";+import {+ ENHANCE_PROMPT_HINT,+ enhancePromptRequestField,+ enhanceRestoreFor,+ enhancementNotice,+ ENHANCE_UNSUPPORTED_FAMILY,+ overrideAfterFamilyChange,+ previewSignature,+ hasEnhancedPrompt,+ parsePromptUpsamplerStatus,+ PROMPT_UPSAMPLER_ROUTES,+ promptUpsamplerCapability,+ readEnhanceChoices,+ resolveEnhancePrompt,+ upsamplerDownloadBytes,+ upsamplerDownloadFraction,+ upsamplerNeedsDownload,+ upsamplerVariantFor,+ withEnhanceChoice,+} from "../src/features/images/lib/prompt-enhance.ts";+import { importedRecipe, recipeSettings } from "../src/features/images/lib/image-metadata.ts";++const FLUX3 = { loaded: true, family: "flux.3-klein", supports_prompt_upsampler: true, prompt_upsampler_default: true };+const SDXL = { loaded: true, family: "sdxl", supports_prompt_upsampler: false, prompt_upsampler_default: false };++test("the toggle defaults to the family default and is off for families without an upsampler", () => {+ const flux = promptUpsamplerCapability(FLUX3);+ assert.deepEqual(flux, { supported: true, defaultOn: true });+ assert.equal(resolveEnhancePrompt({ capability: flux, stored: undefined }), true);++ // A supporting family whose default is off.+ const offByDefault = promptUpsamplerCapability({ ...FLUX3, prompt_upsampler_default: false });+ assert.equal(resolveEnhancePrompt({ capability: offByDefault, stored: undefined }), false);++ for (const status of [SDXL, null, undefined, { loaded: true }, { ...FLUX3, loaded: false }]) {+ const cap = promptUpsamplerCapability(status);+ assert.equal(cap.supported, false);+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: true, override: true }), false);+ }+ // An external image connection never runs the local upsampler.+ assert.equal(promptUpsamplerCapability(FLUX3, true).supported, false);+});++test("a stored choice beats the default, and a restore override beats both", () => {+ const cap = promptUpsamplerCapability(FLUX3);+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: false }), false);+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: false, override: true }), true);+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: true, override: false }), false);+ assert.equal(resolveEnhancePrompt({ capability: cap, stored: undefined, override: null }), true);+});++test("the choice persists per family and a choice equal to the default is not stored", () => {+ const { store } = installLocalStorageFake();+ assert.deepEqual(readEnhanceChoices(), {});+ let choices = withEnhanceChoice({}, "flux.3-klein", false, true);+ assert.deepEqual(choices, { "flux.3-klein": false });+ choices = withEnhanceChoice(choices, "flux.3-klein-base", false, true);+ assert.deepEqual(readEnhanceChoices(), { "flux.3-klein": false, "flux.3-klein-base": false });+ choices = withEnhanceChoice(choices, "flux.3-klein", true, true);+ assert.deepEqual(readEnhanceChoices(), { "flux.3-klein-base": false });+ // No family key: nothing is written.+ assert.equal(withEnhanceChoice(choices, "", true, false), choices);+ // Corrupt storage reads as empty, non-boolean entries are dropped.+ store.set("unsloth:images:enhance-prompt-by-family", "not json");+ assert.deepEqual(readEnhanceChoices(), {});+ store.set("unsloth:images:enhance-prompt-by-family", JSON.stringify({ a: true, b: "yes", c: 1 }));+ assert.deepEqual(readEnhanceChoices(), { a: true });+});++test("enhance_prompt is sent only when the loaded family supports it", () => {+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(FLUX3), true), { enhance_prompt: true });+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(FLUX3), false), { enhance_prompt: false });+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(SDXL), false), {});+ assert.deepEqual(enhancePromptRequestField(promptUpsamplerCapability(null), true), {});+ // Wired into the generate request.+ const page = readSrc("features/images/images-page.tsx");+ const call = page.slice(page.indexOf("res = await generateDiffusionImage({"));+ assert.match(call.slice(0, 4000), /\.\.\.enhancePromptRequestField\(upsamplerCapability, enhanceOn\)/);+});++test("recipe restore: an applied run restores the user's prompt with the toggle on, or the caption with it off", () => {+ const applied = {+ prompt: "a cat",+ enhanced_prompt: "A photograph of a ginger cat sitting on a windowsill.",+ prompt_enhancement: "applied",+ };+ assert.deepEqual(enhanceRestoreFor(applied), { prompt: "a cat", enhance: true });+ assert.deepEqual(enhanceRestoreFor(applied, true), {+ prompt: "A photograph of a ginger cat sitting on a windowsill.",+ enhance: false,+ });+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: "off" }), { prompt: "a cat", enhance: false });+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: "unavailable" }, true), {+ prompt: "a cat",+ enhance: true,+ });+ // An older record, or another family's, leaves the toggle alone.+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat" }), { prompt: "a cat", enhance: null });+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", prompt_enhancement: null }, true), {+ prompt: "a cat",+ enhance: null,+ });+ // Exact with no caption falls back to the user's prompt.+ assert.deepEqual(enhanceRestoreFor({ prompt: "a cat", enhanced_prompt: " ", prompt_enhancement: "applied" }, true), {+ prompt: "a cat",+ enhance: true,+ });+ assert.equal(hasEnhancedPrompt(applied), true);+ assert.equal(hasEnhancedPrompt({ prompt: "a cat", enhanced_prompt: "a cat" }), false);+ assert.equal(hasEnhancedPrompt({ prompt: "a cat" }), false);+});++test("an embedded recipe carries the enhancement fields through to restore", () => {+ const recipe = importedRecipe({+ recipe: JSON.stringify({+ prompt: "a cat",+ enhanced_prompt: "A ginger cat.",+ prompt_enhancement: "applied",+ width: 1024,+ height: 1024,+ steps: 16,+ guidance: 1,+ seed: 3,+ }),+ });+ assert.ok(recipe);+ const settings = recipeSettings(recipe, { width: 1024, height: 1024, steps: 8, guidance: 0 });+ assert.equal(settings.enhanced_prompt, "A ginger cat.");+ assert.equal(settings.prompt_enhancement, "applied");+ const bogus = importedRecipe({ recipe: JSON.stringify({ prompt: "x", prompt_enhancement: "maybe" }) });+ assert.ok(bogus);+ assert.equal(recipeSettings(bogus, { width: 1, height: 1, steps: 1, guidance: 0 }).prompt_enhancement, null);+});++test("status parsing tolerates missing and malformed fields", () => {+ const full = parsePromptUpsamplerStatus({+ supported: true,+ downloaded: false,+ loaded: false,+ repo: "unsloth/FLUX.3-prompt-upsampler-GGUF",+ filename: "upsampler-Q4_K_M.gguf",+ size_bytes: 5_600_000_000,+ mmproj_size_bytes: 900_000_000,+ mmproj_downloaded: false,+ download: { downloading: true, model: "x", error: null, bytes_total: 200, bytes_done: 50, download_id: "d1" },+ });+ assert.equal(full.supported, true);+ assert.equal(full.size_bytes, 5_600_000_000);+ assert.equal(full.download.downloading, true);+ assert.equal(full.download.download_id, "d1");+ assert.equal(upsamplerDownloadFraction(full.download), 0.25);+ // The button fetches the vision projector too, until it is on disk.+ assert.equal(upsamplerDownloadBytes(full), 6_500_000_000);+ assert.equal(upsamplerDownloadBytes({ ...full, mmproj_downloaded: true }), 5_600_000_000);+ // Model cached, projector missing: Download stays offered and fetches only the projector.+ const noProjector = { ...full, downloaded: true };+ assert.equal(upsamplerNeedsDownload(noProjector), true);+ assert.equal(upsamplerDownloadBytes(noProjector), 900_000_000);+ assert.equal(upsamplerNeedsDownload({ ...noProjector, mmproj_downloaded: true }), false);+ assert.equal(upsamplerNeedsDownload({ ...full, supported: false }), false);++ const empty = parsePromptUpsamplerStatus(null);+ assert.deepEqual(+ { supported: empty.supported, downloaded: empty.downloaded, size: empty.size_bytes, repo: empty.repo },+ { supported: false, downloaded: false, size: 0, repo: "" },+ );+ assert.equal(empty.download.downloading, false);+ assert.equal(upsamplerDownloadFraction(empty.download), null);++ const junk = parsePromptUpsamplerStatus({ supported: "yes", size_bytes: -4, download: "nope" });+ assert.equal(junk.supported, false);+ assert.equal(junk.size_bytes, 0);+ assert.equal(junk.download.bytes_total, null);+ assert.equal(+ upsamplerDownloadFraction({ ...full.download, bytes_total: 100, bytes_done: 500 }),+ 1,+ );+});++test("routes live in one place; variant and notices", () => {+ for (const route of Object.values(PROMPT_UPSAMPLER_ROUTES)) {+ assert.match(route, /^\/api\/inference\/images\/prompt-upsampler\//);+ }+ const api = readSrc("features/images/api.ts");+ assert.doesNotMatch(api, /"\/api\/inference\/images\/prompt-upsampler/);+ assert.equal(upsamplerVariantFor("flux.3-klein-base"), "base");+ assert.equal(upsamplerVariantFor("flux.3-klein"), "distilled");+ assert.equal(upsamplerVariantFor(null), "distilled");+ assert.ok(enhancementNotice("unavailable"));+ assert.ok(enhancementNotice("failed"));+ assert.equal(enhancementNotice("applied"), null);+ assert.equal(enhancementNotice("off"), null);+ assert.equal(enhancementNotice(undefined), null);+ // No em dashes in the copy.+ assert.doesNotMatch(ENHANCE_PROMPT_HINT, /—/);+ assert.doesNotMatch(readSrc("features/images/prompt-enhance-panel.tsx"), /—/);+ assert.doesNotMatch(readSrc("features/images/lib/prompt-enhance.ts"), /—/);+});++test("a preview caption goes stale when the prompt, the variant or a reference image changes", () => {+ const base = { prompt: "make it night", variant: "distilled" as const, reference_images: ["data:image/png;base64,AAAA"] };+ const sig = previewSignature(base);+ assert.equal(previewSignature({ ...base, prompt: " make it night " }), sig);+ assert.notEqual(previewSignature({ ...base, prompt: "make it day" }), sig);+ assert.notEqual(previewSignature({ ...base, variant: "base" }), sig);+ assert.notEqual(previewSignature({ ...base, reference_images: ["data:image/png;base64,BBBB"] }), sig);+ assert.notEqual(previewSignature({ prompt: base.prompt, variant: base.variant }), sig);+ // Same length and the same 48 characters at each end: only the body differs.+ const head = "data:image/png;base64," + "A".repeat(60);+ const tail = "Z".repeat(60);+ assert.notEqual(+ previewSignature({ ...base, reference_images: [head + "QQQQ" + tail] }),+ previewSignature({ ...base, reference_images: [head + "RRRR" + tail] }),+ );+});++test("an unavailable outcome does not claim the weights are missing, and overrides offer no download", () => {+ assert.match(enhancementNotice("unavailable")!.description, /llama\.cpp is unavailable/);+ const status = parsePromptUpsamplerStatus({ supported: true, downloaded: true, mmproj_downloaded: false, override: true });+ assert.equal(status.override, true);+ assert.equal(upsamplerNeedsDownload(status), false);+});++test("a family change consumes the one-off override instead of leaving it dormant", () => {+ const unloaded = { family: "", on: false };+ // Set before any model loaded: binds to the first FLUX.3 family that loads.+ const bound = overrideAfterFamilyChange(unloaded, "flux.3-klein");+ assert.deepEqual(bound, { family: "flux.3-klein", on: false });+ assert.equal(overrideAfterFamilyChange(bound, "flux.3-klein"), bound);+ // A switch to the other variant drops it, so switching back does not revive it.+ assert.equal(overrideAfterFamilyChange(bound, "flux.3-klein-base"), null);+ assert.equal(overrideAfterFamilyChange(null, "flux.3-klein"), null);+ // A model without an upsampler consumes a bound override but keeps one set before any model loaded.+ assert.equal(overrideAfterFamilyChange(bound, ENHANCE_UNSUPPORTED_FAMILY), null);+ assert.equal(overrideAfterFamilyChange(unloaded, ENHANCE_UNSUPPORTED_FAMILY), unloaded);+});+diff --git a/studio/frontend/tests/image-restore-negative-prompt.test.ts b/studio/frontend/tests/image-restore-negative-prompt.test.tsindex 616fdb95eb5..baed9c65431 100644--- a/studio/frontend/tests/image-restore-negative-prompt.test.ts+++ b/studio/frontend/tests/image-restore-negative-prompt.test.ts@@ -5,6 +5,7 @@ import assert from "node:assert/strict";import test from "node:test";import ts from "@typescript/typescript6";import { readSrc } from "./helpers/kit.ts";+import { enhanceRestoreFor } from "../src/features/images/lib/prompt-enhance.ts";function restoreSettingsWith(negativeCapable: boolean) {const source = readSrc("features/images/images-page.tsx");@@ -46,6 +47,8 @@ function restoreSettingsWith(negativeCapable: boolean) {AUTOMATIC_SCHEDULE: "automatic",samplerOptions: [],changeSamplerChoice: () => {},+ // Its return value is read, so it cannot be a recording no-op.+ enhanceRestoreFor,toast: { success: () => {} },};// Every closure name the callback reads that is not a real global resolves to a recording no-op,@@ -82,3 +85,30 @@ test("restoring an image keeps its negative prompt while a model that ignores on});assert.equal(state.setNegativePrompt, "blurry, text");});++test("restoring an enhanced FLUX.3 image sets the Enhance prompt toggle from its outcome", () => {+ const image = {+ prompt: "a cat",+ enhanced_prompt: "A ginger cat on a windowsill.",+ prompt_enhancement: "applied",+ guidance: 1,+ steps: 16,+ seed: 7,+ width: 1024,+ height: 1024,+ workflow: "create",+ };+ const asTyped = restoreSettingsWith(true);+ asTyped.restore(image);+ assert.deepEqual(asTyped.state.setPromptFor, ["create", "a cat"]);+ assert.equal(asTyped.state.setEnhanceOverride, true);++ const exact = restoreSettingsWith(true);+ exact.restore(image, true);+ assert.deepEqual(exact.state.setPromptFor, ["create", "A ginger cat on a windowsill."]);+ assert.equal(exact.state.setEnhanceOverride, false);++ const legacy = restoreSettingsWith(true);+ legacy.restore({ ...image, enhanced_prompt: undefined, prompt_enhancement: undefined });+ assert.equal("setEnhanceOverride" in legacy.state, false);+});diff --git a/studio/frontend/tests/media-generate-phase.test.ts b/studio/frontend/tests/media-generate-phase.test.tsindex 7606ca3f6c9..44579db104a 100644--- a/studio/frontend/tests/media-generate-phase.test.ts+++ b/studio/frontend/tests/media-generate-phase.test.ts@@ -38,3 +38,7 @@ test("a new preview re-renders even when nothing else moved", () => {assert.equal(sameGenerateProgress(a, { ...a, preview_seq: 2 }), false);assert.equal(sameGenerateProgress({ step: 1, phase: "encode" }, { step: 1, phase: "denoise" }), false);});++test("the FLUX.3 enhance phase is labelled as such, not as denoising", () => {+ assert.equal(generatePhaseLabel({ phase: "enhance", step: 0, total: 16 }), "Enhancing prompt…");+});