Skip to content

The all-in-one local AI studio for your desktop: chat, image and video generation and a coding agent in one free, open source app. Windows and Linux. No Docker, no terminal, no cloud required.

13.7 KB
Raw Blame
"""MLX-Stable-Diffusion FastAPI sidecar.

Run by `commands::mlx::mlx_start` once the install slot has filled the
venv. Hosts a single POST /generate endpoint that the bridge proxies for
the web UI.

Model-agnostic: every request carries the full load config (repo, dtype,
fp16 variant, cfg kwarg name) resolved by the bridge from its image
catalog (commands/mlx.rs IMAGE_CATALOG). The sidecar keeps exactly one
pipeline in memory and swaps it out when a request names a different
repo — unified memory can't hold two big pipelines at once.
"""

import base64
import gc
import inspect
import io
import os
import socket
import sys
import threading
import time
from typing import Optional



def _fatal(message: str, code: int):
    """Say in one line why the server cannot run, then leave.

    The app reads this line from the log when the process ends and shows it,
    instead of waiting a minute on a port nobody listens on. Without it the
    reason was a traceback only the log file ever saw.
    """
    print(f"LU_MLX_FATAL: {message}", file=sys.stderr, flush=True)
    sys.exit(code)


try:
    import uvicorn
    from fastapi import FastAPI, HTTPException
    from pydantic import BaseModel
except ImportError as exc:
    # A venv whose pip run never finished. Exit code 4 and the words
    # "missing package" are what the app keys on to offer the setup again.
    _fatal(f"missing package: {exc.name or exc}", 4)


# Lazy: importing torch + diffusers takes ~3 s on a cold start. We defer
# it until the first request so /health stays sub-millisecond, which the
# bridge uses for the "is the MLX server up?" probe in mlx_status.
_pipe = None
_pipe_repo = None
# One lock guards every pipeline mutation (load / release / generate) so the
# idle-unload thread can't free the model out from under an in-flight request,
# and two concurrent requests can't both trigger a load.
_pipe_lock = threading.RLock()
_last_activity = time.monotonic()

# Cancel. A request cannot be taken back, so the app asks over /cancel and
# the generation looks at this mark after every step. It is set only while
# an image is under way and cleared when the last one has left, so a cancel
# that arrives late never ends the image after it.
_cancel = threading.Event()
_active = 0
_active_lock = threading.Lock()


class _Cancelled(Exception):
    """Raised from the step callback to leave the pipeline between two steps."""


def _stop_if_cancelled(pipe, step, timestep, callback_kwargs):
    # diffusers calls this at the end of each denoising step and takes back
    # the tensors it handed over. Raising is the one way out that every
    # pipeline with this hook shares: `pipe._interrupt` exists on some only,
    # and those still decode what they have.
    if _cancel.is_set():
        raise _Cancelled()
    return callback_kwargs


# How far the image is (C71). The app used to give a whole request 900 s,
# model load included, and Chroma1-HD at 40 steps and 1024 on an M1 Max needs
# longer than that: the render was cut off and reported as "mlx unreachable".
# The app now asks here and ends a run only when nothing has moved for its
# limit. `changed` is when the phase or the step last moved.
_progress_lock = threading.Lock()
_progress = {"phase": "idle", "step": 0, "steps": 0, "step_hook": False, "changed": time.monotonic()}


def _note_progress(phase=None, step=None, steps=None, step_hook=None):
    with _progress_lock:
        if phase is not None:
            _progress["phase"] = phase
        if step is not None:
            _progress["step"] = step
        if steps is not None:
            _progress["steps"] = steps
        if step_hook is not None:
            _progress["step_hook"] = step_hook
        _progress["changed"] = time.monotonic()


def _on_step_end(pipe, step, timestep, callback_kwargs):
    # `step` counts from 0 and the call comes at the end of that step.
    _note_progress(step=step + 1)
    return _stop_if_cancelled(pipe, step, timestep, callback_kwargs)


def _takes_step_callback(pipe) -> bool:
    """Whether this pipeline calls back after a step. One that does not runs
    to its end, and the app drops what it returns."""
    try:
        return "callback_on_step_end" in inspect.signature(pipe.__call__).parameters
    except (TypeError, ValueError):
        return False


# A resident image pipeline pins multiple GB of unified memory. On a local
# machine that must not linger forever after the user walks away, so a
# background thread releases it once idle. 0 disables (set via env for tests).
IDLE_UNLOAD_SECONDS = int(os.environ.get("LU_MLX_IDLE_UNLOAD_SECONDS", "600"))


def _release_pipe():
    global _pipe, _pipe_repo
    with _pipe_lock:
        if _pipe is None:
            return
        import torch

        _pipe = None
        _pipe_repo = None
        gc.collect()
        if torch.backends.mps.is_available():
            torch.mps.empty_cache()


def _idle_unload_loop():
    """Free the resident pipeline after IDLE_UNLOAD_SECONDS of no requests.
    Keeps the server itself up (fast to reload), just drops the weights."""
    while True:
        time.sleep(30)
        if IDLE_UNLOAD_SECONDS <= 0 or _pipe is None:
            continue
        if time.monotonic() - _last_activity >= IDLE_UNLOAD_SECONDS:
            _release_pipe()


def _ensure_pipe(
    repo: str,
    dtype: str,
    variant: Optional[str],
    disable_safety_checker: bool,
    local_files_only: bool = False,
):
    global _pipe, _pipe_repo
    # Caller already holds _pipe_lock.
    if _pipe is not None and _pipe_repo == repo:
        return _pipe
    _release_pipe()

    import torch
    from diffusers import AutoPipelineForText2Image

    device = "mps" if torch.backends.mps.is_available() else "cpu"
    torch_dtype = torch.float32
    if device == "mps":
        torch_dtype = torch.bfloat16 if dtype == "bfloat16" else torch.float16

    kwargs = {"torch_dtype": torch_dtype}
    # A model the app already installed must load from disk alone. Without
    # this, diffusers re-validates all ~18 files against the hub on every
    # load: measured at 26 s per already-present file when the hub throttles
    # anonymous traffic, and an outright failure with no network. "Local"
    # image generation silently depended on being online.
    if local_files_only:
        kwargs["local_files_only"] = True
    if variant:
        kwargs["variant"] = variant
    if disable_safety_checker:
        # SD1.5 repos bundle a safety_checker component; skipping it here
        # also means the install never had to download its weights.
        kwargs["safety_checker"] = None
        kwargs["requires_safety_checker"] = False

    # No attention/VAE slicing here: on torch 2.12 + MPS it corrupts the
    # forward pass into all-NaN latents (verified live — plain runs are
    # clean, sliced runs decode to black frames).
    _pipe = _without_checker(AutoPipelineForText2Image.from_pretrained(repo, **kwargs).to(device))
    _pipe_repo = repo
    return _pipe


def _without_checker(pipe):
    """No image of this app goes through the diffusers safety checker. It
    blacks out what it reads as nudity, an adult picture included, and says
    nothing. LU's own gate refuses a prompt that sexualizes minors before any
    engine starts (lib/render/safety.ts), on every model.

    The catalog's flag skips the checker at load, where a repo bundles one
    (it also spares its download). On 13.11.2026 no catalog repo shipped
    checker weights, but the install follows each repo's main branch, so
    one that adds them later is caught here, for every model alike."""
    if getattr(pipe, "safety_checker", None) is not None:
        pipe.safety_checker = None
    return pipe


app = FastAPI(title="LU MLX server")


class GenerateRequest(BaseModel):
    prompt: str
    steps: int = 4
    seed: Optional[int] = None
    width: int = 512
    height: int = 512
    negative_prompt: Optional[str] = None
    # Model config resolved by the bridge catalog. Defaults keep the
    # historical sd-turbo behaviour for old callers.
    model_repo: str = "stabilityai/sd-turbo"
    dtype: str = "float16"
    variant: Optional[str] = "fp16"
    guidance: float = 0.0
    cfg_param: str = "guidance_scale"
    disable_safety_checker: bool = False
    # Set by the bridge when it has verified the model is fully on disk.
    local_files_only: bool = False


@app.get("/health")
def health():
    idle = None if _pipe is None else round(time.monotonic() - _last_activity, 1)
    return {
        "ok": True,
        "model_loaded": _pipe is not None,
        "model_repo": _pipe_repo,
        "idle_seconds": idle,
        "idle_unload_seconds": IDLE_UNLOAD_SECONDS,
    }


@app.get("/progress")
async def progress():
    """Where the image under way is: `phase` is idle, waiting, loading, steps
    or saving; `step` of `steps` once the steps run (only a pipeline with the
    step hook counts them, `step_hook`); `quiet_seconds` since anything moved.

    `async` on purpose: it is answered on the event loop itself and never
    waits for a worker thread next to the image. Copying the dict needs no
    lock: the writers replace whole values."""
    snap = dict(_progress)
    snap["quiet_seconds"] = round(time.monotonic() - snap.pop("changed"), 1)
    return snap


@app.post("/unload")
def unload():
    """Free the resident model now, keeping the server up. Idempotent — the
    bridge's `mlx_unload` calls this so the UI can reclaim memory on demand."""
    was_loaded = _pipe is not None
    _release_pipe()
    return {"ok": True, "was_loaded": was_loaded}


@app.post("/cancel")
def cancel():
    """End the image that is being generated, at the end of the step it is
    in. `cancelled` is false when there is none: nothing is remembered then."""
    with _active_lock:
        running = _active > 0
        if running:
            _cancel.set()
    return {"ok": True, "cancelled": running}


def _cancelled_answer():
    # 409 and this word are what the app reads as "ended because it was asked to".
    return HTTPException(status_code=409, detail="cancelled")


@app.post("/generate")
def generate(req: GenerateRequest):
    global _active
    with _active_lock:
        _active += 1
    _note_progress(phase="waiting", step=0, steps=req.steps, step_hook=False)
    try:
        return _generate(req)
    finally:
        with _active_lock:
            _active -= 1
            if _active == 0:
                _cancel.clear()
                _note_progress(phase="idle", step=0, steps=0, step_hook=False)


def _generate(req: GenerateRequest):
    global _last_activity
    import torch

    with _pipe_lock:
        # Cancelled while it waited for the image before it.
        if _cancel.is_set():
            raise _cancelled_answer()
        _note_progress(phase="loading")
        try:
            pipe = _ensure_pipe(
                req.model_repo,
                req.dtype,
                req.variant,
                req.disable_safety_checker,
                req.local_files_only,
            )
        except Exception as exc:
            # A half-loaded pipeline must not shadow the previous model.
            _release_pipe()
            raise HTTPException(status_code=500, detail=f"pipeline init: {exc}") from exc

        # MPS generators are unstable; seeds work better via a CPU generator.
        generator = None
        if req.seed is not None:
            generator = torch.Generator(device="cpu").manual_seed(req.seed)

        gen_kwargs = {
            "prompt": req.prompt,
            "num_inference_steps": req.steps,
            "width": req.width,
            "height": req.height,
            "generator": generator,
            # Turbo/distilled models want 0.0 here; regular models their CFG.
            # Qwen-Image reads "true_cfg_scale" instead of "guidance_scale".
            req.cfg_param: req.guidance,
        }
        if req.negative_prompt:
            gen_kwargs["negative_prompt"] = req.negative_prompt
        hook = _takes_step_callback(pipe)
        if hook:
            gen_kwargs["callback_on_step_end"] = _on_step_end
        _note_progress(phase="steps", step=0, steps=req.steps, step_hook=hook)

        # Cancelled while the model loaded: no step is run for it.
        if _cancel.is_set():
            raise _cancelled_answer()
        try:
            result = pipe(**gen_kwargs)
        except _Cancelled:
            raise _cancelled_answer() from None
        except Exception as exc:
            raise HTTPException(status_code=500, detail=f"generate: {exc}") from exc
        finally:
            # Reset the idle clock AFTER the work, so a slow generation can't be
            # unloaded mid-run and the countdown starts from completion.
            _last_activity = time.monotonic()

    _note_progress(phase="saving")
    img = result.images[0]
    buf = io.BytesIO()
    img.save(buf, format="PNG")
    return {
        "image_base64": base64.b64encode(buf.getvalue()).decode("ascii"),
        "width": req.width,
        "height": req.height,
        "steps": req.steps,
        "model_repo": req.model_repo,
    }


if __name__ == "__main__":
    port = int(os.environ.get("LU_MLX_PORT", "47712"))
    # uvicorn answers an occupied port with a log line and its own
    # sys.exit(1), which no except clause here would see. Asking first turns
    # that into a sentence.
    probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        probe.bind(("127.0.0.1", port))
    except OSError:
        _fatal(f"port {port} is already in use", 3)
    finally:
        probe.close()
    threading.Thread(target=_idle_unload_loop, daemon=True).start()
    try:
        uvicorn.run(app, host="127.0.0.1", port=port, log_level="warning")
    except KeyboardInterrupt:
        sys.exit(0)
    except Exception as exc:  # noqa: BLE001 - the last word before the process ends
        _fatal(f"server stopped: {exc}", 1)