Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

48.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Load and unload models per kind. A load (a big download can take an hour) runs as a job in the job store, so a client that stops waiting never stops it: load_model waits up to ``wait_s`` while reporting progress, then returns the job id for get_job(kind="load")."""

from __future__ import annotations

import asyncio
import dataclasses
import hashlib
import itertools
import os
import time
import uuid
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Optional

from fastmcp import Context
from fastmcp.exceptions import ToolError

from core.inference import model_identity
from studio_mcp import job_store
from studio_mcp import placement as place
from studio_mcp.caller import Caller
from studio_mcp.errors import raise_for_route, tool_error
from studio_mcp.forward import forward
from studio_mcp.outputs import EvictedSlot, LoadResult, SlotModel, UnloadAllResult, UnloadResult
from studio_mcp.tools import as_dict, number, opt_text, route_json, strings, try_json

POLL_INTERVAL_S = 2.0
# The longest a load_model or get_job call waits before returning (user decision); clients must allow more.
MAX_WAIT_S = 240
# The schema bound for wait_s. Tests shorten MAX_WAIT_S to fractions of a second, and get_job's schema can be
# built while they do (its annotations are lazy), so the bound must not read the patchable value.
WAIT_S_LIMIT = 240
# What a caller still waiting on a job is told to wait before its next get_job without wait_s.
POLL_AFTER_S = 10
# Forwarded reads have no HTTP timeout; a status or progress read taking longer counts as failed.
READ_S = 15.0

Progress = Optional[tuple[float, str]]


async def with_progress(
    ctx: Optional[Context],
    work: Awaitable[Any],
    poll: Callable[[], Awaitable[Progress]],
    job: Optional[LoadJob] = None,
) -> Any:
    """Await ``work`` while reporting what ``poll`` sees, to the agent and onto ``job``. A failed poll is skipped, never fatal."""
    task = asyncio.ensure_future(work)
    try:
        while True:
            done, _pending = await asyncio.wait({task}, timeout = POLL_INTERVAL_S)
            if done:
                return task.result()
            try:
                progress = await asyncio.wait_for(poll(), timeout = READ_S)
            except Exception:
                progress = None
            if progress is not None and job is not None:
                job.note(*progress)
            if progress is not None and ctx is not None:
                await ctx.report_progress(progress[0], 1.0, progress[1])
    finally:
        if not task.done():
            task.cancel()


@dataclass
class LoadJob(job_store.Job):
    kind: str = "load"
    # The slot kind asked for (llm, tts, audio, image, video) and the model as asked.
    slot: str = "llm"
    model: str = ""
    variant: Optional[str] = None
    # Any other load settings; a repeat call with different ones starts its own load.
    options: Any = None
    fraction: Optional[float] = None
    # Each distinct (fraction, phase) the load went through, for every caller following it.
    marks: list[tuple[float, str]] = field(default_factory = list, repr = False)
    # The STT download this load started, so unload_model can stop it, and whether the STT load
    # request is out (it then runs to the end: cancelling its in-process route does not stop it).
    download: Optional[dict] = None
    posted: bool = False
    # A media load POST, shielded: cancelling the job leaves it running, since the route registers its load from a thread.
    post: Optional[asyncio.Future] = field(default = None, repr = False)
    # The media load's load_token: an unload naming it ends this load only, even if its POST is still out.
    load_token: Optional[str] = None

    def note(self, fraction: Optional[float], phase: str) -> None:
        # A step without a fraction keeps the last one, so progress never drops back.
        self.phase = phase
        self.fraction = self.fraction if fraction is None else fraction
        if fraction is not None and (not self.marks or self.marks[-1] != (fraction, phase)):
            self.marks.append((fraction, phase))

    async def report_progress(
        self,
        progress: float,
        total: Optional[float] = None,
        message: Optional[str] = None,
    ) -> None:
        """Stands in for the agent's Context, so load code that reports progress records it on the job."""
        self.note(progress, message or self.phase)


LOADS = job_store.JobStore(
    capacity = 32, unknown = "No such load job", failure = "The load stopped unexpectedly."
)


def deadline_after(wait_s: Optional[float]) -> float:
    """When a wait of ``wait_s`` seconds (None: the longest) that starts now ends, on the monotonic clock."""
    wait = MAX_WAIT_S if wait_s is None else max(0, min(wait_s, MAX_WAIT_S))
    return time.monotonic() + wait


def _same(left: Optional[str], right: Optional[str]) -> bool:
    # A relative path that exists on disk keeps its case, as the loaders read it.
    return model_identity.same_model(left, right, probe = True)


def _same_id(left: str, right: str) -> bool:
    return model_identity.same_repo(left, right, probe = True)


def _in_flight(caller: Caller, asked: LoadJob) -> Optional[LoadJob]:
    for job in LOADS.running(caller.account_id):
        if (
            job.slot == asked.slot
            and _same(job.model, asked.model)
            and model_identity.same_variant(job.variant, asked.variant)
            and job.options == asked.options
        ):
            return job
    return None


def _stop_loads(
    caller: Caller,
    slots: Any,
    model: Optional[str] = None,
) -> list[LoadJob]:
    """Mark this caller's loads of ``slots`` still running in the job store cancelled. cancel has no load kind, so unload_model is how an agent stops one; the caller's unload route stops the load server-side."""
    jobs = [
        job
        for job in LOADS.running(caller.account_id)
        if job.slot in slots and (model is None or _same_id(job.model, model))
    ]
    for job in jobs:
        job_store.mark_cancelled(job)
    return jobs


def _early(job: LoadJob) -> LoadResult:
    return LoadResult(
        kind = job.slot,
        model = job.model,
        loaded = False,
        status = "loading",
        fraction = job.fraction,
        job_id = job.job_id,
        poll_after_s = POLL_AFTER_S,
    )


def outcome(job: LoadJob) -> LoadResult:
    """A finished load's result; a failed or cancelled one raises its error."""
    if job.status == "completed" and isinstance(job.result, LoadResult):
        return job.result
    if job.status == "cancelled":
        raise ToolError(f"The load of {job.model} was cancelled")
    raise ToolError(job.error or f"Unsloth Studio did not load {job.model}")


async def wait_load(ctx: Optional[Context], job: LoadJob, deadline: float) -> LoadResult:
    """Wait for ``job`` until ``deadline`` (monotonic), reporting each step it makes; never cancels it."""
    seen = 0 if job.task is None or not job.marks else len(job.marks) - 1
    while True:
        if ctx is not None:
            for fraction, phase in job.marks[seen:]:
                await ctx.report_progress(min(fraction, 1.0), 1.0, phase)
        seen = len(job.marks)
        left = deadline - time.monotonic()
        if job.finished or job.task is None or left <= 0:
            break
        await asyncio.wait({job.task}, timeout = min(POLL_INTERVAL_S, left))
    return outcome(job) if job.finished else _early(job)


async def start_load(
    caller: Caller,
    ctx: Optional[Context],
    *,
    slot: str,
    model: str,
    variant: Optional[str],
    deadline: float,
    run: Callable[[LoadJob, dict[str, list[str]]], Awaitable[LoadResult]],
    options: Any = None,
    hub_token: Optional[str] = None,
    attach: bool = True,
) -> LoadResult:
    """Run a load as a job and wait for it until ``deadline``. A repeat call for the same load while it runs follows it; one with another Hub token (a retry with a fixed token, say) starts its own."""
    token = hub_token or caller.hf_token
    asked = LoadJob(
        job_id = job_store.new_id(),
        account_id = caller.account_id,
        slot = slot,
        model = model,
        variant = variant,
        options = (options, hashlib.sha256(token.encode()).hexdigest()[:16] if token else None),
    )

    async def drive(job: LoadJob) -> LoadResult:
        # Every route read runs inside the job, so a slow one never holds the call past its deadline.
        before, chat = await _resident(caller)
        # A chat load Unsloth Studio is already running (started elsewhere) is followed, not started
        # again. The status names only the repo, so a load asking for a variant always posts.
        loading_now = strings(chat.get("loading"))
        if (
            slot in CHAT_ROUTE_KINDS
            and attach
            and not variant
            and any(_same(n, model) for n in loading_now)
        ):
            return await _watch_llm(caller, slot, model, before, job)
        # K16, after the attach: M5 waits on a download of the same model here, then M7 refuses a partial one.
        from studio_mcp import model_downloads

        # Inside the job, so the wait outlives wait_s: the load starts once the download ends.
        await model_downloads.wait_for_managed(
            caller,
            job,
            model = model,
            kind = slot,
            variant = variant,
            budget_s = model_downloads.MAX_LOAD_WAIT_S,
        )
        await place.refuse_partial_variant(caller, model, slot, variant)
        return await run(job, before)

    job = _in_flight(caller, asked) or LOADS.start(asked, drive)
    return await wait_load(ctx, job, deadline)


# The load_model kinds that load through the chat route.
CHAT_ROUTE_KINDS = ("llm", "tts", "audio")
# Route slot kinds as MCP kinds.
ROUTE_KINDS = {"chat": "llm", "audio": "audio", "image": "image", "video": "video"}


def _media_name(status: Any) -> Optional[str]:
    if not isinstance(status, dict) or status.get("loaded") is not True:
        return None
    return opt_text(status.get("display_repo_id")) or opt_text(status.get("repo_id"))


async def _read_json(caller: Caller, path: str) -> Any:
    """``try_json`` within READ_S: a read that stalls is None, like one that failed."""
    try:
        return await asyncio.wait_for(try_json(caller, path), timeout = READ_S)
    except asyncio.TimeoutError:
        return None


async def _resident(caller: Caller) -> tuple[dict[str, list[str]], dict]:
    chat, image, video = await asyncio.gather(
        _read_json(caller, "/api/inference/status"),
        _read_json(caller, MEDIA_ROUTES["image"]["status"]),
        _read_json(caller, MEDIA_ROUTES["video"]["status"]),
    )
    slots: dict[str, list[str]] = {}
    # A slot that could not be read is left out: this only feeds a report.
    if isinstance(chat, dict):
        slots["llm"] = list(
            dict.fromkeys([*strings(chat.get("serving")), *strings(chat.get("loaded"))])
        )
    for kind, status in (("image", image), ("video", video)):
        if isinstance(status, dict):
            name = _media_name(status)
            slots[kind] = [name] if name else []
    return slots, as_dict(chat)


async def resident_slots(caller: Caller) -> dict[str, list[str]]:
    """What each slot holds now, by MCP kind: llm (chat models), image and video."""
    return (await _resident(caller))[0]


def _slot_rows(rows: Any) -> list[EvictedSlot]:
    found = []
    for row in rows if isinstance(rows, list) else []:
        if isinstance(row, dict) and opt_text(row.get("model")):
            kind = opt_text(row.get("kind")) or "chat"
            found.append(EvictedSlot(kind = ROUTE_KINDS.get(kind, kind), model = row["model"]))
    return found


async def finish_load(
    caller: Caller, kind: str, model: str, before: dict[str, list[str]], answer: Any, **extras: Any
) -> LoadResult:
    """The LoadResult for a finished load, with what it unloaded: the load answer's ``evicted_slots`` (else its ``evicted`` ids placed by the ``before`` snapshot), plus a before/after diff for what the answer leaves out."""
    answer = as_dict(answer)
    named = strings(answer.get("evicted"))
    slots = _slot_rows(answer.get("evicted_slots"))
    if not slots:
        slots = [
            EvictedSlot(
                kind = next(
                    (k for k, models in before.items() if any(_same_id(name, m) for m in models)),
                    "llm",
                ),
                model = name,
            )
            for name in named
        ]
    # The route lists what it unloaded to make room, never the model this load replaced in its own
    # slot (and nothing at all before it had a list), so whatever else left is found by a diff.
    if any(before.values()):
        after = await resident_slots(caller)
        slots += [
            EvictedSlot(kind = slot, model = name)
            for slot, models in before.items()
            if slot in after
            for name in models
            if not any(_same(name, m) for m in after[slot])
            and not _same_id(name, model)
            and not any(_same_id(name, s.model) for s in slots)
        ]
    evicted = [slot.model for slot in slots] if slots else named
    return LoadResult(kind = kind, model = model, evicted = evicted, evicted_slots = slots, **extras)


def _llm_poll(caller: Caller, model: str) -> Callable[[], Awaitable[Progress]]:
    async def poll() -> Progress:
        loading = await route_json("GET", "/api/inference/load-progress", caller = caller)
        fraction = number(loading.get("fraction")) if isinstance(loading, dict) else None
        if fraction and opt_text(loading.get("phase")):
            return min(fraction, 1.0), f"Loading into memory ({loading['phase']})"
        if model_identity.is_local(model):
            return None
        download = await route_json(
            "GET",
            "/api/models/download-progress",
            caller = caller,
            params = {"repo_id": model},
            hub_header = True,
        )
        fraction = number(download.get("progress")) if isinstance(download, dict) else None
        return (min(fraction, 1.0), "Downloading") if fraction else None

    return poll


async def load_llm(
    caller: Caller,
    ctx: Optional[Context],
    *,
    model: str,
    variant: Optional[str],
    max_seq_length: Optional[int],
    load_in_4bit: bool,
    hf_token: Optional[str],
    kind: str = "llm",
    deadline: Optional[float] = None,
    device: Optional[str] = None,
    alongside: Optional[bool] = None,
    kv_cache_type: Optional[str] = None,
    gpu_layers: Optional[int] = None,
    is_lora: bool = False,
) -> LoadResult:
    body: dict[str, Any] = {
        "model_path": model,
        "load_in_4bit": load_in_4bit,
        "max_seq_length": max_seq_length or 0,
    }
    if variant:
        body["gguf_variant"] = variant
    if is_lora:
        body["is_lora"] = True
    # The load routes read the Hub token from the body, not the header.
    token = hf_token or caller.hf_token
    if token:
        body["hf_token"] = token
    if hf_token:
        # The variant lookups before the load read the Hub token from the header.
        caller = dataclasses.replace(caller, hf_token = hf_token)

    async def run(job: LoadJob, before: dict[str, list[str]]) -> LoadResult:
        # The GGUF listing and status reads run inside the job, so a slow one never holds the call
        # past ``wait_s``; device='cpu' for a chat model that is not a GGUF is refused here.
        gguf = await place.require_cpu_gguf(caller, kind, model, variant, device, gpu_layers)
        body.update(
            place.llm_fields(
                kind,
                gguf = gguf,
                device = device,
                alongside = await place.auto_alongside(caller, device, alongside, gpu_layers),
                kv_cache_type = kv_cache_type,
                gpu_layers = gpu_layers,
            )
        )
        payload = await with_progress(
            None,
            route_json("POST", "/api/inference/load", caller = caller, json_body = body),
            _llm_poll(caller, model),
            job,
        )
        if not isinstance(payload, dict):
            raise ToolError("Unsloth Studio did not confirm the load")
        loaded = opt_text(payload.get("model")) or model
        chat = before.get("llm") or []
        # Serving only: a model merely held in memory behind the new primary no longer serves.
        # Bounded: the model is loaded, and a stalled read must not keep the job open.
        status = await _read_json(caller, "/api/inference/status") if chat else None
        # A failed read leaves it unknown whether the previous model still serves.
        after = strings(status.get("serving")) if isinstance(status, dict) else None
        return await finish_load(
            caller,
            kind,
            loaded,
            before,
            payload,
            display_name = opt_text(payload.get("display_name")),
            placement = place.llm_placement(payload, device),
            alongside_applied = place.alongside_applied(chat, after, loaded),
        )

    return await start_load(
        caller,
        ctx,
        slot = kind,
        model = model,
        variant = variant,
        deadline = deadline or deadline_after(None),
        run = run,
        # A load asked with other placement fields is a different load, not one to follow.
        options = (
            max_seq_length or 0,
            load_in_4bit,
            device,
            alongside,
            kv_cache_type,
            gpu_layers,
        ),
        hub_token = token,
        # A load asked with its own settings is never the one already running; following it would
        # skip them (and the CPU check).
        attach = not max_seq_length
        and load_in_4bit
        and all(v is None for v in (device, alongside, kv_cache_type, gpu_layers)),
    )


async def _watch_llm(
    caller: Caller, kind: str, model: str, before: dict[str, list[str]], job: LoadJob
) -> LoadResult:
    """Follow a chat load Unsloth Studio is already running, until it serves ``model`` or stops loading it."""
    poll = _llm_poll(caller, model)
    misses = 0
    while True:
        await asyncio.sleep(POLL_INTERVAL_S)
        status = await _read_json(caller, "/api/inference/status")
        if not isinstance(status, dict):
            # A failed read is skipped; only a run of them ends the follow.
            misses += 1
            if misses >= _IDLE_POLLS:
                raise ToolError(f"Unsloth Studio stopped answering while loading {model}")
            continue
        misses = 0
        if not any(_same(name, model) for name in strings(status.get("loading"))):
            if any(_same_id(name, model) for name in strings(status.get("serving"))):
                return await finish_load(caller, kind, model, before, {})
            raise ToolError(f"Unsloth Studio stopped loading {model} before it was ready")
        try:
            progress = await asyncio.wait_for(poll(), timeout = READ_S)
        except Exception:
            progress = None
        if progress is not None:
            job.note(*progress)


async def _llm_resident(caller: Caller) -> tuple[list[str], list[str], set[str]]:
    status = as_dict(await route_json("GET", "/api/inference/status", caller = caller))
    serving, checkpoints = (
        strings(status.get("serving")),
        strings(status.get("serving_checkpoints")),
    )
    return serving, checkpoints, {*serving, *checkpoints, *strings(status.get("loaded"))}


async def unload_llm(
    caller: Caller,
    model: Optional[str],
    kind: str = "llm",
) -> UnloadResult:
    serving, checkpoints, before = await _llm_resident(caller)
    if model is None:
        if not checkpoints:
            return UnloadResult(kind = kind, unloaded = False)
        target, model = checkpoints[0], (serving[0] if serving else checkpoints[0])
    else:
        # The route unloads by checkpoint id, listed in the same order as serving.
        target = dict(zip(serving, checkpoints)).get(model, model)
    await route_json(
        "POST", "/api/inference/unload", caller = caller, json_body = {"model_path": target}
    )
    # An id that matches nothing still answers success, so compare what is resident before and after.
    _serving, _checkpoints, after = await _llm_resident(caller)
    was_loaded = bool({model, target} & before)
    return UnloadResult(
        kind = kind, model = model, unloaded = was_loaded and not ({model, target} & after)
    )


MEDIA_ROUTES = {
    kind: {
        "plan": f"/api/inference/{prefix}/download-plan",
        "load": f"/api/inference/{prefix}/load",
        "progress": f"/api/inference/{prefix}/load-progress",
        "status": f"/api/inference/{prefix}/status",
        "unload": f"/api/inference/{prefix}/unload",
    }
    for kind, prefix in (("image", "images"), ("video", "video"))
}
# Polls with no load in flight and the model still not resident before the load counts as lost.
_IDLE_POLLS = 5


def _media_fraction(progress: dict) -> Optional[float]:
    # Images report bytes_downloaded/bytes_total, video downloaded_bytes/expected_bytes.
    done = number(progress.get("bytes_downloaded", progress.get("downloaded_bytes")))
    total = number(progress.get("bytes_total", progress.get("expected_bytes")))
    if done is not None and total:
        return min(done / total, 1.0)
    return number(progress.get("fraction"))


def _resident_matches(
    status: Any,
    model: str,
    body: Optional[dict] = None,
    *,
    settled: bool = True,
) -> bool:
    if not isinstance(status, dict) or status.get("loaded") is not True:
        return False
    names = {opt_text(status.get("repo_id")), opt_text(status.get("display_repo_id"))}
    # Any case, paths too: status may spell the resident path differently from the request.
    if model.lower() not in {name.lower() for name in names if name}:
        return False
    # Another file of the same repo is another model: a missing field reads as not loaded yet.
    wanted = opt_text((body or {}).get("gguf_filename"))
    if not wanted:
        return True
    kind = opt_text((body or {}).get("model_kind"))
    if kind and opt_text(status.get("model_kind")) not in (None, kind):
        return False
    resident = opt_text(status.get("gguf_filename"))
    if resident:
        return os.path.normcase(resident) == os.path.normcase(wanted)
    # Video status names only a GGUF's quant; a plain checkpoint has none to compare.
    quant = model_identity.resident_quant_matches(
        opt_text(status.get("gguf_variant")), wanted, settled = settled
    )
    if quant is None:
        return True
    if not quant:
        return False
    # MiniMax-H3 ships its fl2va and ref2va partitions at the same quant; status names the one loaded.
    h3_task = opt_text(status.get("h3_task"))
    if not h3_task:
        return True
    try:
        from core.inference.video_minimax_h3 import h3_transformer_task
    except Exception:
        return False
    return h3_task.lower() == h3_transformer_task(wanted).lower()


async def _gguf_filename(caller: Caller, model: str, variant: Optional[str]) -> Optional[dict]:
    """The variant to load from a GGUF repo (the named one, else the repo's default), with its filename, size_bytes, downloaded and partial."""
    listing = await route_json(
        "GET", "/api/hub/gguf-variants", caller = caller, params = {"repo_id": model}, hub_header = True
    )
    entry = place.pick_variant(listing, variant)
    if entry is not None and opt_text(entry.get("filename")):
        return entry
    if variant:
        raise ToolError(f"{model} has no GGUF variant {variant}")
    return None


async def load_media(
    caller: Caller,
    ctx: Optional[Context],
    *,
    kind: str,
    model: str,
    variant: Optional[str],
    hf_token: Optional[str],
    deadline: Optional[float] = None,
    knobs: Optional[dict[str, str]] = None,
) -> LoadResult:
    routes = MEDIA_ROUTES[kind]
    knobs = knobs or {}
    if hf_token:
        # The GGUF lookup reads the Hub token from the header, so a gated repo needs it there too.
        caller = dataclasses.replace(caller, hf_token = hf_token)
    body: dict[str, Any] = {"model_path": model}
    token = hf_token or caller.hf_token
    if token:
        body["hf_token"] = token

    async def prepare() -> None:
        # Hub and plan reads run inside the job, so a slow one never holds the call past ``wait_s``.
        if variant or "gguf" in model.lower():
            entry = await _gguf_filename(caller, model, variant)
            if entry:
                body["gguf_filename"] = entry["filename"]
                body["model_kind"] = "gguf"
        # The plan validates the pick the way the load does, before any download.
        plan = await place.post_plan(caller, routes["plan"], body, knobs)
        if isinstance(plan, dict):
            if opt_text(plan.get("incompatible_reason")):
                raise tool_error(plan["incompatible_reason"])
            if "gguf_filename" not in body:
                for entry in plan.get("entries") or []:
                    if (
                        isinstance(entry, dict)
                        and entry.get("checkpoint")
                        and opt_text(entry.get("gguf_filename"))
                    ):
                        body["gguf_filename"] = entry["gguf_filename"]
                        break

    async def run(job: LoadJob, before: dict[str, list[str]]) -> LoadResult:
        await prepare()
        # An unload this process sent for the slot could cancel this load if it ran after it: post after it.
        while unloading := {u for u in _MEDIA_UNLOADING.get(kind, ()) if not u.done()}:
            await asyncio.wait(unloading)
        # Named before it goes out, so an unload can end this load even if the POST's answer never comes.
        job.load_token = uuid.uuid4().hex
        _remember_media_token(caller, kind, job.load_token)
        # The answer names what the arbiter unloaded to make room; the load itself goes on in the background.
        job.post = asyncio.ensure_future(
            _post_media_load(
                caller, kind, job.load_token, {**body, **knobs, "load_token": job.load_token}
            )
        )
        job.post.add_done_callback(lambda post: post.cancelled() or post.exception())
        answer = await with_progress(
            None, asyncio.shield(job.post), _media_poll(caller, routes), job
        )
        status = await _media_ready(caller, kind, model, routes, job, body)
        _media_token_ready(caller, kind, job.load_token)
        name = _media_name(status) or model
        return await finish_load(
            caller, kind, name, before, answer, placement = place.media_placement(status)
        )

    return await start_load(
        caller,
        ctx,
        slot = kind,
        model = model,
        variant = variant,
        deadline = deadline or deadline_after(None),
        run = run,
        options = tuple(sorted(knobs.items())),
    )


def _media_poll(caller: Caller, routes: dict[str, str]) -> Callable[[], Awaitable[Progress]]:
    # While the load POST waits (the arbiter freeing the GPU, a chat context being saved), its phase still shows.
    async def poll() -> Progress:
        progress = as_dict(await route_json("GET", routes["progress"], caller = caller))
        phase = opt_text(progress.get("phase"))
        if phase in (None, "ready", "error"):
            return None
        return _media_fraction(progress) or 0.0, "Downloading" if phase == "downloading" else phase

    return poll


async def _media_ready(
    caller: Caller,
    kind: str,
    model: str,
    routes: dict[str, str],
    job: LoadJob,
    body: Optional[dict] = None,
) -> dict:
    idle = 0
    while True:
        progress = as_dict(await route_json("GET", routes["progress"], caller = caller))
        phase = progress.get("phase")
        if phase == "error":
            raise tool_error(opt_text(progress.get("error")) or f"The {kind} model failed to load")
        if phase == "ready":
            break
        if phase is None:
            status = await route_json("GET", routes["status"], caller = caller)
            if _resident_matches(status, model, body, settled = False):
                break
            idle += 1
            if idle >= _IDLE_POLLS:
                raise ToolError(
                    f"Unsloth Studio stopped reporting the {kind} load without loading {model}. "
                    "Another model load or a training run may be using the GPU; check studio_status and try again."
                )
        else:
            idle = 0
            job.note(
                _media_fraction(progress), "Downloading" if phase == "downloading" else str(phase)
            )
        await asyncio.sleep(POLL_INTERVAL_S)
    status = await route_json("GET", routes["status"], caller = caller)
    if not _resident_matches(status, model, body):
        raise ToolError(f"Unsloth Studio did not finish loading {model} as the {kind} model")
    return status


async def unload_media(caller: Caller, kind: str) -> UnloadResult:
    # Raised before anything is read and held until the unload has answered: a load of the slot this
    # process starts meanwhile posts after it, so the status read below never names that load.
    barrier = asyncio.get_running_loop().create_future()
    unloading = _MEDIA_UNLOADING.setdefault(kind, set())
    unloading.add(barrier)
    try:
        return await _unload_media(caller, kind)
    finally:
        unloading.discard(barrier)
        barrier.set_result(None)


async def _unload_media(caller: Caller, kind: str) -> UnloadResult:
    routes = MEDIA_ROUTES[kind]
    # A load_model still loading runs on in the job store; its load is ended below by its token.
    stopped = _stop_loads(caller, (kind,))
    # Each media load this caller posted ends by its load_token: the backend unloads it only while it is
    # the load in flight or resident, and refuses it on arrival if its POST has not reached it yet, so
    # this never waits on a POST and never ends a newer load (this caller's, the UI's or another account's).
    tokens = list(_MEDIA_TOKENS.get(_media_key(caller, kind), ()))
    tokens += [job.load_token for job in stopped if job.load_token and job.load_token not in tokens]
    # What the slot held when this call came, read before any unload goes out: the unload of the
    # resident below names that load, so one admitted later is never touched.
    status_read = asyncio.ensure_future(
        asyncio.wait_for(route_json("GET", routes["status"], caller = caller), READ_S)
    )
    try:
        await asyncio.wait({status_read})
    finally:
        # Even if this call is cancelled meanwhile, so a load it stopped still ends.
        ended = [_unload_media_token(caller, kind, token) for token in tokens]
    outcomes = await asyncio.shield(asyncio.gather(*ended)) if ended else []
    try:
        before = as_dict(status_read.result())
    except asyncio.TimeoutError:
        raise ToolError(
            f"Unsloth Studio did not answer the {kind} status read within {READ_S:.0f} s. Loads this "
            f"call stopped were ended, but a resident {kind} model may still be loaded; try unload_model again."
        ) from None
    model = (
        opt_text(before.get("display_repo_id"))
        or opt_text(before.get("repo_id"))
        or (stopped[-1].model if stopped else None)
    )
    unloaded = "unloaded" in outcomes
    if before.get("loaded") is not True:
        return UnloadResult(kind = kind, model = model if unloaded else None, unloaded = unloaded)
    resident = opt_text(before.get("load_id"))
    if resident is None and "load_id" in before:
        # Studio reports load ids but could not name this state (admissions or unloads kept landing
        # while it read): an unconditional unload could end a load admitted since, so none is sent.
        if not unloaded:
            raise ToolError(
                f"Unsloth Studio could not identify the loaded {kind} model while loads were changing; "
                "nothing was unloaded. Try unload_model again."
            )
    elif resident is None:
        # No load id at all: another account's model (Studio hides its id) or a Studio without load
        # tokens, which refuses or unloads it for this caller just as before.
        after = await _media_unload_post(caller, kind)
        unloaded = unloaded or (isinstance(after, dict) and after.get("loaded") is not True)
    elif resident not in tokens:
        answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": resident}))
        unloaded = unloaded or answer.get("outcome") == "unloaded"
    # load_id names a load in flight; the model it was replacing is resident_load_id. A replacement
    # that failed before its unload above left that model loaded, so end it too, by its own id
    # (a load admitted since is never it), also when it is this caller's own token: its unload above
    # answered superseded while the replacement was still in flight. Studio without the field:
    # nothing more to name.
    held = opt_text(before.get("resident_load_id"))
    if held is not None and held != resident:
        answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": held}))
        unloaded = unloaded or answer.get("outcome") == "unloaded"
    return UnloadResult(kind = kind, model = model, unloaded = unloaded)


@dataclass
class _MediaToken:
    # Order numbers (from _MEDIA_ORDER): when its load POST went out, and when it answered 2xx.
    sent: int
    answered: Optional[int] = None


# Media load tokens this process posted per (account, slot); each drops only on a definitive
# outcome.
_MEDIA_TOKENS: dict[tuple[str, str], dict[str, _MediaToken]] = {}
_MEDIA_ORDER = itertools.count(1)
# Unload POSTs this process sent per slot and not answered yet: a load posts only after them.
_MEDIA_UNLOADING: dict[str, set[asyncio.Future]] = {}
# Tries a token unload makes on route errors before leaving the token for the next unload_model.
_TOKEN_UNLOAD_ATTEMPTS = 5


def _media_key(caller: Caller, kind: str) -> tuple[str, str]:
    return (caller.account_id, kind)


def _remember_media_token(caller: Caller, kind: str, token: str) -> None:
    _MEDIA_TOKENS.setdefault(_media_key(caller, kind), {})[token] = _MediaToken(next(_MEDIA_ORDER))


async def _post_media_load(caller: Caller, kind: str, token: str, body: dict) -> Any:
    response = await forward(caller, "POST", MEDIA_ROUTES[kind]["load"], json_body = body)
    if 400 <= response.status_code < 500:
        # Refused before it was admitted (busy, invalid, a load already running): nothing to end.
        _forget_media_token(caller, kind, token)
    elif response.status_code < 400:
        state = _MEDIA_TOKENS.get(_media_key(caller, kind), {}).get(token)
        if state is not None:
            state.answered = next(_MEDIA_ORDER)
    return raise_for_route(response)


def _media_token_ready(caller: Caller, kind: str, token: Optional[str]) -> None:
    # ``token``'s load is shown ready: one whose POST answered before ``token``'s went out was admitted
    # before it, so it is over (Studio admits no load while another is in flight).
    tokens = _MEDIA_TOKENS.get(_media_key(caller, kind), {})
    newer = tokens.get(token) if token else None
    if newer is None:
        return
    for old, state in list(tokens.items()):
        if state.answered is not None and state.answered < newer.sent:
            del tokens[old]


def _forget_media_token(caller: Caller, kind: str, token: str) -> None:
    key = _media_key(caller, kind)
    tokens = _MEDIA_TOKENS.get(key, {})
    tokens.pop(token, None)
    if not tokens:
        _MEDIA_TOKENS.pop(key, None)


async def _media_unload_post(
    caller: Caller,
    kind: str,
    json_body: Optional[dict] = None,
) -> Any:
    # Tracked while out, so a load of the slot posts after it rather than racing it.
    post = asyncio.ensure_future(
        route_json("POST", MEDIA_ROUTES[kind]["unload"], caller = caller, json_body = json_body)
    )
    unloading = _MEDIA_UNLOADING.setdefault(kind, set())
    unloading.add(post)
    post.add_done_callback(unloading.discard)
    return await asyncio.shield(post)


def _unload_media_token(caller: Caller, kind: str, token: str) -> asyncio.Task:
    """End the load ``token`` names, in the background (held in _LATE_UNLOADS); the task answers its outcome."""

    async def unload() -> Optional[str]:
        for failures in range(1, _TOKEN_UNLOAD_ATTEMPTS + 1):
            try:
                answer = as_dict(
                    await _media_unload_post(caller, kind, {"expected_load_id": token})
                )
            except Exception:
                # A transient route error: try again; past the last try the token stays for the next unload.
                if failures < _TOKEN_UNLOAD_ATTEMPTS:
                    await asyncio.sleep(POLL_INTERVAL_S * failures)
                continue
            # unloaded, superseded or cancelled: in each case that load is over, or will never start.
            _forget_media_token(caller, kind, token)
            return opt_text(answer.get("outcome"))
        return None

    task = asyncio.ensure_future(unload())
    _LATE_UNLOADS.add(task)
    task.add_done_callback(_LATE_UNLOADS.discard)
    return task


STT_STATUS = "/api/inference/audio/stt/status"
# Transformers first: it is Studio's default and serves custom Whisper repos no list names.
STT_ENGINES = ("transformers", "mtmd", "audiocpp", "gguf")


async def stt_status(caller: Caller, model: Optional[str] = None) -> dict:
    params = {"model": model} if model else None
    return as_dict(await route_json("GET", STT_STATUS, caller = caller, params = params))


def stt_engine(status: dict, model: str) -> str:
    # Whisper ids are shared across engines: the one holding the model wins.
    for engine in STT_ENGINES:
        state = status.get(engine)
        if isinstance(state, dict) and state.get("loaded_model") == model:
            return engine
    for engine in STT_ENGINES:
        state = status.get(engine)
        if isinstance(state, dict) and model in {
            *strings(state.get("models")),
            *strings(state.get("downloaded_models")),
        }:
            return engine
    return "transformers"


async def catalog_stt_engine(caller: Caller, status: dict, model: str) -> str:
    engine = stt_engine(status, model)
    known = any(
        isinstance(state, dict)
        and model in {*strings(state.get("models")), *strings(state.get("downloaded_models"))}
        for state in status.values()
    )
    if known:
        return engine
    # list_models(source="catalog") offers audio.cpp ASR rows no status list names until downloaded.
    rows = as_dict(await try_json(caller, "/api/inference/audio/catalog")).get("models")
    for row in rows if isinstance(rows, list) else []:
        if (
            isinstance(row, dict)
            and row.get("task") == "asr"
            and not row.get("unsupported_reason")
            and str(row.get("id") or "").lower() == model.split(":", 1)[0].lower()
        ):
            return "audiocpp"
    return engine


async def download_stt(
    caller: Caller,
    ctx: Optional[Context],
    *,
    model: str,
    engine: str,
    hf_token: Optional[str],
    variant: Optional[str] = None,
) -> None:
    """Download an STT model and wait for it; the download route reads the Hub token from its header."""
    if hf_token:
        caller = dataclasses.replace(caller, hf_token = hf_token)
    if isinstance(ctx, LoadJob):
        ctx.download = {"model": model, "engine": engine}
    started = await route_json(
        "POST",
        "/api/inference/audio/stt/download",
        caller = caller,
        json_body = {
            "model": model,
            "engine": engine,
            **({"gguf_variant": variant} if variant else {}),
        },
        hub_header = True,
    )
    download_id = started.get("download_id") if isinstance(started, dict) else None
    if isinstance(ctx, LoadJob) and download_id is not None:
        ctx.download = {"model": model, "engine": engine, "download_id": download_id}
    while True:
        await asyncio.sleep(POLL_INTERVAL_S)
        state = (await stt_status(caller, model)).get(engine)
        download = as_dict(as_dict(state).get("download"))
        if download.get("downloading"):
            done, total = number(download.get("bytes_done")), number(download.get("bytes_total"))
            if ctx is not None and done is not None and total:
                await ctx.report_progress(min(done / total, 1.0), 1.0, "Downloading")
            continue
        if opt_text(download.get("error")):
            raise tool_error(f"Downloading {model} failed: {download['error']}")
        if download.get("cancelled"):
            raise ToolError(f"The download of {model} was cancelled")
        if download_id is None or download_id in strings(download.get("completed_download_ids")):
            return
        if model in strings(as_dict(state).get("downloaded_models")):
            return
        raise ToolError(f"Unsloth Studio stopped downloading {model} before it finished")


async def load_stt(
    caller: Caller,
    ctx: Optional[Context],
    *,
    model: str,
    variant: Optional[str],
    hf_token: Optional[str],
) -> LoadResult:
    status = await stt_status(caller, model)
    engine = await catalog_stt_engine(caller, status, model)
    state = as_dict(status.get(engine))
    # Load refuses a model that is not on disk, so download first and load only once that finished.
    # The route folds a variant into "model:variant"; another cached variant does not count.
    wanted = f"{model}:{variant}" if variant and ":" not in model else model
    if wanted not in strings(state.get("downloaded_models")):
        await download_stt(
            caller, ctx, model = model, engine = engine, hf_token = hf_token, variant = variant
        )
    body: dict[str, Any] = {"model": model, "engine": engine}
    if variant:
        body["gguf_variant"] = variant
    if isinstance(ctx, LoadJob):
        ctx.posted = True
    loaded = await route_json(
        "POST", "/api/inference/audio/stt/load", caller = caller, json_body = body
    )
    resident = opt_text(loaded.get("loaded_model")) if isinstance(loaded, dict) else None
    if resident is None:
        raise ToolError(
            f"Unsloth Studio did not keep {model} loaded; another load may have replaced it"
        )
    return LoadResult(kind = "stt", model = resident)


# Unloads waiting on an STT load that outlasted the unload call, and media token unloads, held so they are not collected.
_LATE_UNLOADS: set[asyncio.Task] = set()


async def _unload_once_loaded(caller: Caller, job: LoadJob) -> None:
    if job.task is not None:
        await asyncio.wait({job.task})
    if job.status == "completed" and isinstance(job.result, LoadResult):
        try:
            await unload_stt(caller, job.result.model)
        except Exception:
            pass


async def _stop_stt_loads(caller: Caller, model: Optional[str]) -> Optional[str]:
    """Stop this caller's STT loads still downloading, and their downloads, and let one already loading finish so it can be unloaded (later, if it outlasts this call); the last model stopped, if any."""
    stopped = None
    for job in LOADS.running(caller.account_id):
        if job.slot == "stt" and job.posted and (model is None or _same_id(job.model, model)):
            await job_store.settle(job, MAX_WAIT_S)
            if not job.finished:
                late = asyncio.get_running_loop().create_task(_unload_once_loaded(caller, job))
                _LATE_UNLOADS.add(late)
                late.add_done_callback(_LATE_UNLOADS.discard)
    jobs = [
        job
        for job in LOADS.running(caller.account_id)
        if job.slot == "stt" and not job.posted and (model is None or _same_id(job.model, model))
    ]
    for job in jobs:
        job_store.mark_cancelled(job)
        stopped = job.model
        # Partial files stay cached, so a later load resumes the download.
        if job.download is not None:
            await route_json(
                "POST",
                "/api/inference/audio/stt/download/cancel",
                caller = caller,
                json_body = job.download,
            )
    return stopped


async def unload_stt(caller: Caller, model: Optional[str]) -> UnloadResult:
    # A load_model still downloading or loading runs on in the job store, so unloading stops it too.
    stopped = await _stop_stt_loads(caller, model)
    status = await stt_status(caller)
    resident = {
        engine: status[engine]["loaded_model"]
        for engine in STT_ENGINES
        if isinstance(status.get(engine), dict) and opt_text(status[engine].get("loaded_model"))
    }
    targets = [(e, m) for e, m in resident.items() if model is None or m == model]
    if not targets:
        return UnloadResult(kind = "stt", model = stopped or model, unloaded = stopped is not None)
    engine, name = targets[0]
    await route_json(
        "POST",
        "/api/inference/audio/stt/unload",
        caller = caller,
        params = {"engine": engine, "model": name, "wait": "true"},
    )
    after = (await stt_status(caller)).get(engine)
    still = isinstance(after, dict) and after.get("loaded_model") == name
    return UnloadResult(kind = "stt", model = name, unloaded = not still)


async def unload_all(caller: Caller) -> UnloadAllResult:
    """Unload every chat slot, then the image, video and speech-to-text models. The embedding model is Unsloth Studio's own and stays. A step that fails does not stop the rest; the error then names what was unloaded and what was not."""
    failed: list[str] = []

    async def unload(name: str, target: str) -> None:
        try:
            await route_json(
                "POST", "/api/inference/unload", caller = caller, json_body = {"model_path": target}
            )
        except ToolError as exc:
            failed.append(f"{name}: {exc}")

    async def resident() -> tuple[list[str], list[str], set[str]]:
        # Bounded: a stalled status read fails the chat step, and the other slots still go.
        try:
            return await asyncio.wait_for(_llm_resident(caller), READ_S)
        except asyncio.TimeoutError:
            raise ToolError("Unsloth Studio did not say which chat models are loaded") from None

    unloaded: list[SlotModel] = []
    try:
        serving, checkpoints, _before = await resident()
        pairs = list(zip(serving, checkpoints))
        # Kept models first, so unloading the primary never promotes one of them in between.
        for name, target in reversed(pairs):
            await unload(name, target)
        now_serving, now_checkpoints, after = await resident()
        # A model the unloads promoted still serves (by name and by checkpoint): unload it too.
        promoted = [pair for pair in zip(now_serving, now_checkpoints) if pair not in pairs]
        for name, target in promoted:
            await unload(name, target)
        # Models held in memory behind the active one are listed only under loaded.
        known = {*serving, *checkpoints, *now_serving, *now_checkpoints}
        held = [name for name in after if name not in known]
        for name in held:
            await unload(name, name)
        if held or promoted:
            _serving, _checkpoints, after = await resident()
        pairs += promoted + [(name, name) for name in held]
        unloaded += [
            SlotModel(kind = "llm", model = name)
            for name, target in pairs
            if not {name, target} & after
        ]
    except ToolError as exc:
        # Unsloth Studio could not say which chat models are loaded; the other slots still go.
        failed.append(f"llm: {exc}")
    for kind in MEDIA_ROUTES:
        try:
            result = await unload_media(caller, kind)
        except ToolError as exc:
            failed.append(f"{kind}: {exc}")
            continue
        if result.unloaded and result.model:
            unloaded.append(SlotModel(kind = kind, model = result.model))
    for _engine in STT_ENGINES:
        try:
            result = await unload_stt(caller, None)
        except ToolError as exc:
            failed.append(f"stt: {exc}")
            break
        if not (result.unloaded and result.model):
            break
        unloaded.append(SlotModel(kind = "stt", model = result.model))
    if failed:
        done = ", ".join(f"{slot.kind} {slot.model}" for slot in unloaded) or "nothing"
        raise ToolError(f"Unloaded {done}; could not unload {'; '.join(failed)}")
    return UnloadAllResult(unloaded = unloaded)