Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Load and unload models per kind. A load (a big download can take an hour) runs as a job in the job store, so a client that stops waiting never stops it: load_model waits up to ``wait_s`` while reporting progress, then returns the job id for get_job(kind="load")."""
from __future__ import annotations
import asyncio
import dataclasses
import hashlib
import itertools
import os
import time
import uuid
from dataclasses import dataclass, field
from typing import Any, Awaitable, Callable, Optional
from fastmcp import Context
from fastmcp.exceptions import ToolError
from core.inference import model_identity
from studio_mcp import job_store
from studio_mcp import placement as place
from studio_mcp.caller import Caller
from studio_mcp.errors import raise_for_route, tool_error
from studio_mcp.forward import forward
from studio_mcp.outputs import EvictedSlot, LoadResult, SlotModel, UnloadAllResult, UnloadResult
from studio_mcp.tools import as_dict, number, opt_text, route_json, strings, try_json
POLL_INTERVAL_S = 2.0
# The longest a load_model or get_job call waits before returning (user decision); clients must allow more.
MAX_WAIT_S = 240
# The schema bound for wait_s. Tests shorten MAX_WAIT_S to fractions of a second, and get_job's schema can be
# built while they do (its annotations are lazy), so the bound must not read the patchable value.
WAIT_S_LIMIT = 240
# What a caller still waiting on a job is told to wait before its next get_job without wait_s.
POLL_AFTER_S = 10
# Forwarded reads have no HTTP timeout; a status or progress read taking longer counts as failed.
READ_S = 15.0
Progress = Optional[tuple[float, str]]
async def with_progress(
ctx: Optional[Context],
work: Awaitable[Any],
poll: Callable[[], Awaitable[Progress]],
job: Optional[LoadJob] = None,
) -> Any:
"""Await ``work`` while reporting what ``poll`` sees, to the agent and onto ``job``. A failed poll is skipped, never fatal."""
task = asyncio.ensure_future(work)
try:
while True:
done, _pending = await asyncio.wait({task}, timeout = POLL_INTERVAL_S)
if done:
return task.result()
try:
progress = await asyncio.wait_for(poll(), timeout = READ_S)
except Exception:
progress = None
if progress is not None and job is not None:
job.note(*progress)
if progress is not None and ctx is not None:
await ctx.report_progress(progress[0], 1.0, progress[1])
finally:
if not task.done():
task.cancel()
@dataclass
class LoadJob(job_store.Job):
kind: str = "load"
# The slot kind asked for (llm, tts, audio, image, video) and the model as asked.
slot: str = "llm"
model: str = ""
variant: Optional[str] = None
# Any other load settings; a repeat call with different ones starts its own load.
options: Any = None
fraction: Optional[float] = None
# Each distinct (fraction, phase) the load went through, for every caller following it.
marks: list[tuple[float, str]] = field(default_factory = list, repr = False)
# The STT download this load started, so unload_model can stop it, and whether the STT load
# request is out (it then runs to the end: cancelling its in-process route does not stop it).
download: Optional[dict] = None
posted: bool = False
# A media load POST, shielded: cancelling the job leaves it running, since the route registers its load from a thread.
post: Optional[asyncio.Future] = field(default = None, repr = False)
# The media load's load_token: an unload naming it ends this load only, even if its POST is still out.
load_token: Optional[str] = None
def note(self, fraction: Optional[float], phase: str) -> None:
# A step without a fraction keeps the last one, so progress never drops back.
self.phase = phase
self.fraction = self.fraction if fraction is None else fraction
if fraction is not None and (not self.marks or self.marks[-1] != (fraction, phase)):
self.marks.append((fraction, phase))
async def report_progress(
self,
progress: float,
total: Optional[float] = None,
message: Optional[str] = None,
) -> None:
"""Stands in for the agent's Context, so load code that reports progress records it on the job."""
self.note(progress, message or self.phase)
LOADS = job_store.JobStore(
capacity = 32, unknown = "No such load job", failure = "The load stopped unexpectedly."
)
def deadline_after(wait_s: Optional[float]) -> float:
"""When a wait of ``wait_s`` seconds (None: the longest) that starts now ends, on the monotonic clock."""
wait = MAX_WAIT_S if wait_s is None else max(0, min(wait_s, MAX_WAIT_S))
return time.monotonic() + wait
def _same(left: Optional[str], right: Optional[str]) -> bool:
# A relative path that exists on disk keeps its case, as the loaders read it.
return model_identity.same_model(left, right, probe = True)
def _same_id(left: str, right: str) -> bool:
return model_identity.same_repo(left, right, probe = True)
def _in_flight(caller: Caller, asked: LoadJob) -> Optional[LoadJob]:
for job in LOADS.running(caller.account_id):
if (
job.slot == asked.slot
and _same(job.model, asked.model)
and model_identity.same_variant(job.variant, asked.variant)
and job.options == asked.options
):
return job
return None
def _stop_loads(
caller: Caller,
slots: Any,
model: Optional[str] = None,
) -> list[LoadJob]:
"""Mark this caller's loads of ``slots`` still running in the job store cancelled. cancel has no load kind, so unload_model is how an agent stops one; the caller's unload route stops the load server-side."""
jobs = [
job
for job in LOADS.running(caller.account_id)
if job.slot in slots and (model is None or _same_id(job.model, model))
]
for job in jobs:
job_store.mark_cancelled(job)
return jobs
def _early(job: LoadJob) -> LoadResult:
return LoadResult(
kind = job.slot,
model = job.model,
loaded = False,
status = "loading",
fraction = job.fraction,
job_id = job.job_id,
poll_after_s = POLL_AFTER_S,
)
def outcome(job: LoadJob) -> LoadResult:
"""A finished load's result; a failed or cancelled one raises its error."""
if job.status == "completed" and isinstance(job.result, LoadResult):
return job.result
if job.status == "cancelled":
raise ToolError(f"The load of {job.model} was cancelled")
raise ToolError(job.error or f"Unsloth Studio did not load {job.model}")
async def wait_load(ctx: Optional[Context], job: LoadJob, deadline: float) -> LoadResult:
"""Wait for ``job`` until ``deadline`` (monotonic), reporting each step it makes; never cancels it."""
seen = 0 if job.task is None or not job.marks else len(job.marks) - 1
while True:
if ctx is not None:
for fraction, phase in job.marks[seen:]:
await ctx.report_progress(min(fraction, 1.0), 1.0, phase)
seen = len(job.marks)
left = deadline - time.monotonic()
if job.finished or job.task is None or left <= 0:
break
await asyncio.wait({job.task}, timeout = min(POLL_INTERVAL_S, left))
return outcome(job) if job.finished else _early(job)
async def start_load(
caller: Caller,
ctx: Optional[Context],
*,
slot: str,
model: str,
variant: Optional[str],
deadline: float,
run: Callable[[LoadJob, dict[str, list[str]]], Awaitable[LoadResult]],
options: Any = None,
hub_token: Optional[str] = None,
attach: bool = True,
) -> LoadResult:
"""Run a load as a job and wait for it until ``deadline``. A repeat call for the same load while it runs follows it; one with another Hub token (a retry with a fixed token, say) starts its own."""
token = hub_token or caller.hf_token
asked = LoadJob(
job_id = job_store.new_id(),
account_id = caller.account_id,
slot = slot,
model = model,
variant = variant,
options = (options, hashlib.sha256(token.encode()).hexdigest()[:16] if token else None),
)
async def drive(job: LoadJob) -> LoadResult:
# Every route read runs inside the job, so a slow one never holds the call past its deadline.
before, chat = await _resident(caller)
# A chat load Unsloth Studio is already running (started elsewhere) is followed, not started
# again. The status names only the repo, so a load asking for a variant always posts.
loading_now = strings(chat.get("loading"))
if (
slot in CHAT_ROUTE_KINDS
and attach
and not variant
and any(_same(n, model) for n in loading_now)
):
return await _watch_llm(caller, slot, model, before, job)
# K16, after the attach: M5 waits on a download of the same model here, then M7 refuses a partial one.
from studio_mcp import model_downloads
# Inside the job, so the wait outlives wait_s: the load starts once the download ends.
await model_downloads.wait_for_managed(
caller,
job,
model = model,
kind = slot,
variant = variant,
budget_s = model_downloads.MAX_LOAD_WAIT_S,
)
await place.refuse_partial_variant(caller, model, slot, variant)
return await run(job, before)
job = _in_flight(caller, asked) or LOADS.start(asked, drive)
return await wait_load(ctx, job, deadline)
# The load_model kinds that load through the chat route.
CHAT_ROUTE_KINDS = ("llm", "tts", "audio")
# Route slot kinds as MCP kinds.
ROUTE_KINDS = {"chat": "llm", "audio": "audio", "image": "image", "video": "video"}
def _media_name(status: Any) -> Optional[str]:
if not isinstance(status, dict) or status.get("loaded") is not True:
return None
return opt_text(status.get("display_repo_id")) or opt_text(status.get("repo_id"))
async def _read_json(caller: Caller, path: str) -> Any:
"""``try_json`` within READ_S: a read that stalls is None, like one that failed."""
try:
return await asyncio.wait_for(try_json(caller, path), timeout = READ_S)
except asyncio.TimeoutError:
return None
async def _resident(caller: Caller) -> tuple[dict[str, list[str]], dict]:
chat, image, video = await asyncio.gather(
_read_json(caller, "/api/inference/status"),
_read_json(caller, MEDIA_ROUTES["image"]["status"]),
_read_json(caller, MEDIA_ROUTES["video"]["status"]),
)
slots: dict[str, list[str]] = {}
# A slot that could not be read is left out: this only feeds a report.
if isinstance(chat, dict):
slots["llm"] = list(
dict.fromkeys([*strings(chat.get("serving")), *strings(chat.get("loaded"))])
)
for kind, status in (("image", image), ("video", video)):
if isinstance(status, dict):
name = _media_name(status)
slots[kind] = [name] if name else []
return slots, as_dict(chat)
async def resident_slots(caller: Caller) -> dict[str, list[str]]:
"""What each slot holds now, by MCP kind: llm (chat models), image and video."""
return (await _resident(caller))[0]
def _slot_rows(rows: Any) -> list[EvictedSlot]:
found = []
for row in rows if isinstance(rows, list) else []:
if isinstance(row, dict) and opt_text(row.get("model")):
kind = opt_text(row.get("kind")) or "chat"
found.append(EvictedSlot(kind = ROUTE_KINDS.get(kind, kind), model = row["model"]))
return found
async def finish_load(
caller: Caller, kind: str, model: str, before: dict[str, list[str]], answer: Any, **extras: Any
) -> LoadResult:
"""The LoadResult for a finished load, with what it unloaded: the load answer's ``evicted_slots`` (else its ``evicted`` ids placed by the ``before`` snapshot), plus a before/after diff for what the answer leaves out."""
answer = as_dict(answer)
named = strings(answer.get("evicted"))
slots = _slot_rows(answer.get("evicted_slots"))
if not slots:
slots = [
EvictedSlot(
kind = next(
(k for k, models in before.items() if any(_same_id(name, m) for m in models)),
"llm",
),
model = name,
)
for name in named
]
# The route lists what it unloaded to make room, never the model this load replaced in its own
# slot (and nothing at all before it had a list), so whatever else left is found by a diff.
if any(before.values()):
after = await resident_slots(caller)
slots += [
EvictedSlot(kind = slot, model = name)
for slot, models in before.items()
if slot in after
for name in models
if not any(_same(name, m) for m in after[slot])
and not _same_id(name, model)
and not any(_same_id(name, s.model) for s in slots)
]
evicted = [slot.model for slot in slots] if slots else named
return LoadResult(kind = kind, model = model, evicted = evicted, evicted_slots = slots, **extras)
def _llm_poll(caller: Caller, model: str) -> Callable[[], Awaitable[Progress]]:
async def poll() -> Progress:
loading = await route_json("GET", "/api/inference/load-progress", caller = caller)
fraction = number(loading.get("fraction")) if isinstance(loading, dict) else None
if fraction and opt_text(loading.get("phase")):
return min(fraction, 1.0), f"Loading into memory ({loading['phase']})"
if model_identity.is_local(model):
return None
download = await route_json(
"GET",
"/api/models/download-progress",
caller = caller,
params = {"repo_id": model},
hub_header = True,
)
fraction = number(download.get("progress")) if isinstance(download, dict) else None
return (min(fraction, 1.0), "Downloading") if fraction else None
return poll
async def load_llm(
caller: Caller,
ctx: Optional[Context],
*,
model: str,
variant: Optional[str],
max_seq_length: Optional[int],
load_in_4bit: bool,
hf_token: Optional[str],
kind: str = "llm",
deadline: Optional[float] = None,
device: Optional[str] = None,
alongside: Optional[bool] = None,
kv_cache_type: Optional[str] = None,
gpu_layers: Optional[int] = None,
is_lora: bool = False,
) -> LoadResult:
body: dict[str, Any] = {
"model_path": model,
"load_in_4bit": load_in_4bit,
"max_seq_length": max_seq_length or 0,
}
if variant:
body["gguf_variant"] = variant
if is_lora:
body["is_lora"] = True
# The load routes read the Hub token from the body, not the header.
token = hf_token or caller.hf_token
if token:
body["hf_token"] = token
if hf_token:
# The variant lookups before the load read the Hub token from the header.
caller = dataclasses.replace(caller, hf_token = hf_token)
async def run(job: LoadJob, before: dict[str, list[str]]) -> LoadResult:
# The GGUF listing and status reads run inside the job, so a slow one never holds the call
# past ``wait_s``; device='cpu' for a chat model that is not a GGUF is refused here.
gguf = await place.require_cpu_gguf(caller, kind, model, variant, device, gpu_layers)
body.update(
place.llm_fields(
kind,
gguf = gguf,
device = device,
alongside = await place.auto_alongside(caller, device, alongside, gpu_layers),
kv_cache_type = kv_cache_type,
gpu_layers = gpu_layers,
)
)
payload = await with_progress(
None,
route_json("POST", "/api/inference/load", caller = caller, json_body = body),
_llm_poll(caller, model),
job,
)
if not isinstance(payload, dict):
raise ToolError("Unsloth Studio did not confirm the load")
loaded = opt_text(payload.get("model")) or model
chat = before.get("llm") or []
# Serving only: a model merely held in memory behind the new primary no longer serves.
# Bounded: the model is loaded, and a stalled read must not keep the job open.
status = await _read_json(caller, "/api/inference/status") if chat else None
# A failed read leaves it unknown whether the previous model still serves.
after = strings(status.get("serving")) if isinstance(status, dict) else None
return await finish_load(
caller,
kind,
loaded,
before,
payload,
display_name = opt_text(payload.get("display_name")),
placement = place.llm_placement(payload, device),
alongside_applied = place.alongside_applied(chat, after, loaded),
)
return await start_load(
caller,
ctx,
slot = kind,
model = model,
variant = variant,
deadline = deadline or deadline_after(None),
run = run,
# A load asked with other placement fields is a different load, not one to follow.
options = (
max_seq_length or 0,
load_in_4bit,
device,
alongside,
kv_cache_type,
gpu_layers,
),
hub_token = token,
# A load asked with its own settings is never the one already running; following it would
# skip them (and the CPU check).
attach = not max_seq_length
and load_in_4bit
and all(v is None for v in (device, alongside, kv_cache_type, gpu_layers)),
)
async def _watch_llm(
caller: Caller, kind: str, model: str, before: dict[str, list[str]], job: LoadJob
) -> LoadResult:
"""Follow a chat load Unsloth Studio is already running, until it serves ``model`` or stops loading it."""
poll = _llm_poll(caller, model)
misses = 0
while True:
await asyncio.sleep(POLL_INTERVAL_S)
status = await _read_json(caller, "/api/inference/status")
if not isinstance(status, dict):
# A failed read is skipped; only a run of them ends the follow.
misses += 1
if misses >= _IDLE_POLLS:
raise ToolError(f"Unsloth Studio stopped answering while loading {model}")
continue
misses = 0
if not any(_same(name, model) for name in strings(status.get("loading"))):
if any(_same_id(name, model) for name in strings(status.get("serving"))):
return await finish_load(caller, kind, model, before, {})
raise ToolError(f"Unsloth Studio stopped loading {model} before it was ready")
try:
progress = await asyncio.wait_for(poll(), timeout = READ_S)
except Exception:
progress = None
if progress is not None:
job.note(*progress)
async def _llm_resident(caller: Caller) -> tuple[list[str], list[str], set[str]]:
status = as_dict(await route_json("GET", "/api/inference/status", caller = caller))
serving, checkpoints = (
strings(status.get("serving")),
strings(status.get("serving_checkpoints")),
)
return serving, checkpoints, {*serving, *checkpoints, *strings(status.get("loaded"))}
async def unload_llm(
caller: Caller,
model: Optional[str],
kind: str = "llm",
) -> UnloadResult:
serving, checkpoints, before = await _llm_resident(caller)
if model is None:
if not checkpoints:
return UnloadResult(kind = kind, unloaded = False)
target, model = checkpoints[0], (serving[0] if serving else checkpoints[0])
else:
# The route unloads by checkpoint id, listed in the same order as serving.
target = dict(zip(serving, checkpoints)).get(model, model)
await route_json(
"POST", "/api/inference/unload", caller = caller, json_body = {"model_path": target}
)
# An id that matches nothing still answers success, so compare what is resident before and after.
_serving, _checkpoints, after = await _llm_resident(caller)
was_loaded = bool({model, target} & before)
return UnloadResult(
kind = kind, model = model, unloaded = was_loaded and not ({model, target} & after)
)
MEDIA_ROUTES = {
kind: {
"plan": f"/api/inference/{prefix}/download-plan",
"load": f"/api/inference/{prefix}/load",
"progress": f"/api/inference/{prefix}/load-progress",
"status": f"/api/inference/{prefix}/status",
"unload": f"/api/inference/{prefix}/unload",
}
for kind, prefix in (("image", "images"), ("video", "video"))
}
# Polls with no load in flight and the model still not resident before the load counts as lost.
_IDLE_POLLS = 5
def _media_fraction(progress: dict) -> Optional[float]:
# Images report bytes_downloaded/bytes_total, video downloaded_bytes/expected_bytes.
done = number(progress.get("bytes_downloaded", progress.get("downloaded_bytes")))
total = number(progress.get("bytes_total", progress.get("expected_bytes")))
if done is not None and total:
return min(done / total, 1.0)
return number(progress.get("fraction"))
def _resident_matches(
status: Any,
model: str,
body: Optional[dict] = None,
*,
settled: bool = True,
) -> bool:
if not isinstance(status, dict) or status.get("loaded") is not True:
return False
names = {opt_text(status.get("repo_id")), opt_text(status.get("display_repo_id"))}
# Any case, paths too: status may spell the resident path differently from the request.
if model.lower() not in {name.lower() for name in names if name}:
return False
# Another file of the same repo is another model: a missing field reads as not loaded yet.
wanted = opt_text((body or {}).get("gguf_filename"))
if not wanted:
return True
kind = opt_text((body or {}).get("model_kind"))
if kind and opt_text(status.get("model_kind")) not in (None, kind):
return False
resident = opt_text(status.get("gguf_filename"))
if resident:
return os.path.normcase(resident) == os.path.normcase(wanted)
# Video status names only a GGUF's quant; a plain checkpoint has none to compare.
quant = model_identity.resident_quant_matches(
opt_text(status.get("gguf_variant")), wanted, settled = settled
)
if quant is None:
return True
if not quant:
return False
# MiniMax-H3 ships its fl2va and ref2va partitions at the same quant; status names the one loaded.
h3_task = opt_text(status.get("h3_task"))
if not h3_task:
return True
try:
from core.inference.video_minimax_h3 import h3_transformer_task
except Exception:
return False
return h3_task.lower() == h3_transformer_task(wanted).lower()
async def _gguf_filename(caller: Caller, model: str, variant: Optional[str]) -> Optional[dict]:
"""The variant to load from a GGUF repo (the named one, else the repo's default), with its filename, size_bytes, downloaded and partial."""
listing = await route_json(
"GET", "/api/hub/gguf-variants", caller = caller, params = {"repo_id": model}, hub_header = True
)
entry = place.pick_variant(listing, variant)
if entry is not None and opt_text(entry.get("filename")):
return entry
if variant:
raise ToolError(f"{model} has no GGUF variant {variant}")
return None
async def load_media(
caller: Caller,
ctx: Optional[Context],
*,
kind: str,
model: str,
variant: Optional[str],
hf_token: Optional[str],
deadline: Optional[float] = None,
knobs: Optional[dict[str, str]] = None,
) -> LoadResult:
routes = MEDIA_ROUTES[kind]
knobs = knobs or {}
if hf_token:
# The GGUF lookup reads the Hub token from the header, so a gated repo needs it there too.
caller = dataclasses.replace(caller, hf_token = hf_token)
body: dict[str, Any] = {"model_path": model}
token = hf_token or caller.hf_token
if token:
body["hf_token"] = token
async def prepare() -> None:
# Hub and plan reads run inside the job, so a slow one never holds the call past ``wait_s``.
if variant or "gguf" in model.lower():
entry = await _gguf_filename(caller, model, variant)
if entry:
body["gguf_filename"] = entry["filename"]
body["model_kind"] = "gguf"
# The plan validates the pick the way the load does, before any download.
plan = await place.post_plan(caller, routes["plan"], body, knobs)
if isinstance(plan, dict):
if opt_text(plan.get("incompatible_reason")):
raise tool_error(plan["incompatible_reason"])
if "gguf_filename" not in body:
for entry in plan.get("entries") or []:
if (
isinstance(entry, dict)
and entry.get("checkpoint")
and opt_text(entry.get("gguf_filename"))
):
body["gguf_filename"] = entry["gguf_filename"]
break
async def run(job: LoadJob, before: dict[str, list[str]]) -> LoadResult:
await prepare()
# An unload this process sent for the slot could cancel this load if it ran after it: post after it.
while unloading := {u for u in _MEDIA_UNLOADING.get(kind, ()) if not u.done()}:
await asyncio.wait(unloading)
# Named before it goes out, so an unload can end this load even if the POST's answer never comes.
job.load_token = uuid.uuid4().hex
_remember_media_token(caller, kind, job.load_token)
# The answer names what the arbiter unloaded to make room; the load itself goes on in the background.
job.post = asyncio.ensure_future(
_post_media_load(
caller, kind, job.load_token, {**body, **knobs, "load_token": job.load_token}
)
)
job.post.add_done_callback(lambda post: post.cancelled() or post.exception())
answer = await with_progress(
None, asyncio.shield(job.post), _media_poll(caller, routes), job
)
status = await _media_ready(caller, kind, model, routes, job, body)
_media_token_ready(caller, kind, job.load_token)
name = _media_name(status) or model
return await finish_load(
caller, kind, name, before, answer, placement = place.media_placement(status)
)
return await start_load(
caller,
ctx,
slot = kind,
model = model,
variant = variant,
deadline = deadline or deadline_after(None),
run = run,
options = tuple(sorted(knobs.items())),
)
def _media_poll(caller: Caller, routes: dict[str, str]) -> Callable[[], Awaitable[Progress]]:
# While the load POST waits (the arbiter freeing the GPU, a chat context being saved), its phase still shows.
async def poll() -> Progress:
progress = as_dict(await route_json("GET", routes["progress"], caller = caller))
phase = opt_text(progress.get("phase"))
if phase in (None, "ready", "error"):
return None
return _media_fraction(progress) or 0.0, "Downloading" if phase == "downloading" else phase
return poll
async def _media_ready(
caller: Caller,
kind: str,
model: str,
routes: dict[str, str],
job: LoadJob,
body: Optional[dict] = None,
) -> dict:
idle = 0
while True:
progress = as_dict(await route_json("GET", routes["progress"], caller = caller))
phase = progress.get("phase")
if phase == "error":
raise tool_error(opt_text(progress.get("error")) or f"The {kind} model failed to load")
if phase == "ready":
break
if phase is None:
status = await route_json("GET", routes["status"], caller = caller)
if _resident_matches(status, model, body, settled = False):
break
idle += 1
if idle >= _IDLE_POLLS:
raise ToolError(
f"Unsloth Studio stopped reporting the {kind} load without loading {model}. "
"Another model load or a training run may be using the GPU; check studio_status and try again."
)
else:
idle = 0
job.note(
_media_fraction(progress), "Downloading" if phase == "downloading" else str(phase)
)
await asyncio.sleep(POLL_INTERVAL_S)
status = await route_json("GET", routes["status"], caller = caller)
if not _resident_matches(status, model, body):
raise ToolError(f"Unsloth Studio did not finish loading {model} as the {kind} model")
return status
async def unload_media(caller: Caller, kind: str) -> UnloadResult:
# Raised before anything is read and held until the unload has answered: a load of the slot this
# process starts meanwhile posts after it, so the status read below never names that load.
barrier = asyncio.get_running_loop().create_future()
unloading = _MEDIA_UNLOADING.setdefault(kind, set())
unloading.add(barrier)
try:
return await _unload_media(caller, kind)
finally:
unloading.discard(barrier)
barrier.set_result(None)
async def _unload_media(caller: Caller, kind: str) -> UnloadResult:
routes = MEDIA_ROUTES[kind]
# A load_model still loading runs on in the job store; its load is ended below by its token.
stopped = _stop_loads(caller, (kind,))
# Each media load this caller posted ends by its load_token: the backend unloads it only while it is
# the load in flight or resident, and refuses it on arrival if its POST has not reached it yet, so
# this never waits on a POST and never ends a newer load (this caller's, the UI's or another account's).
tokens = list(_MEDIA_TOKENS.get(_media_key(caller, kind), ()))
tokens += [job.load_token for job in stopped if job.load_token and job.load_token not in tokens]
# What the slot held when this call came, read before any unload goes out: the unload of the
# resident below names that load, so one admitted later is never touched.
status_read = asyncio.ensure_future(
asyncio.wait_for(route_json("GET", routes["status"], caller = caller), READ_S)
)
try:
await asyncio.wait({status_read})
finally:
# Even if this call is cancelled meanwhile, so a load it stopped still ends.
ended = [_unload_media_token(caller, kind, token) for token in tokens]
outcomes = await asyncio.shield(asyncio.gather(*ended)) if ended else []
try:
before = as_dict(status_read.result())
except asyncio.TimeoutError:
raise ToolError(
f"Unsloth Studio did not answer the {kind} status read within {READ_S:.0f} s. Loads this "
f"call stopped were ended, but a resident {kind} model may still be loaded; try unload_model again."
) from None
model = (
opt_text(before.get("display_repo_id"))
or opt_text(before.get("repo_id"))
or (stopped[-1].model if stopped else None)
)
unloaded = "unloaded" in outcomes
if before.get("loaded") is not True:
return UnloadResult(kind = kind, model = model if unloaded else None, unloaded = unloaded)
resident = opt_text(before.get("load_id"))
if resident is None and "load_id" in before:
# Studio reports load ids but could not name this state (admissions or unloads kept landing
# while it read): an unconditional unload could end a load admitted since, so none is sent.
if not unloaded:
raise ToolError(
f"Unsloth Studio could not identify the loaded {kind} model while loads were changing; "
"nothing was unloaded. Try unload_model again."
)
elif resident is None:
# No load id at all: another account's model (Studio hides its id) or a Studio without load
# tokens, which refuses or unloads it for this caller just as before.
after = await _media_unload_post(caller, kind)
unloaded = unloaded or (isinstance(after, dict) and after.get("loaded") is not True)
elif resident not in tokens:
answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": resident}))
unloaded = unloaded or answer.get("outcome") == "unloaded"
# load_id names a load in flight; the model it was replacing is resident_load_id. A replacement
# that failed before its unload above left that model loaded, so end it too, by its own id
# (a load admitted since is never it), also when it is this caller's own token: its unload above
# answered superseded while the replacement was still in flight. Studio without the field:
# nothing more to name.
held = opt_text(before.get("resident_load_id"))
if held is not None and held != resident:
answer = as_dict(await _media_unload_post(caller, kind, {"expected_load_id": held}))
unloaded = unloaded or answer.get("outcome") == "unloaded"
return UnloadResult(kind = kind, model = model, unloaded = unloaded)
@dataclass
class _MediaToken:
# Order numbers (from _MEDIA_ORDER): when its load POST went out, and when it answered 2xx.
sent: int
answered: Optional[int] = None
# Media load tokens this process posted per (account, slot); each drops only on a definitive
# outcome.
_MEDIA_TOKENS: dict[tuple[str, str], dict[str, _MediaToken]] = {}
_MEDIA_ORDER = itertools.count(1)
# Unload POSTs this process sent per slot and not answered yet: a load posts only after them.
_MEDIA_UNLOADING: dict[str, set[asyncio.Future]] = {}
# Tries a token unload makes on route errors before leaving the token for the next unload_model.
_TOKEN_UNLOAD_ATTEMPTS = 5
def _media_key(caller: Caller, kind: str) -> tuple[str, str]:
return (caller.account_id, kind)
def _remember_media_token(caller: Caller, kind: str, token: str) -> None:
_MEDIA_TOKENS.setdefault(_media_key(caller, kind), {})[token] = _MediaToken(next(_MEDIA_ORDER))
async def _post_media_load(caller: Caller, kind: str, token: str, body: dict) -> Any:
response = await forward(caller, "POST", MEDIA_ROUTES[kind]["load"], json_body = body)
if 400 <= response.status_code < 500:
# Refused before it was admitted (busy, invalid, a load already running): nothing to end.
_forget_media_token(caller, kind, token)
elif response.status_code < 400:
state = _MEDIA_TOKENS.get(_media_key(caller, kind), {}).get(token)
if state is not None:
state.answered = next(_MEDIA_ORDER)
return raise_for_route(response)
def _media_token_ready(caller: Caller, kind: str, token: Optional[str]) -> None:
# ``token``'s load is shown ready: one whose POST answered before ``token``'s went out was admitted
# before it, so it is over (Studio admits no load while another is in flight).
tokens = _MEDIA_TOKENS.get(_media_key(caller, kind), {})
newer = tokens.get(token) if token else None
if newer is None:
return
for old, state in list(tokens.items()):
if state.answered is not None and state.answered < newer.sent:
del tokens[old]
def _forget_media_token(caller: Caller, kind: str, token: str) -> None:
key = _media_key(caller, kind)
tokens = _MEDIA_TOKENS.get(key, {})
tokens.pop(token, None)
if not tokens:
_MEDIA_TOKENS.pop(key, None)
async def _media_unload_post(
caller: Caller,
kind: str,
json_body: Optional[dict] = None,
) -> Any:
# Tracked while out, so a load of the slot posts after it rather than racing it.
post = asyncio.ensure_future(
route_json("POST", MEDIA_ROUTES[kind]["unload"], caller = caller, json_body = json_body)
)
unloading = _MEDIA_UNLOADING.setdefault(kind, set())
unloading.add(post)
post.add_done_callback(unloading.discard)
return await asyncio.shield(post)
def _unload_media_token(caller: Caller, kind: str, token: str) -> asyncio.Task:
"""End the load ``token`` names, in the background (held in _LATE_UNLOADS); the task answers its outcome."""
async def unload() -> Optional[str]:
for failures in range(1, _TOKEN_UNLOAD_ATTEMPTS + 1):
try:
answer = as_dict(
await _media_unload_post(caller, kind, {"expected_load_id": token})
)
except Exception:
# A transient route error: try again; past the last try the token stays for the next unload.
if failures < _TOKEN_UNLOAD_ATTEMPTS:
await asyncio.sleep(POLL_INTERVAL_S * failures)
continue
# unloaded, superseded or cancelled: in each case that load is over, or will never start.
_forget_media_token(caller, kind, token)
return opt_text(answer.get("outcome"))
return None
task = asyncio.ensure_future(unload())
_LATE_UNLOADS.add(task)
task.add_done_callback(_LATE_UNLOADS.discard)
return task
STT_STATUS = "/api/inference/audio/stt/status"
# Transformers first: it is Studio's default and serves custom Whisper repos no list names.
STT_ENGINES = ("transformers", "mtmd", "audiocpp", "gguf")
async def stt_status(caller: Caller, model: Optional[str] = None) -> dict:
params = {"model": model} if model else None
return as_dict(await route_json("GET", STT_STATUS, caller = caller, params = params))
def stt_engine(status: dict, model: str) -> str:
# Whisper ids are shared across engines: the one holding the model wins.
for engine in STT_ENGINES:
state = status.get(engine)
if isinstance(state, dict) and state.get("loaded_model") == model:
return engine
for engine in STT_ENGINES:
state = status.get(engine)
if isinstance(state, dict) and model in {
*strings(state.get("models")),
*strings(state.get("downloaded_models")),
}:
return engine
return "transformers"
async def catalog_stt_engine(caller: Caller, status: dict, model: str) -> str:
engine = stt_engine(status, model)
known = any(
isinstance(state, dict)
and model in {*strings(state.get("models")), *strings(state.get("downloaded_models"))}
for state in status.values()
)
if known:
return engine
# list_models(source="catalog") offers audio.cpp ASR rows no status list names until downloaded.
rows = as_dict(await try_json(caller, "/api/inference/audio/catalog")).get("models")
for row in rows if isinstance(rows, list) else []:
if (
isinstance(row, dict)
and row.get("task") == "asr"
and not row.get("unsupported_reason")
and str(row.get("id") or "").lower() == model.split(":", 1)[0].lower()
):
return "audiocpp"
return engine
async def download_stt(
caller: Caller,
ctx: Optional[Context],
*,
model: str,
engine: str,
hf_token: Optional[str],
variant: Optional[str] = None,
) -> None:
"""Download an STT model and wait for it; the download route reads the Hub token from its header."""
if hf_token:
caller = dataclasses.replace(caller, hf_token = hf_token)
if isinstance(ctx, LoadJob):
ctx.download = {"model": model, "engine": engine}
started = await route_json(
"POST",
"/api/inference/audio/stt/download",
caller = caller,
json_body = {
"model": model,
"engine": engine,
**({"gguf_variant": variant} if variant else {}),
},
hub_header = True,
)
download_id = started.get("download_id") if isinstance(started, dict) else None
if isinstance(ctx, LoadJob) and download_id is not None:
ctx.download = {"model": model, "engine": engine, "download_id": download_id}
while True:
await asyncio.sleep(POLL_INTERVAL_S)
state = (await stt_status(caller, model)).get(engine)
download = as_dict(as_dict(state).get("download"))
if download.get("downloading"):
done, total = number(download.get("bytes_done")), number(download.get("bytes_total"))
if ctx is not None and done is not None and total:
await ctx.report_progress(min(done / total, 1.0), 1.0, "Downloading")
continue
if opt_text(download.get("error")):
raise tool_error(f"Downloading {model} failed: {download['error']}")
if download.get("cancelled"):
raise ToolError(f"The download of {model} was cancelled")
if download_id is None or download_id in strings(download.get("completed_download_ids")):
return
if model in strings(as_dict(state).get("downloaded_models")):
return
raise ToolError(f"Unsloth Studio stopped downloading {model} before it finished")
async def load_stt(
caller: Caller,
ctx: Optional[Context],
*,
model: str,
variant: Optional[str],
hf_token: Optional[str],
) -> LoadResult:
status = await stt_status(caller, model)
engine = await catalog_stt_engine(caller, status, model)
state = as_dict(status.get(engine))
# Load refuses a model that is not on disk, so download first and load only once that finished.
# The route folds a variant into "model:variant"; another cached variant does not count.
wanted = f"{model}:{variant}" if variant and ":" not in model else model
if wanted not in strings(state.get("downloaded_models")):
await download_stt(
caller, ctx, model = model, engine = engine, hf_token = hf_token, variant = variant
)
body: dict[str, Any] = {"model": model, "engine": engine}
if variant:
body["gguf_variant"] = variant
if isinstance(ctx, LoadJob):
ctx.posted = True
loaded = await route_json(
"POST", "/api/inference/audio/stt/load", caller = caller, json_body = body
)
resident = opt_text(loaded.get("loaded_model")) if isinstance(loaded, dict) else None
if resident is None:
raise ToolError(
f"Unsloth Studio did not keep {model} loaded; another load may have replaced it"
)
return LoadResult(kind = "stt", model = resident)
# Unloads waiting on an STT load that outlasted the unload call, and media token unloads, held so they are not collected.
_LATE_UNLOADS: set[asyncio.Task] = set()
async def _unload_once_loaded(caller: Caller, job: LoadJob) -> None:
if job.task is not None:
await asyncio.wait({job.task})
if job.status == "completed" and isinstance(job.result, LoadResult):
try:
await unload_stt(caller, job.result.model)
except Exception:
pass
async def _stop_stt_loads(caller: Caller, model: Optional[str]) -> Optional[str]:
"""Stop this caller's STT loads still downloading, and their downloads, and let one already loading finish so it can be unloaded (later, if it outlasts this call); the last model stopped, if any."""
stopped = None
for job in LOADS.running(caller.account_id):
if job.slot == "stt" and job.posted and (model is None or _same_id(job.model, model)):
await job_store.settle(job, MAX_WAIT_S)
if not job.finished:
late = asyncio.get_running_loop().create_task(_unload_once_loaded(caller, job))
_LATE_UNLOADS.add(late)
late.add_done_callback(_LATE_UNLOADS.discard)
jobs = [
job
for job in LOADS.running(caller.account_id)
if job.slot == "stt" and not job.posted and (model is None or _same_id(job.model, model))
]
for job in jobs:
job_store.mark_cancelled(job)
stopped = job.model
# Partial files stay cached, so a later load resumes the download.
if job.download is not None:
await route_json(
"POST",
"/api/inference/audio/stt/download/cancel",
caller = caller,
json_body = job.download,
)
return stopped
async def unload_stt(caller: Caller, model: Optional[str]) -> UnloadResult:
# A load_model still downloading or loading runs on in the job store, so unloading stops it too.
stopped = await _stop_stt_loads(caller, model)
status = await stt_status(caller)
resident = {
engine: status[engine]["loaded_model"]
for engine in STT_ENGINES
if isinstance(status.get(engine), dict) and opt_text(status[engine].get("loaded_model"))
}
targets = [(e, m) for e, m in resident.items() if model is None or m == model]
if not targets:
return UnloadResult(kind = "stt", model = stopped or model, unloaded = stopped is not None)
engine, name = targets[0]
await route_json(
"POST",
"/api/inference/audio/stt/unload",
caller = caller,
params = {"engine": engine, "model": name, "wait": "true"},
)
after = (await stt_status(caller)).get(engine)
still = isinstance(after, dict) and after.get("loaded_model") == name
return UnloadResult(kind = "stt", model = name, unloaded = not still)
async def unload_all(caller: Caller) -> UnloadAllResult:
"""Unload every chat slot, then the image, video and speech-to-text models. The embedding model is Unsloth Studio's own and stays. A step that fails does not stop the rest; the error then names what was unloaded and what was not."""
failed: list[str] = []
async def unload(name: str, target: str) -> None:
try:
await route_json(
"POST", "/api/inference/unload", caller = caller, json_body = {"model_path": target}
)
except ToolError as exc:
failed.append(f"{name}: {exc}")
async def resident() -> tuple[list[str], list[str], set[str]]:
# Bounded: a stalled status read fails the chat step, and the other slots still go.
try:
return await asyncio.wait_for(_llm_resident(caller), READ_S)
except asyncio.TimeoutError:
raise ToolError("Unsloth Studio did not say which chat models are loaded") from None
unloaded: list[SlotModel] = []
try:
serving, checkpoints, _before = await resident()
pairs = list(zip(serving, checkpoints))
# Kept models first, so unloading the primary never promotes one of them in between.
for name, target in reversed(pairs):
await unload(name, target)
now_serving, now_checkpoints, after = await resident()
# A model the unloads promoted still serves (by name and by checkpoint): unload it too.
promoted = [pair for pair in zip(now_serving, now_checkpoints) if pair not in pairs]
for name, target in promoted:
await unload(name, target)
# Models held in memory behind the active one are listed only under loaded.
known = {*serving, *checkpoints, *now_serving, *now_checkpoints}
held = [name for name in after if name not in known]
for name in held:
await unload(name, name)
if held or promoted:
_serving, _checkpoints, after = await resident()
pairs += promoted + [(name, name) for name in held]
unloaded += [
SlotModel(kind = "llm", model = name)
for name, target in pairs
if not {name, target} & after
]
except ToolError as exc:
# Unsloth Studio could not say which chat models are loaded; the other slots still go.
failed.append(f"llm: {exc}")
for kind in MEDIA_ROUTES:
try:
result = await unload_media(caller, kind)
except ToolError as exc:
failed.append(f"{kind}: {exc}")
continue
if result.unloaded and result.model:
unloaded.append(SlotModel(kind = kind, model = result.model))
for _engine in STT_ENGINES:
try:
result = await unload_stt(caller, None)
except ToolError as exc:
failed.append(f"stt: {exc}")
break
if not (result.unloaded and result.model):
break
unloaded.append(SlotModel(kind = "stt", model = result.model))
if failed:
done = ", ".join(f"{slot.kind} {slot.model}" for slot in unloaded) or "nothing"
raise ToolError(f"Unloaded {done}; could not unload {'; '.join(failed)}")
return UnloadAllResult(unloaded = unloaded)