Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: list each on-disk quant of a multi-quant GGUF in GET /v1/models (#13422)
Daniel Hancfcf587af979
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.pyindex 2bcd1bd9c1f..899f934be88 100644--- a/studio/backend/core/inference/local_model_resolver.py+++ b/studio/backend/core/inference/local_model_resolver.py@@ -1340,17 +1340,17 @@ def _resolve_from_index(return None-def local_gguf_pinned_variant(requested: str) -> Optional[str]:+def local_gguf_pinned_variant(requested: str, allow_scan: bool = True) -> Optional[str]:"""The on-disk quant ``repo:VARIANT`` pins, by current label or legacy alias, else None.Unlike :func:`resolve_local_gguf`, a suffix naming no quant (``:latest``) never falls back to the- repo's preferred one.+ repo's preferred one. ``allow_scan=False`` answers from the last built index, never rebuilding it."""base, sep, wanted = requested.rpartition(":")if not sep:return Nonetry:- entry = _index().get(base.strip().lower())+ entry = (_index() if allow_scan else _snapshot()[1]).get(base.strip().lower())except Exception:return Noneif entry is None:diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex cfcaae87e38..a831910a46b 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -36521,11 +36521,12 @@ def _audio_cpp_speech_model_objects(created: int) -> list[dict]:return []-async def _openai_catalog_objects() -> list[dict]:+async def _openai_catalog_objects(quant_pins: bool = False) -> list[dict]:"""Every model the server knows about for ``GET /v1/models``: the loadedmodel(s) plus locally available (downloaded/cached) models discovered byscanning. Loaded entries keep their context fields and are marked- ``loaded: true``. All ids are clean public ids (never absolute paths)."""+ ``loaded: true``. All ids are clean public ids (never absolute paths). ``quant_pins``+ adds a ``<id>:<quant>`` row per on-disk quant of a multi-quant GGUF repo."""_created = int(time.time())# Loaded models first (clean ids + context fields), marked loaded.by_id: dict[str, dict] = {}@@ -36544,11 +36545,16 @@ async def _openai_catalog_objects() -> list[dict]:catalog = await asyncio.to_thread(account_access.filter_model_rows, catalog)visible = {id(info) for info in catalog}rows = [row for row in rows if id(row[0]) in visible]+ pins: dict[str, tuple[str, ...]] = {}for info, is_gguf, quants, loaded in rows:if account_access.resident_hidden("chat"):loaded = Falsecid = getattr(info, "model_id", None) or public_model_id(getattr(info, "id", None))- if not cid or cid in by_id:+ if not cid:+ continue+ if is_gguf and len(quants) > 1:+ pins.setdefault(cid, quants)+ if cid in by_id:continueif loaded and not is_gguf:if resident_id in by_id:@@ -36583,7 +36589,39 @@ async def _openai_catalog_objects() -> list[dict]:for obj in media:by_id.setdefault(obj["id"], obj)- return list(by_id.values())+ if not quant_pins or not pins:+ return list(by_id.values())+ # Off-loop: a loaded row on a cold index reads residency from every slot.+ return await asyncio.to_thread(_with_quant_pin_rows, list(by_id.values()), pins)+++def _with_quant_pin_rows(objects: list[dict], pins: dict[str, tuple[str, ...]]) -> list[dict]:+ """*objects* plus, after each multi-quant repo's bare row, one ``<id>:<quant>`` row per on-disk+ quant, as ``GET /v1/models/<id>:<quant>`` returns it: dropdown clients cannot type a pin (#9340)."""+ from core.inference.local_model_resolver import (+ index_is_built,+ local_gguf_pinned_variant,+ warm_index_soon,+ )++ # Never rebuild per listing (5 s TTL, multi-root scan); build only if it never was, or a+ # client that lists once at connect sees no pins.+ allow_scan = not index_is_built()+ warm_index_soon()+ listed = {obj["id"] for obj in objects}+ resident_quants = functools.lru_cache(maxsize = 1)(_resident_gguf_quants)+ out: list[dict] = []+ for obj in objects:+ out.append(obj)+ if obj.get("task"):+ continue+ out.extend(+ _quant_pin_object(obj, quant, resident_quants)+ for quant in pins.get(obj["id"], ())+ if f"{obj['id']}:{quant}" not in listed+ and local_gguf_pinned_variant(f"{obj['id']}:{quant}", allow_scan = allow_scan) == quant+ )+ return out@studio_router.get("/loaded-models")@@ -36617,10 +36655,10 @@ async def openai_list_models("""wanted = {m.strip() for m in (output_modalities or "").split(",")}if not wanted & {"all", "decisions"}:- return {"object": "list", "data": await _openai_catalog_objects()}+ return {"object": "list", "data": await _openai_catalog_objects(quant_pins = True)}from routes.systemone import decision_model_objects- data = [] if wanted == {"decisions"} else await _openai_catalog_objects()+ data = [] if wanted == {"decisions"} else await _openai_catalog_objects(quant_pins = True)data += await asyncio.to_thread(decision_model_objects)return {"object": "list", "data": data}@@ -36658,10 +36696,15 @@ def _pinned_quant_object(model_id: str, objects: list[dict]) -> Optional[dict]:variant = local_gguf_pinned_variant(f"{listed['id']}:{model_id[len(base) + 1:]}")if not variant:return None+ return _quant_pin_object(listed, variant, _resident_gguf_quants)+++def _quant_pin_object(listed: dict, variant: str, resident_quants) -> dict:+ """*listed* pinned to its on-disk *variant*: the loaded row when that quant is resident."""pinned_id = f"{listed['id']}:{variant}"if listed.get("loaded"):# A cold index leaves "quant" off the loaded row until it can prove the pin.- resident = listed.get("quant") or _resident_gguf_quants().get(listed["id"].lower())+ resident = listed.get("quant") or resident_quants().get(listed["id"].lower())if str(resident or "").lower() == variant.lower():return {**listed, "id": pinned_id, "quant": variant}# The listed row's context fields describe the resident quant, not this one.diff --git a/studio/backend/tests/test_openai_catalog.py b/studio/backend/tests/test_openai_catalog.pyindex 986e6230a42..42f20a182df 100644--- a/studio/backend/tests/test_openai_catalog.py+++ b/studio/backend/tests/test_openai_catalog.py@@ -677,3 +677,116 @@ def test_retrieve_pin_404s_when_the_quant_or_model_is_not_listed(monkeypatch):_pin_setup(monkeypatch, [])with pytest.raises(HTTPException):_retrieve("publisher/Qwen3:Q4_K_M")+++def _multi_quant_listing(+ monkeypatch,+ llama,+ quants = ("Q8_0", "Q4_K_M"),+):+ from core.inference.local_model_resolver import _LocalGgufEntry++ repo = _Info("/hf/models--publisher--Qwen3", "Qwen3", model_id = "publisher/Qwen3")+ repo.path = "/hf/models--publisher--Qwen3"+ single = _Info("/hf/models--org--Foo", "Foo", model_id = "org/Foo")+ single.path = "/hf/models--org--Foo"+ mistral = _Info("/data/models/Mistral-7B", "Mistral-7B", is_gguf = False)+ mistral.path = "/data/models/Mistral-7B"++ async def _fake_catalog():+ return [repo, single, mistral]++ def _servable(info):+ if not info.is_gguf:+ return (False, ())+ return (True, quants) if info is repo else (True, ("Q4_K_M",))++ monkeypatch.setattr(inf, "_cached_local_catalog", _fake_catalog)+ monkeypatch.setattr(resolver, "local_servable_model", _servable)+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: llama)+ monkeypatch.setattr(inf, "get_inference_backend", lambda: _FakeUnsloth())+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: None)+ _index_holding(monkeypatch, quants)+++def _index_holding(monkeypatch, quants):+ from core.inference.local_model_resolver import _LocalGgufEntry++ index = {+ "publisher/qwen3": _LocalGgufEntry(+ "publisher/Qwen3", "/hf/models--publisher--Qwen3/snapshots/a", quants+ )+ }+ monkeypatch.setattr(resolver, "_scan", (1.0, index))+ monkeypatch.setattr(resolver, "_index", lambda: index)+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)+++def _listed():+ return asyncio.run(inf.openai_list_models(current_subject = "t"))["data"]+++def test_list_names_every_on_disk_quant_of_a_multi_quant_repo(monkeypatch):+ # #9340: dropdown-only clients cannot type ":Q4_K_M", so each quant needs its own row.+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))+ ids = [m["id"] for m in _listed()]+ at = ids.index("publisher/Qwen3")+ assert ids[at : at + 3] == ["publisher/Qwen3", "publisher/Qwen3:Q8_0", "publisher/Qwen3:Q4_K_M"]+ # One quant on disk, or no quant at all: the bare id already names it.+ assert not [i for i in ids if i.startswith(("org/Foo:", "Mistral-7B:"))]+ rows = {m["id"]: m for m in _listed()}+ assert rows["publisher/Qwen3"]["quant"] == "Q8_0"+ assert rows["publisher/Qwen3:Q4_K_M"] == {+ "id": "publisher/Qwen3:Q4_K_M",+ "object": "model",+ "created": rows["publisher/Qwen3"]["created"],+ "owned_by": "unsloth-studio",+ "display_name": "Qwen3",+ "loaded": False,+ "quant": "Q4_K_M",+ }+ # Every listed pin retrieves as the same object, so validating clients accept it.+ # Less "created": each call stamps its own second.+ for mid in ("publisher/Qwen3:Q8_0", "publisher/Qwen3:Q4_K_M"):+ assert {**_retrieve(mid), "created": 0} == {**rows[mid], "created": 0}+ # Internal callers (404 messages, retrieve) keep the bare catalog.+ assert "publisher/Qwen3:Q8_0" not in {+ m["id"] for m in asyncio.run(inf._openai_catalog_objects())+ }+++def test_list_marks_only_the_resident_quant_pin_loaded(monkeypatch):+ llama = _FakeLlama()+ llama.model_identifier = "publisher/Qwen3"+ llama.hf_variant = "Q4_K_M"+ _multi_quant_listing(monkeypatch, llama)+ monkeypatch.setattr(inf, "_llama_public_model_id", lambda _b: "publisher/Qwen3")+ rows = {m["id"]: m for m in _listed()}+ assert rows["publisher/Qwen3"]["loaded"] is True+ assert rows["publisher/Qwen3:Q4_K_M"]["loaded"] is True+ assert rows["publisher/Qwen3:Q4_K_M"]["context_length"] == 4096+ assert rows["publisher/Qwen3:Q8_0"]["loaded"] is False+ assert "context_length" not in rows["publisher/Qwen3:Q8_0"]+ assert [m["id"] for m in _listed()].count("publisher/Qwen3:Q4_K_M") == 1+++def test_list_skips_a_quant_pin_the_resolver_would_not_honour(monkeypatch):+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))+ _index_holding(monkeypatch, ("Q8_0",))+ ids = [m["id"] for m in _listed()]+ assert "publisher/Qwen3:Q8_0" in ids+ assert "publisher/Qwen3:Q4_K_M" not in ids+++def test_list_builds_the_index_for_quant_pins_only_when_it_never_was(monkeypatch):+ # The index TTL is 5 s; rebuilding it per listing is the multi-root scan the catalog caches avoid.+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))+ monkeypatch.setattr(resolver, "_index", lambda: pytest.fail("listing rebuilt the index"))+ assert "publisher/Qwen3:Q4_K_M" in [m["id"] for m in _listed()]++ # Never built (first listing after start): build it, or a client that reads the list once+ # at connect never sees a pin.+ built = resolver._scan[1]+ monkeypatch.setattr(resolver, "_scan", (0.0, {}))+ monkeypatch.setattr(resolver, "_index", lambda: built)+ assert "publisher/Qwen3:Q4_K_M" in [m["id"] for m in _listed()]diff --git a/studio/backend/tests/test_systemone_connections.py b/studio/backend/tests/test_systemone_connections.pyindex a01a8a60f1b..b753ac5f136 100644--- a/studio/backend/tests/test_systemone_connections.py+++ b/studio/backend/tests/test_systemone_connections.py@@ -505,7 +505,7 @@ def test_a_system_one_model_list_explains_what_went_wrong(upstream, reply, detaidef test_studio_lists_its_decision_models_only_when_asked(monkeypatch, studio):from routes import inference- async def chat_catalog():+ async def chat_catalog(quant_pins = False):return [{"id": "unsloth/Qwen3-0.6B", "object": "model"}]monkeypatch.setattr(inference, "_openai_catalog_objects", chat_catalog)