Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: list each on-disk quant of a multi-quant GGUF in GET /v1/models (#13422)

Daniel Hancfcf587af979
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py
index 2bcd1bd9c1f..899f934be88 100644
--- a/studio/backend/core/inference/local_model_resolver.py
+++ b/studio/backend/core/inference/local_model_resolver.py
@@ -1340,17 +1340,17 @@ def _resolve_from_index(
return None
-def local_gguf_pinned_variant(requested: str) -> Optional[str]:
+def local_gguf_pinned_variant(requested: str, allow_scan: bool = True) -> Optional[str]:
"""The on-disk quant ``repo:VARIANT`` pins, by current label or legacy alias, else None.
Unlike :func:`resolve_local_gguf`, a suffix naming no quant (``:latest``) never falls back to the
- repo's preferred one.
+ repo's preferred one. ``allow_scan=False`` answers from the last built index, never rebuilding it.
"""
base, sep, wanted = requested.rpartition(":")
if not sep:
return None
try:
- entry = _index().get(base.strip().lower())
+ entry = (_index() if allow_scan else _snapshot()[1]).get(base.strip().lower())
except Exception:
return None
if entry is None:
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index cfcaae87e38..a831910a46b 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -36521,11 +36521,12 @@ def _audio_cpp_speech_model_objects(created: int) -> list[dict]:
return []
-async def _openai_catalog_objects() -> list[dict]:
+async def _openai_catalog_objects(quant_pins: bool = False) -> list[dict]:
"""Every model the server knows about for ``GET /v1/models``: the loaded
model(s) plus locally available (downloaded/cached) models discovered by
scanning. Loaded entries keep their context fields and are marked
- ``loaded: true``. All ids are clean public ids (never absolute paths)."""
+ ``loaded: true``. All ids are clean public ids (never absolute paths). ``quant_pins``
+ adds a ``<id>:<quant>`` row per on-disk quant of a multi-quant GGUF repo."""
_created = int(time.time())
# Loaded models first (clean ids + context fields), marked loaded.
by_id: dict[str, dict] = {}
@@ -36544,11 +36545,16 @@ async def _openai_catalog_objects() -> list[dict]:
catalog = await asyncio.to_thread(account_access.filter_model_rows, catalog)
visible = {id(info) for info in catalog}
rows = [row for row in rows if id(row[0]) in visible]
+ pins: dict[str, tuple[str, ...]] = {}
for info, is_gguf, quants, loaded in rows:
if account_access.resident_hidden("chat"):
loaded = False
cid = getattr(info, "model_id", None) or public_model_id(getattr(info, "id", None))
- if not cid or cid in by_id:
+ if not cid:
+ continue
+ if is_gguf and len(quants) > 1:
+ pins.setdefault(cid, quants)
+ if cid in by_id:
continue
if loaded and not is_gguf:
if resident_id in by_id:
@@ -36583,7 +36589,39 @@ async def _openai_catalog_objects() -> list[dict]:
for obj in media:
by_id.setdefault(obj["id"], obj)
- return list(by_id.values())
+ if not quant_pins or not pins:
+ return list(by_id.values())
+ # Off-loop: a loaded row on a cold index reads residency from every slot.
+ return await asyncio.to_thread(_with_quant_pin_rows, list(by_id.values()), pins)
+
+
+def _with_quant_pin_rows(objects: list[dict], pins: dict[str, tuple[str, ...]]) -> list[dict]:
+ """*objects* plus, after each multi-quant repo's bare row, one ``<id>:<quant>`` row per on-disk
+ quant, as ``GET /v1/models/<id>:<quant>`` returns it: dropdown clients cannot type a pin (#9340)."""
+ from core.inference.local_model_resolver import (
+ index_is_built,
+ local_gguf_pinned_variant,
+ warm_index_soon,
+ )
+
+ # Never rebuild per listing (5 s TTL, multi-root scan); build only if it never was, or a
+ # client that lists once at connect sees no pins.
+ allow_scan = not index_is_built()
+ warm_index_soon()
+ listed = {obj["id"] for obj in objects}
+ resident_quants = functools.lru_cache(maxsize = 1)(_resident_gguf_quants)
+ out: list[dict] = []
+ for obj in objects:
+ out.append(obj)
+ if obj.get("task"):
+ continue
+ out.extend(
+ _quant_pin_object(obj, quant, resident_quants)
+ for quant in pins.get(obj["id"], ())
+ if f"{obj['id']}:{quant}" not in listed
+ and local_gguf_pinned_variant(f"{obj['id']}:{quant}", allow_scan = allow_scan) == quant
+ )
+ return out
@studio_router.get("/loaded-models")
@@ -36617,10 +36655,10 @@ async def openai_list_models(
"""
wanted = {m.strip() for m in (output_modalities or "").split(",")}
if not wanted & {"all", "decisions"}:
- return {"object": "list", "data": await _openai_catalog_objects()}
+ return {"object": "list", "data": await _openai_catalog_objects(quant_pins = True)}
from routes.systemone import decision_model_objects
- data = [] if wanted == {"decisions"} else await _openai_catalog_objects()
+ data = [] if wanted == {"decisions"} else await _openai_catalog_objects(quant_pins = True)
data += await asyncio.to_thread(decision_model_objects)
return {"object": "list", "data": data}
@@ -36658,10 +36696,15 @@ def _pinned_quant_object(model_id: str, objects: list[dict]) -> Optional[dict]:
variant = local_gguf_pinned_variant(f"{listed['id']}:{model_id[len(base) + 1:]}")
if not variant:
return None
+ return _quant_pin_object(listed, variant, _resident_gguf_quants)
+
+
+def _quant_pin_object(listed: dict, variant: str, resident_quants) -> dict:
+ """*listed* pinned to its on-disk *variant*: the loaded row when that quant is resident."""
pinned_id = f"{listed['id']}:{variant}"
if listed.get("loaded"):
# A cold index leaves "quant" off the loaded row until it can prove the pin.
- resident = listed.get("quant") or _resident_gguf_quants().get(listed["id"].lower())
+ resident = listed.get("quant") or resident_quants().get(listed["id"].lower())
if str(resident or "").lower() == variant.lower():
return {**listed, "id": pinned_id, "quant": variant}
# The listed row's context fields describe the resident quant, not this one.
diff --git a/studio/backend/tests/test_openai_catalog.py b/studio/backend/tests/test_openai_catalog.py
index 986e6230a42..42f20a182df 100644
--- a/studio/backend/tests/test_openai_catalog.py
+++ b/studio/backend/tests/test_openai_catalog.py
@@ -677,3 +677,116 @@ def test_retrieve_pin_404s_when_the_quant_or_model_is_not_listed(monkeypatch):
_pin_setup(monkeypatch, [])
with pytest.raises(HTTPException):
_retrieve("publisher/Qwen3:Q4_K_M")
+
+
+def _multi_quant_listing(
+ monkeypatch,
+ llama,
+ quants = ("Q8_0", "Q4_K_M"),
+):
+ from core.inference.local_model_resolver import _LocalGgufEntry
+
+ repo = _Info("/hf/models--publisher--Qwen3", "Qwen3", model_id = "publisher/Qwen3")
+ repo.path = "/hf/models--publisher--Qwen3"
+ single = _Info("/hf/models--org--Foo", "Foo", model_id = "org/Foo")
+ single.path = "/hf/models--org--Foo"
+ mistral = _Info("/data/models/Mistral-7B", "Mistral-7B", is_gguf = False)
+ mistral.path = "/data/models/Mistral-7B"
+
+ async def _fake_catalog():
+ return [repo, single, mistral]
+
+ def _servable(info):
+ if not info.is_gguf:
+ return (False, ())
+ return (True, quants) if info is repo else (True, ("Q4_K_M",))
+
+ monkeypatch.setattr(inf, "_cached_local_catalog", _fake_catalog)
+ monkeypatch.setattr(resolver, "local_servable_model", _servable)
+ monkeypatch.setattr(inf, "get_llama_cpp_backend", lambda: llama)
+ monkeypatch.setattr(inf, "get_inference_backend", lambda: _FakeUnsloth())
+ monkeypatch.setattr(inf, "_peek_inference_backend", lambda: None)
+ _index_holding(monkeypatch, quants)
+
+
+def _index_holding(monkeypatch, quants):
+ from core.inference.local_model_resolver import _LocalGgufEntry
+
+ index = {
+ "publisher/qwen3": _LocalGgufEntry(
+ "publisher/Qwen3", "/hf/models--publisher--Qwen3/snapshots/a", quants
+ )
+ }
+ monkeypatch.setattr(resolver, "_scan", (1.0, index))
+ monkeypatch.setattr(resolver, "_index", lambda: index)
+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)
+
+
+def _listed():
+ return asyncio.run(inf.openai_list_models(current_subject = "t"))["data"]
+
+
+def test_list_names_every_on_disk_quant_of_a_multi_quant_repo(monkeypatch):
+ # #9340: dropdown-only clients cannot type ":Q4_K_M", so each quant needs its own row.
+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))
+ ids = [m["id"] for m in _listed()]
+ at = ids.index("publisher/Qwen3")
+ assert ids[at : at + 3] == ["publisher/Qwen3", "publisher/Qwen3:Q8_0", "publisher/Qwen3:Q4_K_M"]
+ # One quant on disk, or no quant at all: the bare id already names it.
+ assert not [i for i in ids if i.startswith(("org/Foo:", "Mistral-7B:"))]
+ rows = {m["id"]: m for m in _listed()}
+ assert rows["publisher/Qwen3"]["quant"] == "Q8_0"
+ assert rows["publisher/Qwen3:Q4_K_M"] == {
+ "id": "publisher/Qwen3:Q4_K_M",
+ "object": "model",
+ "created": rows["publisher/Qwen3"]["created"],
+ "owned_by": "unsloth-studio",
+ "display_name": "Qwen3",
+ "loaded": False,
+ "quant": "Q4_K_M",
+ }
+ # Every listed pin retrieves as the same object, so validating clients accept it.
+ # Less "created": each call stamps its own second.
+ for mid in ("publisher/Qwen3:Q8_0", "publisher/Qwen3:Q4_K_M"):
+ assert {**_retrieve(mid), "created": 0} == {**rows[mid], "created": 0}
+ # Internal callers (404 messages, retrieve) keep the bare catalog.
+ assert "publisher/Qwen3:Q8_0" not in {
+ m["id"] for m in asyncio.run(inf._openai_catalog_objects())
+ }
+
+
+def test_list_marks_only_the_resident_quant_pin_loaded(monkeypatch):
+ llama = _FakeLlama()
+ llama.model_identifier = "publisher/Qwen3"
+ llama.hf_variant = "Q4_K_M"
+ _multi_quant_listing(monkeypatch, llama)
+ monkeypatch.setattr(inf, "_llama_public_model_id", lambda _b: "publisher/Qwen3")
+ rows = {m["id"]: m for m in _listed()}
+ assert rows["publisher/Qwen3"]["loaded"] is True
+ assert rows["publisher/Qwen3:Q4_K_M"]["loaded"] is True
+ assert rows["publisher/Qwen3:Q4_K_M"]["context_length"] == 4096
+ assert rows["publisher/Qwen3:Q8_0"]["loaded"] is False
+ assert "context_length" not in rows["publisher/Qwen3:Q8_0"]
+ assert [m["id"] for m in _listed()].count("publisher/Qwen3:Q4_K_M") == 1
+
+
+def test_list_skips_a_quant_pin_the_resolver_would_not_honour(monkeypatch):
+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))
+ _index_holding(monkeypatch, ("Q8_0",))
+ ids = [m["id"] for m in _listed()]
+ assert "publisher/Qwen3:Q8_0" in ids
+ assert "publisher/Qwen3:Q4_K_M" not in ids
+
+
+def test_list_builds_the_index_for_quant_pins_only_when_it_never_was(monkeypatch):
+ # The index TTL is 5 s; rebuilding it per listing is the multi-root scan the catalog caches avoid.
+ _multi_quant_listing(monkeypatch, _FakeLlama(loaded = False))
+ monkeypatch.setattr(resolver, "_index", lambda: pytest.fail("listing rebuilt the index"))
+ assert "publisher/Qwen3:Q4_K_M" in [m["id"] for m in _listed()]
+
+ # Never built (first listing after start): build it, or a client that reads the list once
+ # at connect never sees a pin.
+ built = resolver._scan[1]
+ monkeypatch.setattr(resolver, "_scan", (0.0, {}))
+ monkeypatch.setattr(resolver, "_index", lambda: built)
+ assert "publisher/Qwen3:Q4_K_M" in [m["id"] for m in _listed()]
diff --git a/studio/backend/tests/test_systemone_connections.py b/studio/backend/tests/test_systemone_connections.py
index a01a8a60f1b..b753ac5f136 100644
--- a/studio/backend/tests/test_systemone_connections.py
+++ b/studio/backend/tests/test_systemone_connections.py
@@ -505,7 +505,7 @@ def test_a_system_one_model_list_explains_what_went_wrong(upstream, reply, detai
def test_studio_lists_its_decision_models_only_when_asked(monkeypatch, studio):
from routes import inference
- async def chat_catalog():
+ async def chat_catalog(quant_pins = False):
return [{"id": "unsloth/Qwen3-0.6B", "object": "model"}]
monkeypatch.setattr(inference, "_openai_catalog_objects", chat_catalog)