Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: name an MLX or managed-engine token count by the id status publishes (#13282)
Daniel Hanfc5cf5017e30
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex fd2841d413c..e2df5ad67c8 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -40586,7 +40586,11 @@ async def _resident_model_reads_images() -> bool:return bool(entry.get("is_vision"))-async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONResponse]:+async def _mlx_count_chat_tokens(+ payload,+ request = None,+ via_api_key = None,+) -> Optional[JSONResponse]:"""Count with a resident MLX or managed engine tokenizer, otherwise return None.This has to answer the question the safetensors completion answers -- which of its two@@ -40878,7 +40882,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespondetail = "Cannot count tokens while a generation is in progress.",)try:- count, model = await asyncio.to_thread(+ count, _counted = await asyncio.to_thread(backend.count_chat_tokens,messages,system_prompt or "",@@ -40901,22 +40905,28 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONResponstatus_code = 503,detail = "The loaded model changed while counting tokens.",)- # The advertised id, not the resident path: an auto-switch loads from a resolved- # snapshot while publishing the repo id, and the caller drops a count whose model- # does not match the checkpoint it captured.+ # The id /status publishes as model_identifier (the raw active name, a local folder or+ # snapshot path included): the client drops a count naming any other checkpoint.return JSONResponse(- content = {- "input_tokens": int(count),- "model": _orchestrator_public_model_id(backend) or model or active,- }+ content = {"input_tokens": int(count), "model": _count_model_id(active, via_api_key)})+def _count_model_id(model_id, via_api_key):+ """A count's ``model``, redacted for an API-key caller exactly as /status redacts its+ ``model_identifier`` (the redactor keys on that field name)."""+ from hub.utils.host_paths import redact_host_paths+ return redact_host_paths({"model_identifier": model_id}, via_api_key = via_api_key)[+ "model_identifier"+ ]++@router.post("/chat/count_tokens")async def chat_count_tokens(payload: ChatCountTokensRequest,current_subject: str = Depends(get_current_subject),request: Request = None,+ via_api_key: bool = Depends(authenticated_via_api_key),):"""Count prompt tokens for OpenAI-form chat messages using the loaded tokenizer.@@ -41010,7 +41020,7 @@ async def chat_count_tokens(if not llama_backend.is_loaded:# The refusals above are about the request and so are shared; the ones below are# llama.cpp's own render concerns, so this returns rather than falling through.- _mlx_count = await _mlx_count_chat_tokens(payload, request)+ _mlx_count = await _mlx_count_chat_tokens(payload, request, via_api_key = via_api_key)if _mlx_count is not None:return _mlx_countraise HTTPException(@@ -41211,7 +41221,12 @@ async def chat_count_tokens(status_code = 503,detail = "The loaded model changed while counting tokens.",)- return JSONResponse(content = {"input_tokens": int(count), "model": _tokenizer_model})+ return JSONResponse(+ content = {+ "input_tokens": int(count),+ "model": _count_model_id(_tokenizer_model, via_api_key),+ }+ )@router.post("/messages/count_tokens")diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.pyindex 96aaa3dbf7d..8c73c092cb7 100644--- a/studio/backend/tests/test_chat_input_audio_part.py+++ b/studio/backend/tests/test_chat_input_audio_part.py@@ -208,12 +208,13 @@ def _route_client(prefix = ""):"""The real inference router, with only the auth dependency stubbed."""from fastapi import FastAPIfrom fastapi.testclient import TestClient- from auth.authentication import get_current_subject+ from auth.authentication import authenticated_via_api_key, get_current_subjectimport routes.inference as inference_routeapp = FastAPI()app.include_router(inference_route.router, prefix = prefix)app.dependency_overrides[get_current_subject] = lambda: "test"+ app.dependency_overrides[authenticated_via_api_key] = lambda: Falsereturn TestClient(app, raise_server_exceptions = False)diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.pyindex 71696423ccb..b435df9534d 100644--- a/studio/backend/tests/test_inference_status_route.py+++ b/studio/backend/tests/test_inference_status_route.py@@ -235,6 +235,33 @@ def test_status_reports_the_mlx_video_capability(monkeypatch):assert response.has_video_input is True[email protected]("via_api_key", [False, True], ids = ["ui_session", "api_key"])+def test_status_publishes_an_mlx_local_folder_as_the_count_names_it(monkeypatch, via_api_key):+ """/chat/count_tokens answers with this same id (redacted the same way for an API-key caller);+ the client drops a count naming any other checkpoint."""+ from hub.utils.host_paths import redact_host_paths++ active = "/Users/alice/outputs/my-finetune-mlx"+ backend = _FakeInferenceBackend()+ backend.active_model_name = active+ backend.models = {active: {"is_mlx": True}}+ backend.loading_models = set()+ _patch_fast_status(monkeypatch, backend)+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)++ response = asyncio.run(+ inference_route.inference_status(current_subject = "test", via_api_key = via_api_key)+ )++ # Redaction dumps the response model to a dict.+ published = (+ response["model_identifier"] if isinstance(response, dict) else response.model_identifier+ )+ expected = redact_host_paths({"model_identifier": active}, via_api_key = via_api_key)+ assert published == expected["model_identifier"]+ assert (published == active) is (not via_api_key)++def test_status_reports_no_video_capability_for_a_model_without_one(monkeypatch):backend = _FakeInferenceBackend()backend.active_model_name = "org/plain-vision-model"diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex b5712df569e..b22dffbaf87 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -5930,15 +5930,56 @@ def test_an_mlx_count_prices_the_date_an_api_key_tool_loop_still_gets(monkeypatc), f"the count dropped the date the tool-loop completion adds: {backend.system!r}"-def test_an_mlx_count_reports_the_advertised_model_id(monkeypatch):- """The caller drops a count naming a different model, so one loaded from a resolved- path must still answer as the repo id."""+ ("active", "advertised"),+ [+ ("/Users/alice/outputs/my-finetune-mlx", None),+ ("/hf/hub/models--org--repo/snapshots/abc123", "org/repo"),+ ("mlx-community/Qwen3.5-2B-4bit", None),+ ],+ ids = ["local_folder", "auto_switch_snapshot", "hub_repo_id"],+)+def test_an_mlx_count_reports_the_id_status_publishes(monkeypatch, active, advertised):+ """/status publishes the raw active name as model_identifier and the client adopts it as its+ checkpoint, dropping any count that names another model; a cleaned or advertised id here+ left a local-folder or auto-switched MLX / vLLM model on the estimate forever."""+ backend = _RenderRecordingBackend()+ backend.active_model_name = active+ backend._openai_advertised_id = advertised+ models = {active: next(iter(_mirror(_TOOL_TEMPLATE).values()))}+ served = _count_route(+ monkeypatch, backend, models = models, messages = [{"role": "user", "content": "hello"}]+ )+ assert json.loads(served.body)["model"] == active++[email protected]("via_api_key", [True, False], ids = ["api_key", "ui_session"])+def test_an_mlx_count_redacts_a_local_folder_for_an_api_key_caller(monkeypatch, via_api_key):+ """/status hides host paths from API-key callers; the count must not hand the same path back."""+ from hub.utils.host_paths import redact_host_pathsfrom routes import inference as route+ active = "/Users/alice/outputs/my-finetune-mlx"backend = _RenderRecordingBackend()- monkeypatch.setattr(route, "_orchestrator_public_model_id", lambda _b: "org/advertised-repo-id")- served = _count_route(monkeypatch, backend, messages = [{"role": "user", "content": "hello"}])- assert json.loads(served.body)["model"] == "org/advertised-repo-id"+ backend.active_model_name = active+ backend.models = {active: next(iter(_mirror(_TOOL_TEMPLATE).values()))}+ monkeypatch.setattr(route, "get_inference_backend", lambda: backend)+ monkeypatch.setattr(route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False))+ monkeypatch.setattr(route.active_generations, "count", lambda: 0)+ served = asyncio.run(+ route.chat_count_tokens(+ route.ChatCountTokensRequest(messages = [{"role": "user", "content": "hi"}]),+ current_subject = "tester",+ via_api_key = via_api_key,+ )+ )+ model = json.loads(served.body)["model"]+ if via_api_key:+ assert "/Users/alice" not in model+ status_view = redact_host_paths({"model_identifier": active}, via_api_key = True)+ assert model == status_view["model_identifier"]+ else:+ assert model == activedef test_an_mlx_count_is_dropped_when_a_same_model_reload_lands_under_it(monkeypatch):diff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.pyindex f7dda71bce3..8fcc09596f3 100644--- a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py@@ -23,7 +23,7 @@ from fastapi import FastAPI # noqa: E402from fastapi.testclient import TestClient # noqa: E402from PIL import Image # noqa: E402-from auth.authentication import get_current_subject # noqa: E402+from auth.authentication import authenticated_via_api_key, get_current_subject # noqa: E402import core.inference.external_provider as external_provider # noqa: E402import routes.inference as inference_route # noqa: E402@@ -74,6 +74,7 @@ def _client(monkeypatch, backend):app = FastAPI()app.include_router(inference_route.router, prefix = "/v1")app.dependency_overrides[get_current_subject] = lambda: "tester"+ app.dependency_overrides[authenticated_via_api_key] = lambda: Falsereturn TestClient(app, raise_server_exceptions = False)diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.pyindex 8b0ee439f22..71ea2adc725 100644--- a/studio/backend/tests/test_video_chat_completions_behaviour.py+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py@@ -26,7 +26,7 @@ pytest.importorskip("torch")from fastapi import FastAPI # noqa: E402from fastapi.testclient import TestClient # noqa: E402-from auth.authentication import get_current_subject # noqa: E402+from auth.authentication import authenticated_via_api_key, get_current_subject # noqa: E402import routes.inference as inference_route # noqa: E402from .llama_backend_double import FakeLlamaCppBackend # noqa: E402@@ -76,6 +76,7 @@ def _client(app = FastAPI()app.include_router(inference_route.router, prefix = prefix)app.dependency_overrides[get_current_subject] = lambda: "tester"+ app.dependency_overrides[authenticated_via_api_key] = lambda: Falsereturn TestClient(app, raise_server_exceptions = False)