Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: name an MLX or managed-engine token count by the id status publishes (#13282)

Daniel Hanfc5cf5017e30
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index fd2841d413c..e2df5ad67c8 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -40586,7 +40586,11 @@ async def _resident_model_reads_images() -> bool:
return bool(entry.get("is_vision"))
-async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONResponse]:
+async def _mlx_count_chat_tokens(
+ payload,
+ request = None,
+ via_api_key = None,
+) -> Optional[JSONResponse]:
"""Count with a resident MLX or managed engine tokenizer, otherwise return None.
This has to answer the question the safetensors completion answers -- which of its two
@@ -40878,7 +40882,7 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
detail = "Cannot count tokens while a generation is in progress.",
)
try:
- count, model = await asyncio.to_thread(
+ count, _counted = await asyncio.to_thread(
backend.count_chat_tokens,
messages,
system_prompt or "",
@@ -40901,22 +40905,28 @@ async def _mlx_count_chat_tokens(payload, request = None) -> Optional[JSONRespon
status_code = 503,
detail = "The loaded model changed while counting tokens.",
)
- # The advertised id, not the resident path: an auto-switch loads from a resolved
- # snapshot while publishing the repo id, and the caller drops a count whose model
- # does not match the checkpoint it captured.
+ # The id /status publishes as model_identifier (the raw active name, a local folder or
+ # snapshot path included): the client drops a count naming any other checkpoint.
return JSONResponse(
- content = {
- "input_tokens": int(count),
- "model": _orchestrator_public_model_id(backend) or model or active,
- }
+ content = {"input_tokens": int(count), "model": _count_model_id(active, via_api_key)}
)
+def _count_model_id(model_id, via_api_key):
+ """A count's ``model``, redacted for an API-key caller exactly as /status redacts its
+ ``model_identifier`` (the redactor keys on that field name)."""
+ from hub.utils.host_paths import redact_host_paths
+ return redact_host_paths({"model_identifier": model_id}, via_api_key = via_api_key)[
+ "model_identifier"
+ ]
+
+
@router.post("/chat/count_tokens")
async def chat_count_tokens(
payload: ChatCountTokensRequest,
current_subject: str = Depends(get_current_subject),
request: Request = None,
+ via_api_key: bool = Depends(authenticated_via_api_key),
):
"""Count prompt tokens for OpenAI-form chat messages using the loaded tokenizer.
@@ -41010,7 +41020,7 @@ async def chat_count_tokens(
if not llama_backend.is_loaded:
# The refusals above are about the request and so are shared; the ones below are
# llama.cpp's own render concerns, so this returns rather than falling through.
- _mlx_count = await _mlx_count_chat_tokens(payload, request)
+ _mlx_count = await _mlx_count_chat_tokens(payload, request, via_api_key = via_api_key)
if _mlx_count is not None:
return _mlx_count
raise HTTPException(
@@ -41211,7 +41221,12 @@ async def chat_count_tokens(
status_code = 503,
detail = "The loaded model changed while counting tokens.",
)
- return JSONResponse(content = {"input_tokens": int(count), "model": _tokenizer_model})
+ return JSONResponse(
+ content = {
+ "input_tokens": int(count),
+ "model": _count_model_id(_tokenizer_model, via_api_key),
+ }
+ )
@router.post("/messages/count_tokens")
diff --git a/studio/backend/tests/test_chat_input_audio_part.py b/studio/backend/tests/test_chat_input_audio_part.py
index 96aaa3dbf7d..8c73c092cb7 100644
--- a/studio/backend/tests/test_chat_input_audio_part.py
+++ b/studio/backend/tests/test_chat_input_audio_part.py
@@ -208,12 +208,13 @@ def _route_client(prefix = ""):
"""The real inference router, with only the auth dependency stubbed."""
from fastapi import FastAPI
from fastapi.testclient import TestClient
- from auth.authentication import get_current_subject
+ from auth.authentication import authenticated_via_api_key, get_current_subject
import routes.inference as inference_route
app = FastAPI()
app.include_router(inference_route.router, prefix = prefix)
app.dependency_overrides[get_current_subject] = lambda: "test"
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
return TestClient(app, raise_server_exceptions = False)
diff --git a/studio/backend/tests/test_inference_status_route.py b/studio/backend/tests/test_inference_status_route.py
index 71696423ccb..b435df9534d 100644
--- a/studio/backend/tests/test_inference_status_route.py
+++ b/studio/backend/tests/test_inference_status_route.py
@@ -235,6 +235,33 @@ def test_status_reports_the_mlx_video_capability(monkeypatch):
assert response.has_video_input is True
[email protected]("via_api_key", [False, True], ids = ["ui_session", "api_key"])
+def test_status_publishes_an_mlx_local_folder_as_the_count_names_it(monkeypatch, via_api_key):
+ """/chat/count_tokens answers with this same id (redacted the same way for an API-key caller);
+ the client drops a count naming any other checkpoint."""
+ from hub.utils.host_paths import redact_host_paths
+
+ active = "/Users/alice/outputs/my-finetune-mlx"
+ backend = _FakeInferenceBackend()
+ backend.active_model_name = active
+ backend.models = {active: {"is_mlx": True}}
+ backend.loading_models = set()
+ _patch_fast_status(monkeypatch, backend)
+ monkeypatch.setattr(inference_route, "load_inference_config", lambda _model: None)
+
+ response = asyncio.run(
+ inference_route.inference_status(current_subject = "test", via_api_key = via_api_key)
+ )
+
+ # Redaction dumps the response model to a dict.
+ published = (
+ response["model_identifier"] if isinstance(response, dict) else response.model_identifier
+ )
+ expected = redact_host_paths({"model_identifier": active}, via_api_key = via_api_key)
+ assert published == expected["model_identifier"]
+ assert (published == active) is (not via_api_key)
+
+
def test_status_reports_no_video_capability_for_a_model_without_one(monkeypatch):
backend = _FakeInferenceBackend()
backend.active_model_name = "org/plain-vision-model"
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index b5712df569e..b22dffbaf87 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -5930,15 +5930,56 @@ def test_an_mlx_count_prices_the_date_an_api_key_tool_loop_still_gets(monkeypatc
), f"the count dropped the date the tool-loop completion adds: {backend.system!r}"
-def test_an_mlx_count_reports_the_advertised_model_id(monkeypatch):
- """The caller drops a count naming a different model, so one loaded from a resolved
- path must still answer as the repo id."""
+ ("active", "advertised"),
+ [
+ ("/Users/alice/outputs/my-finetune-mlx", None),
+ ("/hf/hub/models--org--repo/snapshots/abc123", "org/repo"),
+ ("mlx-community/Qwen3.5-2B-4bit", None),
+ ],
+ ids = ["local_folder", "auto_switch_snapshot", "hub_repo_id"],
+)
+def test_an_mlx_count_reports_the_id_status_publishes(monkeypatch, active, advertised):
+ """/status publishes the raw active name as model_identifier and the client adopts it as its
+ checkpoint, dropping any count that names another model; a cleaned or advertised id here
+ left a local-folder or auto-switched MLX / vLLM model on the estimate forever."""
+ backend = _RenderRecordingBackend()
+ backend.active_model_name = active
+ backend._openai_advertised_id = advertised
+ models = {active: next(iter(_mirror(_TOOL_TEMPLATE).values()))}
+ served = _count_route(
+ monkeypatch, backend, models = models, messages = [{"role": "user", "content": "hello"}]
+ )
+ assert json.loads(served.body)["model"] == active
+
+
[email protected]("via_api_key", [True, False], ids = ["api_key", "ui_session"])
+def test_an_mlx_count_redacts_a_local_folder_for_an_api_key_caller(monkeypatch, via_api_key):
+ """/status hides host paths from API-key callers; the count must not hand the same path back."""
+ from hub.utils.host_paths import redact_host_paths
from routes import inference as route
+ active = "/Users/alice/outputs/my-finetune-mlx"
backend = _RenderRecordingBackend()
- monkeypatch.setattr(route, "_orchestrator_public_model_id", lambda _b: "org/advertised-repo-id")
- served = _count_route(monkeypatch, backend, messages = [{"role": "user", "content": "hello"}])
- assert json.loads(served.body)["model"] == "org/advertised-repo-id"
+ backend.active_model_name = active
+ backend.models = {active: next(iter(_mirror(_TOOL_TEMPLATE).values()))}
+ monkeypatch.setattr(route, "get_inference_backend", lambda: backend)
+ monkeypatch.setattr(route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False))
+ monkeypatch.setattr(route.active_generations, "count", lambda: 0)
+ served = asyncio.run(
+ route.chat_count_tokens(
+ route.ChatCountTokensRequest(messages = [{"role": "user", "content": "hi"}]),
+ current_subject = "tester",
+ via_api_key = via_api_key,
+ )
+ )
+ model = json.loads(served.body)["model"]
+ if via_api_key:
+ assert "/Users/alice" not in model
+ status_view = redact_host_paths({"model_identifier": active}, via_api_key = True)
+ assert model == status_view["model_identifier"]
+ else:
+ assert model == active
def test_an_mlx_count_is_dropped_when_a_same_model_reload_lands_under_it(monkeypatch):
diff --git a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
index f7dda71bce3..8fcc09596f3 100644
--- a/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
+++ b/studio/backend/tests/test_remote_image_url_fetch_behaviour.py
@@ -23,7 +23,7 @@ from fastapi import FastAPI # noqa: E402
from fastapi.testclient import TestClient # noqa: E402
from PIL import Image # noqa: E402
-from auth.authentication import get_current_subject # noqa: E402
+from auth.authentication import authenticated_via_api_key, get_current_subject # noqa: E402
import core.inference.external_provider as external_provider # noqa: E402
import routes.inference as inference_route # noqa: E402
@@ -74,6 +74,7 @@ def _client(monkeypatch, backend):
app = FastAPI()
app.include_router(inference_route.router, prefix = "/v1")
app.dependency_overrides[get_current_subject] = lambda: "tester"
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
return TestClient(app, raise_server_exceptions = False)
diff --git a/studio/backend/tests/test_video_chat_completions_behaviour.py b/studio/backend/tests/test_video_chat_completions_behaviour.py
index 8b0ee439f22..71ea2adc725 100644
--- a/studio/backend/tests/test_video_chat_completions_behaviour.py
+++ b/studio/backend/tests/test_video_chat_completions_behaviour.py
@@ -26,7 +26,7 @@ pytest.importorskip("torch")
from fastapi import FastAPI # noqa: E402
from fastapi.testclient import TestClient # noqa: E402
-from auth.authentication import get_current_subject # noqa: E402
+from auth.authentication import authenticated_via_api_key, get_current_subject # noqa: E402
import routes.inference as inference_route # noqa: E402
from .llama_backend_double import FakeLlamaCppBackend # noqa: E402
@@ -76,6 +76,7 @@ def _client(
app = FastAPI()
app.include_router(inference_route.router, prefix = prefix)
app.dependency_overrides[get_current_subject] = lambda: "tester"
+ app.dependency_overrides[authenticated_via_api_key] = lambda: False
return TestClient(app, raise_server_exceptions = False)