Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: logprobs on /v1/completions and /tokenize for MLX models (#14229)
Daniel Han6a19cb20b98c
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 13415cf03fb..bda64ad75f1 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -3289,6 +3289,71 @@ def _raw_prompt_constraint(grammar, tokenizer):return GrammarConstraint(grammar, tokenizer, strips_special_tokens = True)+class LogprobsUnsupported(ValueError):+ """``n_probs`` asked of a model that does not sample one token per step."""++ public = True+ openai_param = "logprobs"+++def _make_mlx_raw_logprob_recorder():+ """A first logits processor queueing each step's log-softmax of the model's own logits, and+ the queue. llama-server reports ``n_probs`` from the logits before any sampler, so a token+ forced by ``logit_bias`` keeps its real logprob; mlx_lm and mlx_vlm compute theirs after the+ processors. Neither runtime calls processors during prefill, so entry i is generated token i."""+ from collections import deque++ import mlx.core as mx++ recorded = deque()++ def _processor(tokens, logits):+ raw = logits.astype(mx.float32)+ recorded.append(raw - mx.logsumexp(raw, axis = -1, keepdims = True))+ return logits++ return _processor, recorded+++def _token_bytes_text(tokenizer, token_id):+ kw = dict(skip_special_tokens = False, clean_up_tokenization_spaces = False)+ # SentencePiece decoders strip a sequence's leading space, so "▁the" alone decodes to+ # "the"; after a copy of itself it keeps the " the" llama-server reports.+ alone = tokenizer.decode([token_id], **kw)+ twice = tokenizer.decode([token_id, token_id], **kw)+ text = twice[len(alone) :] if twice.startswith(alone) else alone+ return text, list(text.encode("utf-8"))+++def _mlx_token_logprob(raw, token_id, n_probs, tokenizer) -> dict:+ """One llama-server ``logprobs.content`` entry: the sampled token's logprob and the+ ``n_probs`` most likely tokens, all read from the unprocessed logits."""+ import mlx.core as mx++ raw = raw.reshape(-1)+ n = min(int(n_probs), raw.shape[0])+ top = mx.argpartition(-raw, kth = n - 1)[:n]+ values = raw[top]+ order = mx.argsort(-values)+ # llama-server writes the lowest float for -inf, which JSON cannot carry.+ floor = -3.4028234663852886e38+ entry_ids, entry_values = top[order].tolist(), values[order].tolist()+ text, data = _token_bytes_text(tokenizer, int(token_id))+ top_logprobs = []+ for tid, value in zip(entry_ids, entry_values):+ tid_text, tid_bytes = _token_bytes_text(tokenizer, int(tid))+ top_logprobs.append(+ {"id": int(tid), "token": tid_text, "bytes": tid_bytes, "logprob": max(value, floor)}+ )+ return {+ "id": int(token_id),+ "token": text,+ "bytes": data,+ "logprob": max(raw[int(token_id)].item(), floor),+ "top_logprobs": top_logprobs,+ }++def _with_prompt_history(processor, prompt, window):"""*processor* also scoring the prompt tokens the runtime withholds: mlx_lm and mlx_vlm handprocessors only what is left of the prompt after prefill and prompt-cache reuse."""@@ -3577,6 +3642,8 @@ def _request_batch_gap(request):return "a reply runs the tool protocol"if request.get("response_format") is not None or request.get("grammar") is not None:return "a reply is constrained to a response format"+ if request.get("n_probs"):+ return "a reply reports token probabilities"return None@@ -5613,9 +5680,11 @@ class MLXInferenceBackend:stop = None,penalty_window = None,grammar = None,+ n_probs = 0,) -> Generator[str, None, None]:"""Cumulative text sampled after ``prompt_ids`` with no chat template.- ``penalty_window`` is llama-server's ``repeat_last_n``."""+ ``penalty_window`` is llama-server's ``repeat_last_n``; ``n_probs`` > 0 adds each+ token's ``logprobs.content`` entry to the stats as ``token_logprobs``."""if self._model is None:raise RuntimeError("No model loaded")self.last_generation_stats = None@@ -5629,6 +5698,8 @@ class MLXInferenceBackend:penalty_window = penalty_window,grammar = grammar,)+ if n_probs:+ sampling["n_probs"] = int(n_probs)args = (temperature, top_p, top_k, min_p, max_new_tokens, repetition_penalty, cancel_event)if self._is_vlm:yield from self._generate_vlm(None, None, *args, **sampling)@@ -5668,6 +5739,7 @@ class MLXInferenceBackend:prompt_ids = None,penalty_window = None,grammar = None,+ n_probs = 0,):from mlx_lm import stream_generatefrom mlx_lm.sample_utils import make_sampler@@ -5744,6 +5816,11 @@ class MLXInferenceBackend:penalty_window = penalty_window,penalty_prompt = prompt_ids,)+ recorded = token_logprobs = None+ if n_probs:+ recorder, recorded = _make_mlx_raw_logprob_recorder()+ logits_processors = [recorder, *(logits_processors or ())]+ token_logprobs = []native_token_decoder = (NativeToolTokenDecoder(@@ -5840,6 +5917,12 @@ class MLXInferenceBackend:):final_response = responsetoken_ids.append(response.token)+ if recorded:+ token_logprobs.append(+ _mlx_token_logprob(+ recorded.popleft(), response.token, n_probs, self._tokenizer+ )+ )if preserve_native_channels:sampled += streamed_text.feed(response)if sequences:@@ -5929,6 +6012,8 @@ class MLXInferenceBackend:max_new_tokens,),)+ if token_logprobs is not None:+ self.last_generation_stats["token_logprobs"] = token_logprobs# The turn's settled text: delivered once for the plain path, as the tail for the native-channel one. Every# snapshot was matched as it arrived, so a turn no sequence ended owes all of its text, held-back partial# included.@@ -6391,6 +6476,7 @@ class MLXInferenceBackend:prompt_ids = None,penalty_window = None,grammar = None,+ n_probs = 0,):from mlx_vlm import stream_generate as vlm_stream@@ -6505,6 +6591,15 @@ class MLXInferenceBackend:"constrain the next token. Load an autoregressive model to use ""guided decoding.")+ recorded = token_logprobs = None+ if n_probs:+ if _vlm_generation_is_diffusion(self._model):+ raise LogprobsUnsupported(+ "logprobs is not supported on this model: it generates by diffusion rather "+ "than one token at a time."+ )+ recorder, recorded = _make_mlx_raw_logprob_recorder()+ token_logprobs = []draft = self._speculative_draftif (presence_penalty@@ -6513,6 +6608,7 @@ class MLXInferenceBackend:or constraint is not Noneor (_rep_active and draft is not None)or penalty_window is not None+ or recorded is not None):# These need custom processors: pass the full list (repetition + the rest) instead of the# repetition_penalty shortcut so all apply.@@ -6525,6 +6621,12 @@ class MLXInferenceBackend:penalty_window = penalty_window,penalty_prompt = prompt_ids,)+ if recorded is not None:+ # Not history_only, so no draft runs: its verify pass would record rejected tokens.+ vlm_kwargs["logits_processors"] = [+ recorder,+ *(vlm_kwargs["logits_processors"] or ()),+ ]elif _rep_active:vlm_kwargs["repetition_penalty"] = float(repetition_penalty)@@ -6678,6 +6780,15 @@ class MLXInferenceBackend:):# The streaming detokenizer trims the first leading space; decode the ids.raw_ids.append(int(raw_id))+ if recorded:+ token_logprobs.append(+ _mlx_token_logprob(+ recorded.popleft(),+ int(raw_id),+ n_probs,+ self._tokenizer,+ )+ )decoded = self._tokenizer.decode(raw_ids,skip_special_tokens = True,@@ -6769,6 +6880,8 @@ class MLXInferenceBackend:if draft is Noneelse (draft.draft_n, draft.draft_n_accepted),)+ if token_logprobs is not None:+ self.last_generation_stats["token_logprobs"] = token_logprobsif document_only or prompt_ids is not None:# The normalizer would rewrite marker text inside the document.diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex cf9600e9459..e42822157d3 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -1470,6 +1470,7 @@ class InferenceOrchestrator:prompt_ids: Optional[list] = None,penalty_window: Optional[int] = None,grammar: Optional[str] = None,+ n_probs: int = 0,) -> dict:"""Build the 'generate' command shared by the locked and dispatched paths."""cmd = {@@ -1524,6 +1525,8 @@ class InferenceOrchestrator:cmd["penalty_window"] = penalty_windowif grammar is not None:cmd["grammar"] = grammar+ if n_probs:+ cmd["n_probs"] = int(n_probs)return cmddef _consume_token_stream(@@ -1775,6 +1778,7 @@ class InferenceOrchestrator:prompt_ids: Optional[list] = None,penalty_window: Optional[int] = None,grammar: Optional[str] = None,+ n_probs: int = 0,) -> Generator[Any, None, None]:"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-requestmailbox for tokens so two compare-mode requests can be queued at once. The subprocess@@ -1835,6 +1839,7 @@ class InferenceOrchestrator:prompt_ids = prompt_ids,penalty_window = penalty_window,grammar = grammar,+ n_probs = n_probs,)mailbox = _WorkerMailbox(self._proc)@@ -3595,6 +3600,7 @@ class InferenceOrchestrator:prompt_ids: Optional[list] = None,penalty_window: Optional[int] = None,grammar: Optional[str] = None,+ n_probs: int = 0,) -> Generator[Any, None, None]:"""Inner generation logic: sends the command to the subprocess and yields tokens. Serializedby _gen_lock (one generation at a time) so concurrent readers don't consume each other's@@ -3602,7 +3608,7 @@ class InferenceOrchestrator:managed = getattr(self, "_managed_engine", None)if managed is not None:kwargs = dict(locals())- for key in ("self", "managed", "kwargs", "diffusion_frames"):+ for key in ("self", "managed", "kwargs", "diffusion_frames", "n_probs"):kwargs.pop(key, None)from .engine_transport import EngineHTTPError@@ -3661,6 +3667,7 @@ class InferenceOrchestrator:prompt_ids = prompt_ids,penalty_window = penalty_window,grammar = grammar,+ n_probs = n_probs,)return@@ -3706,6 +3713,7 @@ class InferenceOrchestrator:prompt_ids = prompt_ids,penalty_window = penalty_window,grammar = grammar,+ n_probs = n_probs,)read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex a663a746b73..ad0212de98b 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -1195,7 +1195,7 @@ def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:raise RuntimeError("The loaded backend does not read video.")gen_kwargs["video"] = cmd["video_base64"]# Only a raw completion carries these; its batch rows are planned from them.- for raw in ("prompt_ids", "penalty_window", "grammar"):+ for raw in ("prompt_ids", "penalty_window", "grammar", "n_probs"):if raw in cmd:gen_kwargs[raw] = cmd[raw]@@ -1279,6 +1279,7 @@ _COMPLETION_OPTIONS = ("stop","penalty_window","grammar",+ "n_probs",)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex d1a2674455f..455a55ada9c 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -39056,13 +39056,14 @@ _LLAMA_DEFAULT_SAMPLERS = (# (tools/server/server-schema.cpp) plus OpenAI's. A field llama-server reads only while another# is on is off with it._MLX_COMPLETION_FIELDS_OFF = {- "logprobs": lambda v, b: v == 0,- "n_probs": lambda v, b: v == 0,# best_of equal to the choice count asks for no hidden candidates; llama-server ignores it."best_of": lambda v, b: v in (1, _llama_value(b, "n_cmpl")),"suffix": lambda v, b: v == "",# A lazy grammar waits for a trigger; this one constrains from the first token."grammar_lazy": lambda v, b: v is False or _llama_grammar_off(b),+ # Ours are pre-sampler; llama-server's post_sampling_probs flips that and renames fields,+ # but only with n_probs, so refuse only together.+ "post_sampling_probs": lambda v, b: v is False or not _llama_value(b, "n_probs"),"timings_per_token": lambda v, b: v is False,"return_tokens": lambda v, b: v is False,"return_progress": lambda v, b: v is False,@@ -39215,6 +39216,17 @@ def _mlx_completions_refusal(body: dict) -> Optional[str]:)if body.get("stream") is True and len(prompts) * (_llama_value(body, "n_cmpl") or 1) > 1:return "Streaming more than one completion is not supported for MLX models."+ n_probs = _llama_value(body, "n_probs")+ if n_probs not in (None, False) and not (_is_token_id(n_probs) and n_probs >= 0):+ return f"Field 'n_probs': Value must be a non-negative integer, but got {n_probs}"+ if n_probs:+ # Only the reported tokens: a prompt's own logprobs (echo) are not computed.+ if body.get("stream") is True:+ return "'logprobs' is not supported with stream for MLX models."+ if body.get("echo") is True:+ return "'logprobs' is not supported with echo for MLX models."+ if len(prompts) * (_llama_value(body, "n_cmpl") or 1) > 1:+ return "'logprobs' is supported for one completion per request on MLX models."try:_llama_logit_bias(body.get("logit_bias"))_mlx_completion_grammar(body)@@ -39373,6 +39385,8 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusasubject = current_subject,)+ n_probs = _llama_value(body, "n_probs") or 0+def generate(prompt_ids, choice = 0):return backend.generate_completion(prompt_ids,@@ -39380,6 +39394,7 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusastats_holder = stats_holder,max_new_tokens = budget,**{**sampling, "seed": _choice_seed(sampling["seed"], choice)},+ **({"n_probs": n_probs} if n_probs else {}),)def choice_end(prompt_ids, stats):@@ -39563,8 +39578,15 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusacached_tokens = (cached_tokens or 0) + cachedcompletion_tokens += usage["completion_tokens"]prefix = resolved[prompt_index]["text"] if echo else ""+ token_logprobs = (stats or {}).get("token_logprobs") if n_probs else Nonechoices.append(- {"text": prefix + text, "index": index, "logprobs": None, "finish_reason": reason}+ {+ "text": prefix + text,+ "index": index,+ # llama-server's shape, null when nothing was generated.+ "logprobs": {"content": token_logprobs} if token_logprobs else None,+ "finish_reason": reason,+ })replies.append(text)except asyncio.CancelledError:diff --git a/studio/backend/routes/llama_compat.py b/studio/backend/routes/llama_compat.pyindex 0cc0fad4fcd..387d3d0c9f3 100644--- a/studio/backend/routes/llama_compat.py+++ b/studio/backend/routes/llama_compat.py@@ -19,6 +19,7 @@ from __future__ import annotationsimport asyncioimport functools+import threadingfrom typing import Any, Optionalimport httpx@@ -234,7 +235,11 @@ def _upstream_client() -> httpx.AsyncClient:def _engine_error(status_code: int, message: str) -> JSONResponse:- error_type = "unavailable_error" if status_code == 503 else "not_found_error"+ # llama-server: not_found_error is 404 only; a rejected body is invalid_request_error.+ if status_code == 400:+ error_type = "invalid_request_error"+ else:+ error_type = "unavailable_error" if status_code == 503 else "not_found_error"return JSONResponse(status_code = status_code,content = {"error": {"code": status_code, "message": message, "type": error_type}},@@ -249,6 +254,9 @@ async def _forward_to_llama_server(request: Request, path: str) -> Response:# Unload flips is_loaded before clearing _port; "127.0.0.1:None" raises InvalidURL, not HTTPError.port = getattr(llama_backend, "_port", None)if not getattr(llama_backend, "is_loaded", False) or not port:+ mlx_backend = _inference()._active_mlx_backend() if path == "tokenize" else None+ if mlx_backend is not None:+ return await _mlx_tokenize(request, mlx_backend)return _engine_error(503, "No llama.cpp model is loaded")url = f"http://127.0.0.1:{port}/{path}"headers = dict(llama_backend._auth_headers or {})@@ -275,6 +283,53 @@ async def _forward_to_llama_server(request: Request, path: str) -> Response:)+async def _mlx_tokenize(request: Request, backend) -> Response:+ """llama-server's ``POST /tokenize`` for a resident MLX model, which lm-eval's gguf backend+ calls before scoring with ``/v1/completions`` logprobs: specials parsed, BOS only with+ ``add_special``."""+ try:+ body = await request.json()+ except ValueError:+ body = None+ content = body.get("content") if isinstance(body, dict) else None+ # llama-server's tokenize_mixed accepts mixed arrays of strings and ids.+ if not isinstance(content, (str, list)):+ return _engine_error(400, "'content' must be a string or an array")+ if body.get("with_pieces"):+ return _engine_error(400, "'with_pieces' is not supported for MLX models")+ # Specials are always parsed, so refuse rather than ignore: GGUF would tokenize differently.+ if body.get("parse_special") is False:+ return _engine_error(400, "'parse_special': false is not supported for MLX models")++ add_special = body.get("add_special") is True+ pieces = [content] if isinstance(content, str) else list(content)++ # Resolution polls this event; without it an abandoned request pins a worker thread.+ cancel = threading.Event()++ def tokenize():+ # Only a leading string gets specials, so a leading id (dropped after) tokenizes without.+ prompt = pieces if add_special else [0, *pieces]+ ids = backend.resolve_completion_prompt(prompt, cancel_event = cancel)["ids"]+ return ids if add_special else ids[1:]++ watcher = asyncio.create_task(_inference()._await_disconnect_then_cancel(request, cancel))+ try:+ tokens = await asyncio.to_thread(tokenize)+ except ValueError as exc:+ if any(piece != "" for piece in pieces):+ return _engine_error(400, str(exc))+ tokens = [] # llama-server answers empty content with an empty list+ except RuntimeError:+ # Unloaded since selection: an unavailable engine, not a server fault.+ return _engine_error(503, "No llama.cpp model is loaded")+ except InterruptedError:+ return _engine_error(499, "Request cancelled.")+ finally:+ await _inference()._stop_local_disconnect_cancel_watcher(watcher)+ return JSONResponse({"tokens": tokens})++def _make_passthrough(path: str):async def passthrough(request: Request, current_subject: str = Depends(get_current_subject)):return await _forward_to_llama_server(request, path)diff --git a/studio/backend/tests/test_llama_compat_routes.py b/studio/backend/tests/test_llama_compat_routes.pyindex 11e251e08c1..339f829cc75 100644--- a/studio/backend/tests/test_llama_compat_routes.py+++ b/studio/backend/tests/test_llama_compat_routes.py@@ -16,6 +16,7 @@ on the body. The module is loaded standalone with a double for routes.inference.from __future__ import annotations+import asyncioimport importlib.utilimport sysimport time@@ -82,6 +83,7 @@ def _inference_double(backend,catalog,orchestrator = None,+ mlx = None,):"""Stand-in for routes.inference. Deliberately not a sys.modules entry: stubbing itglobally broke two unrelated route tests when the suite ran in one process."""@@ -90,12 +92,27 @@ def _inference_double(inference._llama_public_model_id = lambda b, fallback = None: b._openai_advertised_idinference._peek_inference_backend = lambda: orchestratorinference._orchestrator_public_model_id = lambda b: b.active_model_name+ inference._active_mlx_backend = lambda: mlxasync def _objects(quant_pins = False):return cataloginference._openai_catalog_objects = _objectsinference.model_slots = types.SimpleNamespace(visible_loading = lambda: None)++ # Inert: is_disconnected() never returns under TestClient; one test supplies its own watcher.+ async def _await_disconnect_then_cancel(request, cancel_event):+ try:+ await asyncio.Event().wait()+ except asyncio.CancelledError:+ return++ async def _stop_watcher(watcher, timeout_s = 1.0):+ watcher.cancel()+ await asyncio.wait({watcher}, timeout = timeout_s)++ inference._await_disconnect_then_cancel = _await_disconnect_then_cancel+ inference._stop_local_disconnect_cancel_watcher = _stop_watcherreturn inference@@ -106,6 +123,7 @@ def _load(backend = None,catalog = None,orchestrator = None,+ mlx = None,):"""The real routes/llama_compat.py, with routes.inference replaced by a double."""global _MOD@@ -120,6 +138,7 @@ def _load(backend if backend is not None else _Backend(),CATALOG if catalog is None else catalog,orchestrator,+ mlx,)_MOD._inference = lambda: doublereturn _MOD@@ -735,6 +754,45 @@ def test_nothing_loaded_is_503_in_llama_servers_error_shape(monkeypatch):assert upstream.calls == []+class _MLXTokenizer:+ """resolve_completion_prompt's rules: only a leading string gets BOS (1), empty raises."""++ def resolve_completion_prompt(+ self,+ prompt,+ cancel_event = None,+ ):+ pieces = [prompt] if isinstance(prompt, str) else prompt+ ids = []+ for index, piece in enumerate(pieces):+ if isinstance(piece, int):+ ids.append(piece)+ elif isinstance(piece, str):+ if piece:+ ids += ([1] if index == 0 else []) + [40 + i for i in range(len(piece))]+ else:+ raise ValueError("Invalid prompt.")+ if not ids:+ raise ValueError("The prompt is empty.")+ return {"ids": ids, "text": ""}+++def test_tokenize_answers_from_a_resident_mlx_model(monkeypatch):+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())+ upstream = _Upstream()+ _with_upstream(monkeypatch, mod, upstream)+ with _client(mod) as c:+ special = c.post("/tokenize", json = {"content": "ab", "add_special": True})+ plain = c.post("/tokenize", json = {"content": "ab"})+ empty = c.post("/tokenize", json = {"content": ""})+ bad = c.post("/tokenize", json = {"content": 3})+ pieces = c.post("/tokenize", json = {"content": "ab", "with_pieces": True})+ assert special.json() == {"tokens": [1, 40, 41]} and plain.json() == {"tokens": [40, 41]}+ assert empty.json() == {"tokens": []}+ assert bad.status_code == 400 and pieces.status_code == 400+ assert upstream.calls == []++def test_an_unload_racing_the_request_is_503_not_500(monkeypatch):"""unload_model flips is_loaded before it clears _port."""backend = _Backend()@@ -1592,3 +1650,87 @@ def test_management_unload_routes_a_kept_slot_quant_past_the_primary(monkeypatchwith _client(mod) as c:assert c.post("/models/unload", json = {"model": pin["id"]}).status_code == 200assert calls == [("unload", pin["id"])]+++def test_tokenize_takes_llama_server_mixed_content_on_mlx(monkeypatch):+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())+ upstream = _Upstream()+ _with_upstream(monkeypatch, mod, upstream)+ with _client(mod) as c:+ mixed = c.post("/tokenize", json = {"content": ["ab", 7, "c"]})+ leading_special = c.post("/tokenize", json = {"content": ["ab", 7], "add_special": True})+ bad = c.post("/tokenize", json = {"content": [{}]})+ assert mixed.json() == {"tokens": [40, 41, 7, 40]}+ assert leading_special.json() == {"tokens": [1, 40, 41, 7]}+ assert bad.status_code == 400+ assert upstream.calls == []+++def test_tokenize_refuses_parse_special_false_on_mlx(monkeypatch):+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())+ upstream = _Upstream()+ _with_upstream(monkeypatch, mod, upstream)+ with _client(mod) as c:+ refused = c.post("/tokenize", json = {"content": "ab", "parse_special": False})+ allowed = c.post("/tokenize", json = {"content": "ab", "parse_special": True})+ assert refused.status_code == 400 and "parse_special" in refused.text+ assert allowed.json() == {"tokens": [40, 41]}+ assert upstream.calls == []+++def test_tokenize_reports_an_mlx_unload_race_as_an_unavailable_engine(monkeypatch):+ class _Racing:+ def resolve_completion_prompt(+ self,+ prompt,+ cancel_event = None,+ ):+ raise RuntimeError("No active model")++ mod = _load(backend = _Backend(loaded = False), mlx = _Racing())+ upstream = _Upstream()+ _with_upstream(monkeypatch, mod, upstream)+ with _client(mod) as c:+ raced = c.post("/tokenize", json = {"content": "ab"})+ assert raced.status_code == 503+ assert upstream.calls == []+++def test_tokenize_rejections_are_invalid_request_errors(monkeypatch):+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())+ _with_upstream(monkeypatch, mod, _Upstream())+ with _client(mod) as c:+ bad = c.post("/tokenize", json = {"content": 3})+ unavailable = c.post("/tokenize", json = {"content": "ab", "parse_special": False})+ assert bad.status_code == 400 and bad.json()["error"]["type"] == "invalid_request_error"+ assert unavailable.json()["error"]["type"] == "invalid_request_error"+++def test_tokenize_stops_waiting_for_the_generation_lock_when_the_client_goes(monkeypatch):+ import threading++ started = threading.Event()++ class _Blocked:+ def resolve_completion_prompt(+ self,+ prompt,+ cancel_event = None,+ ):+ started.set()+ # Event.wait, not a spin: a busy loop here starves the loop thread of the GIL.+ assert cancel_event.wait(10), "the disconnect never reached the waiting resolve"+ raise InterruptedError("Request cancelled.")++ mod = _load(backend = _Backend(loaded = False), mlx = _Blocked())+ _with_upstream(monkeypatch, mod, _Upstream())++ async def disconnected(request, cancel_event):+ while not started.is_set():+ await asyncio.sleep(0.01)+ cancel_event.set()++ monkeypatch.setattr(mod._inference(), "_await_disconnect_then_cancel", disconnected)+ with _client(mod) as c:+ gone = c.post("/tokenize", json = {"content": "ab"})+ assert gone.status_code == 499diff --git a/studio/backend/tests/test_mlx_completions_apple.py b/studio/backend/tests/test_mlx_completions_apple.pyindex deb9359bce1..a863c0b7ae6 100644--- a/studio/backend/tests/test_mlx_completions_apple.py+++ b/studio/backend/tests/test_mlx_completions_apple.py@@ -114,3 +114,56 @@ def test_raw_vision_prompt_matches_the_decoded_sampled_ids():tokens, skip_special_tokens = True, clean_up_tokenization_spaces = False)assert direct.strip() and _greedy(backend, ids) == direct+++def _direct_logprobs(backend, ids):+ """mlx_lm's log_softmax for the first generated position with no processors, under the+ scopes Studio decodes in: the logits llama-server reads ``n_probs`` from."""+ from mlx_lm import stream_generate++ with backend._int8_prefill_scope(), backend._generation_scopes():+ (first,) = list(+ stream_generate(backend._model, backend._tokenizer, prompt = ids, max_tokens = 1)+ )+ return first.logprobs+++# mlx_lm reduces its log_softmax in the model's dtype, so its reference carries a few ulps of+# that dtype (bf16 for a 4-bit checkpoint); the recorder reads float32 like llama-server.+_REF_EPS = {"bfloat16": 2.0**-8, "float16": 2.0**-11, "float32": 2.0**-24}+++def _ref_approx(direct, value):+ eps = _REF_EPS.get(str(direct.dtype).rsplit(".", 1)[-1], 2.0**-8)+ return pytest.approx(value, abs = max(1e-3, 4 * eps * abs(value)))+++def _scored(backend, ids, **kw):+ for _ in backend.generate_completion(+ ids, temperature = 0.0, top_p = 1.0, top_k = 0, min_p = 0.0, max_new_tokens = 1, **kw+ ):+ pass+ return backend.last_generation_stats["token_logprobs"]+++def test_raw_prompt_logprobs_match_a_direct_forward_pass(backend):+ ids = backend.resolve_completion_prompt(_PROMPT)["ids"]+ direct = _direct_logprobs(backend, ids)+ greedy = int(direct.argmax().item())+ (entry,) = _scored(backend, ids, n_probs = 3)+ assert entry["id"] == greedy+ assert entry["logprob"] == _ref_approx(direct, direct[greedy].item())+ assert entry["top_logprobs"][0]["id"] == greedy and len(entry["top_logprobs"]) == 3+++def test_raw_prompt_logprobs_report_a_forced_token_unbiased(backend):+ import mlx.core as mx++ ids = backend.resolve_completion_prompt(_PROMPT)["ids"]+ direct = _direct_logprobs(backend, ids)+ forced = int(mx.argsort(-direct)[100].item())+ (entry,) = _scored(backend, ids, n_probs = 2, logit_bias = {forced: 100.0})+ assert entry["id"] == forced+ assert entry["logprob"] == _ref_approx(direct, direct[forced].item())+ assert entry["logprob"] < -1.0+ assert entry["top_logprobs"][0]["id"] == int(direct.argmax().item())diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 1020300d9d5..fd41f91092c 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -2855,6 +2855,172 @@ def test_mlx_vlm_completion_prompt_runs_untemplated_from_its_ids(monkeypatch):assert list(backend.generate_completion([5, 6], max_new_tokens = 3))[-1] == " Paris!"+def _numpy_mx(monkeypatch):+ np = pytest.importorskip("numpy")++ class _At:+ def __init__(self, array):+ self.array = array++ def __getitem__(self, idx):+ def add(values):+ out = np.array(self.array)+ np.add.at(out, np.asarray(idx), np.asarray(values))+ return out++ return SimpleNamespace(add = add)++ class _Arr(np.ndarray):+ @property+ def at(self):+ return _At(self)++ def _logsumexp(+ a,+ axis = -1,+ keepdims = False,+ ):+ peak = a.max(axis = axis, keepdims = True)+ out = np.log(np.exp(a - peak).sum(axis = axis, keepdims = True)) + peak+ return out if keepdims else out.squeeze(axis)++ core = sys.modules["mlx.core"]+ monkeypatch.setattr(core, "float32", np.float32, raising = False)+ monkeypatch.setattr(core, "int32", np.int32, raising = False)+ monkeypatch.setattr(+ core, "array", lambda v, dtype = None: np.array(v, dtype = dtype), raising = False+ )+ monkeypatch.setattr(+ core,+ "zeros",+ lambda shape, dtype = None: np.zeros(shape, dtype = dtype).view(_Arr),+ raising = False,+ )+ monkeypatch.setattr(core, "logsumexp", _logsumexp, raising = False)+ monkeypatch.setattr(core, "argpartition", lambda a, kth: np.argpartition(a, kth), raising = False)+ monkeypatch.setattr(core, "argsort", np.argsort, raising = False)+ return np+++# Probabilities the fake model gives every step: token 0 is the argmax, token 3 unlikely.+_STEP_PROBS = [0.6, 0.3, 0.05, 0.04, 0.01]+++def _processed_steps(np, processors, prompt, steps):+ """(token, response) per step as mlx_lm / mlx_vlm run processors: once per step, never+ during prefill, step n + 1 computed before token n is yielded."""+ raw = np.log(np.array([_STEP_PROBS], dtype = np.float32))+ tokens = np.array(prompt)++ def step(tokens):+ logits = raw.copy()+ for processor in processors or ():+ logits = processor(tokens, logits)+ return int(np.argmax(logits))++ y = step(tokens)+ for n in range(steps):+ tokens = np.append(tokens, y)+ upcoming = step(tokens) if n + 1 < steps else None+ yield y, n + 1+ y = upcoming+++def test_mlx_completion_logprobs_read_the_logits_before_logit_bias(monkeypatch):+ import math++ _install_fake_prompt_cache_api(monkeypatch)+ captured = []+ backend = _install_fake_text_stack(monkeypatch, {"generated": []}, captured)+ np = _numpy_mx(monkeypatch)++ def _stream_generate(_model, _tokenizer, **kw):+ captured.append(kw)+ steps = _processed_steps(np, kw.get("logits_processors"), kw["prompt"], kw["max_tokens"])+ for token, n in steps:+ yield SimpleNamespace(+ token = token,+ text = "",+ prompt_tokens = len(kw["prompt"]),+ prompt_tps = 1.0,+ generation_tokens = n,+ generation_tps = 1.0,+ finish_reason = None,+ )++ monkeypatch.setattr(sys.modules["mlx_lm"], "stream_generate", _stream_generate)+ sampling = dict(max_new_tokens = 2, temperature = 0.0, logit_bias = {3: 100.0})+ out = list(backend.generate_completion([1, 2], n_probs = 2, **sampling))+ entries = backend.last_generation_stats["token_logprobs"]+ assert out[-1] == "33" and [e["id"] for e in entries] == [3, 3]+ first = entries[0]+ assert first["logprob"] == pytest.approx(math.log(0.04), abs = 1e-5)+ assert first["token"] == "3" and first["bytes"] == [51]+ assert [(t["id"], t["token"], t["bytes"]) for t in first["top_logprobs"]] == [+ (0, "0", [48]),+ (1, "1", [49]),+ ]+ assert first["top_logprobs"][0]["logprob"] == pytest.approx(math.log(0.6), abs = 1e-5)+ list(backend.generate_completion([1, 2], max_new_tokens = 2))+ assert "logits_processors" not in captured[-1]+ assert "token_logprobs" not in backend.last_generation_stats+++def test_mlx_logprob_tokens_keep_a_sentencepiece_word_space(monkeypatch):+ from core.inference import mlx_inference++ _install_fake_mlx(monkeypatch)+ pieces = {7: "\u2581the", 8: "Hand"}+ tokenizer = SimpleNamespace(+ decode = lambda ids, **_kw: "".join(pieces[i] for i in ids)+ .replace("\u2581", " ")+ .removeprefix(" ")+ )+ assert mlx_inference._token_bytes_text(tokenizer, 7) == (" the", [32, 116, 104, 101])+ assert mlx_inference._token_bytes_text(tokenizer, 8) == ("Hand", [72, 97, 110, 100])+++def test_mlx_vlm_completion_logprobs_skip_the_draft_and_repeated_responses(monkeypatch):+ from core.inference import mlx_inference++ _install_fake_mlx(monkeypatch)+ np = _numpy_mx(monkeypatch)+ seen = {}++ def _vlm_stream(*_a, **kw):+ seen.update(kw)+ steps = list(_processed_steps(np, kw["logits_processors"], kw["input_ids"][0], 2))+ for token, n in steps:+ yield SimpleNamespace(text = "", token = token, generation_tokens = n, prompt_tokens = 2)+ # mlx-vlm repeats the last token in its closing response.+ yield SimpleNamespace(text = "", token = token, generation_tokens = n, prompt_tokens = 2)++ mlx_vlm = types.ModuleType("mlx_vlm")+ mlx_vlm.stream_generate = _vlm_stream+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)+ monkeypatch.setattr(+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a: contextlib.nullcontext()+ )+ backend = mlx_inference.MLXInferenceBackend()+ backend._model, backend._is_vlm = SimpleNamespace(), True+ backend._processor = SimpleNamespace()+ backend._tokenizer = SimpleNamespace(+ decode = lambda ids, skip_special_tokens, clean_up_tokenization_spaces: "".join(+ str(i) for i in ids+ )+ )+ monkeypatch.setattr(backend, "_vlm_prompt_cache_session", lambda *_a, **_k: None)+ backend._speculative_draft = SimpleNamespace(drafter = None, draft_kind = "mtp")++ out = list(backend.generate_completion([5, 6], max_new_tokens = 2, n_probs = 1))+ entries = backend.last_generation_stats["token_logprobs"]+ assert out[-1] == "00" and [e["id"] for e in entries] == [0, 0]+ assert [t["id"] for t in entries[0]["top_logprobs"]] == [0]+ assert "draft_model" not in seen+ assert mlx_inference._request_batch_gap({"n_probs": 2})+ assert mlx_inference._request_batch_gap({"n_probs": 0}) is None++def test_a_raw_vision_prompt_decodes_on_its_own_even_when_zoo_could_batch_it(monkeypatch):from core.inference import mlx_inferencediff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex badf3936e30..b74589abec1 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -11943,11 +11943,11 @@ def test_mlx_completion_refusals_are_client_errors_as_in_chat(monkeypatch, stop,_LLAMA_SERVED = ("stream stream_options n_predict max_tokens max_completion_tokens top_k top_p min_p ""temperature repeat_last_n repeat_penalty frequency_penalty presence_penalty seed "- "logit_bias stop echo n n_cmpl grammar json_schema response_format"+ "logit_bias stop echo n n_cmpl grammar json_schema response_format logprobs n_probs")_LLAMA_IGNORED = ("cache_prompt sse_ping_interval n_keep n_discard n_cache_reuse backend_sampling "- "post_sampling_probs grammar_triggers chat_format "+ "grammar_triggers chat_format ""reasoning_format generation_prompt parse_tool_calls chat_parser continue_final_message ""reasoning_control reasoning_budget_start_tag reasoning_budget_end_tags ""reasoning_budget_end_tag reasoning_budget_message"@@ -11955,7 +11955,7 @@ _LLAMA_IGNORED = (# The rest change the text or the shape with no MLX equivalent: a value MLX refuses, and the one# that turns the field off._LLAMA_UNSERVED = json.loads(- '{"logprobs": [2, 0], "n_probs": [1, 0], "best_of": [2, 1], "suffix": ["x", ""],'+ '{"best_of": [2, 1], "suffix": ["x", ""],'' "timings_per_token": [true, false], "return_tokens": [true, false],'' "return_progress": [true, false], "verbose": [true, false], "response_fields": [["id"], []], "lora":'' [[{"id": 0, "scale": 1}], []], "n_indent": [2, 0], "t_max_predict_ms": [100, -1], "ignore_eos":'@@ -11969,7 +11969,8 @@ _LLAMA_DEPENDENT = json.loads('{"xtc_threshold": "xtc_probability", "dynatemp_exponent": "dynatemp_range", "mirostat_tau":'' "mirostat", "mirostat_eta": "mirostat", "dry_base": "dry_multiplier", "dry_allowed_length":'' "dry_multiplier", "dry_penalty_last_n": "dry_multiplier", "dry_sequence_breakers":'- ' "dry_multiplier", "adaptive_decay": "adaptive_target", "grammar_lazy": "grammar"}'+ ' "dry_multiplier", "adaptive_decay": "adaptive_target", "grammar_lazy": "grammar",'+ ' "post_sampling_probs": "n_probs"}')@@ -12016,6 +12017,11 @@ def test_mlx_completion_refuses_only_what_mlx_cannot_reproduce(monkeypatch):assert refusal({"prompt": "p", "xtc_probability": 0.5, "xtc_threshold": 0.6}) is Noneassert refusal({"prompt": "p", "dry_multiplier": 0.8, "dry_penalty_last_n": 0}) is Noneassert refusal({"prompt": "p", "n_cmpl": 1, "n": 2, "n_probs": 0, "logprobs": 5}) is None+ # llama-server reads post_sampling_probs only with n_probs, and then renames the fields.+ assert refusal({"prompt": "p", "post_sampling_probs": True}) is None+ assert "'post_sampling_probs'" in refusal(+ {"prompt": "p", "logprobs": 2, "post_sampling_probs": True}+ )assert refusal({"prompt": "p", "n": 2, "best_of": 2}) is Noneassert refusal({"prompt": "p", "grammar": None, "json_schema": {}}) is Nonefor prompt in ([1, "a"], ["a", 1], [[1, 2]], {"prompt_string": [1, "a"]}):@@ -12035,5 +12041,42 @@ def test_mlx_completion_refuses_only_what_mlx_cannot_reproduce(monkeypatch):{"prompt": "p", "repeat_last_n": -1},{"prompt": "p", "logit_bias": {"hi": 1}},{"prompt": "p", "grammar": "root ::= nowhere"},+ {"prompt": "p", "logprobs": 2, "stream": True},+ {"prompt": "p", "logprobs": 2, "echo": True},+ {"prompt": "p", "logprobs": 2, "n": 2},+ {"prompt": ["a", "b"], "n_probs": 1},+ {"prompt": "p", "n_probs": -1},+ {"prompt": "p", "logprobs": True},):assert refusal(bad), bad+++def test_mlx_completion_reports_logprobs_in_llama_server_shape(monkeypatch):+ entry = {+ "id": 7,+ "token": " a",+ "bytes": [32, 97],+ "logprob": -2.5,+ "top_logprobs": [+ {"id": 3, "token": " b", "bytes": [32, 98], "logprob": -0.1},+ {"id": 7, "token": " a", "bytes": [32, 97], "logprob": -2.5},+ ],+ }+ real = _MLXCompletionBackend.generate_completion++ def generate_completion(self, prompt_ids, cancel_event, stats_holder, **kw):+ yield from real(self, prompt_ids, cancel_event, stats_holder, **kw)+ if kw.get("n_probs"):+ stats_holder["stats"]["token_logprobs"] = [entry]++ monkeypatch.setattr(_MLXCompletionBackend, "generate_completion", generate_completion)+ body = {"prompt": [1, 2], "max_tokens": 1, "temperature": 0, "logprobs": 2}+ backend, out = _run_mlx_completion(monkeypatch, {**body, "logit_bias": [[7, 100]]})+ assert backend.call[1]["n_probs"] == 2 and backend.call[1]["logit_bias"] == {7: 100.0}+ assert out["choices"][0]["logprobs"] == {"content": [entry]}+ # n_probs is llama-server's own name and wins over the alias.+ backend, _ = _run_mlx_completion(monkeypatch, {**body, "n_probs": 4})+ assert backend.call[1]["n_probs"] == 4+ for off in ({}, {"logprobs": 0}, {"logprobs": False}):+ plain, out = _run_mlx_completion(monkeypatch, {"prompt": [1, 2], **off})+ assert "n_probs" not in plain.call[1] and out["choices"][0]["logprobs"] is Nonediff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.pyindex d479603cd31..cadc53016c3 100644--- a/studio/backend/tests/test_presence_penalty.py+++ b/studio/backend/tests/test_presence_penalty.py@@ -495,10 +495,14 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serve_ for _ in ()).throw(_Built())with pytest.raises(_Built):- list(o.generate_completion([5, 6], max_new_tokens = 3, penalty_window = 0, grammar = "G"))+ list(+ o.generate_completion(+ [5, 6], max_new_tokens = 3, penalty_window = 0, grammar = "G", n_probs = 2+ )+ )cmd = built[0]assert (cmd["messages"], cmd["prompt_ids"], cmd["penalty_window"]) == ([], [5, 6], 0)- assert cmd["grammar"] == "G"+ assert cmd["grammar"] == "G" and cmd["n_probs"] == 2with pytest.raises(_Built):list(o.generate_completion_batch([{"prompt_ids": [5]}], penalty_window = 1))assert (built[1]["messages"], built[1]["rows"], built[1]["penalty_window"]) == (@@ -506,6 +510,7 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serve[{"prompt_ids": [5]}],1,)+ assert "n_probs" not in built[1]class _MLX:last_generation_stats = None@@ -521,7 +526,7 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_servemlx, queue = _MLX(), _Queue()_handle_generate(mlx, cmd, queue, threading.Event())assert mlx.received[0] == [5, 6] and mlx.received[1]["penalty_window"] == 0- assert mlx.received[1]["grammar"] == "G"+ assert mlx.received[1]["grammar"] == "G" and mlx.received[1]["n_probs"] == 2assert [item["type"] for item in queue] == ["token", "gen_done"]queue.clear()_handle_generate(chat, cmd, queue, threading.Event())