Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: logprobs on /v1/completions and /tokenize for MLX models (#14229)

Daniel Han6a19cb20b98c
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 13415cf03fb..bda64ad75f1 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -3289,6 +3289,71 @@ def _raw_prompt_constraint(grammar, tokenizer):
return GrammarConstraint(grammar, tokenizer, strips_special_tokens = True)
+class LogprobsUnsupported(ValueError):
+ """``n_probs`` asked of a model that does not sample one token per step."""
+
+ public = True
+ openai_param = "logprobs"
+
+
+def _make_mlx_raw_logprob_recorder():
+ """A first logits processor queueing each step's log-softmax of the model's own logits, and
+ the queue. llama-server reports ``n_probs`` from the logits before any sampler, so a token
+ forced by ``logit_bias`` keeps its real logprob; mlx_lm and mlx_vlm compute theirs after the
+ processors. Neither runtime calls processors during prefill, so entry i is generated token i."""
+ from collections import deque
+
+ import mlx.core as mx
+
+ recorded = deque()
+
+ def _processor(tokens, logits):
+ raw = logits.astype(mx.float32)
+ recorded.append(raw - mx.logsumexp(raw, axis = -1, keepdims = True))
+ return logits
+
+ return _processor, recorded
+
+
+def _token_bytes_text(tokenizer, token_id):
+ kw = dict(skip_special_tokens = False, clean_up_tokenization_spaces = False)
+ # SentencePiece decoders strip a sequence's leading space, so "▁the" alone decodes to
+ # "the"; after a copy of itself it keeps the " the" llama-server reports.
+ alone = tokenizer.decode([token_id], **kw)
+ twice = tokenizer.decode([token_id, token_id], **kw)
+ text = twice[len(alone) :] if twice.startswith(alone) else alone
+ return text, list(text.encode("utf-8"))
+
+
+def _mlx_token_logprob(raw, token_id, n_probs, tokenizer) -> dict:
+ """One llama-server ``logprobs.content`` entry: the sampled token's logprob and the
+ ``n_probs`` most likely tokens, all read from the unprocessed logits."""
+ import mlx.core as mx
+
+ raw = raw.reshape(-1)
+ n = min(int(n_probs), raw.shape[0])
+ top = mx.argpartition(-raw, kth = n - 1)[:n]
+ values = raw[top]
+ order = mx.argsort(-values)
+ # llama-server writes the lowest float for -inf, which JSON cannot carry.
+ floor = -3.4028234663852886e38
+ entry_ids, entry_values = top[order].tolist(), values[order].tolist()
+ text, data = _token_bytes_text(tokenizer, int(token_id))
+ top_logprobs = []
+ for tid, value in zip(entry_ids, entry_values):
+ tid_text, tid_bytes = _token_bytes_text(tokenizer, int(tid))
+ top_logprobs.append(
+ {"id": int(tid), "token": tid_text, "bytes": tid_bytes, "logprob": max(value, floor)}
+ )
+ return {
+ "id": int(token_id),
+ "token": text,
+ "bytes": data,
+ "logprob": max(raw[int(token_id)].item(), floor),
+ "top_logprobs": top_logprobs,
+ }
+
+
def _with_prompt_history(processor, prompt, window):
"""*processor* also scoring the prompt tokens the runtime withholds: mlx_lm and mlx_vlm hand
processors only what is left of the prompt after prefill and prompt-cache reuse."""
@@ -3577,6 +3642,8 @@ def _request_batch_gap(request):
return "a reply runs the tool protocol"
if request.get("response_format") is not None or request.get("grammar") is not None:
return "a reply is constrained to a response format"
+ if request.get("n_probs"):
+ return "a reply reports token probabilities"
return None
@@ -5613,9 +5680,11 @@ class MLXInferenceBackend:
stop = None,
penalty_window = None,
grammar = None,
+ n_probs = 0,
) -> Generator[str, None, None]:
"""Cumulative text sampled after ``prompt_ids`` with no chat template.
- ``penalty_window`` is llama-server's ``repeat_last_n``."""
+ ``penalty_window`` is llama-server's ``repeat_last_n``; ``n_probs`` > 0 adds each
+ token's ``logprobs.content`` entry to the stats as ``token_logprobs``."""
if self._model is None:
raise RuntimeError("No model loaded")
self.last_generation_stats = None
@@ -5629,6 +5698,8 @@ class MLXInferenceBackend:
penalty_window = penalty_window,
grammar = grammar,
)
+ if n_probs:
+ sampling["n_probs"] = int(n_probs)
args = (temperature, top_p, top_k, min_p, max_new_tokens, repetition_penalty, cancel_event)
if self._is_vlm:
yield from self._generate_vlm(None, None, *args, **sampling)
@@ -5668,6 +5739,7 @@ class MLXInferenceBackend:
prompt_ids = None,
penalty_window = None,
grammar = None,
+ n_probs = 0,
):
from mlx_lm import stream_generate
from mlx_lm.sample_utils import make_sampler
@@ -5744,6 +5816,11 @@ class MLXInferenceBackend:
penalty_window = penalty_window,
penalty_prompt = prompt_ids,
)
+ recorded = token_logprobs = None
+ if n_probs:
+ recorder, recorded = _make_mlx_raw_logprob_recorder()
+ logits_processors = [recorder, *(logits_processors or ())]
+ token_logprobs = []
native_token_decoder = (
NativeToolTokenDecoder(
@@ -5840,6 +5917,12 @@ class MLXInferenceBackend:
):
final_response = response
token_ids.append(response.token)
+ if recorded:
+ token_logprobs.append(
+ _mlx_token_logprob(
+ recorded.popleft(), response.token, n_probs, self._tokenizer
+ )
+ )
if preserve_native_channels:
sampled += streamed_text.feed(response)
if sequences:
@@ -5929,6 +6012,8 @@ class MLXInferenceBackend:
max_new_tokens,
),
)
+ if token_logprobs is not None:
+ self.last_generation_stats["token_logprobs"] = token_logprobs
# The turn's settled text: delivered once for the plain path, as the tail for the native-channel one. Every
# snapshot was matched as it arrived, so a turn no sequence ended owes all of its text, held-back partial
# included.
@@ -6391,6 +6476,7 @@ class MLXInferenceBackend:
prompt_ids = None,
penalty_window = None,
grammar = None,
+ n_probs = 0,
):
from mlx_vlm import stream_generate as vlm_stream
@@ -6505,6 +6591,15 @@ class MLXInferenceBackend:
"constrain the next token. Load an autoregressive model to use "
"guided decoding."
)
+ recorded = token_logprobs = None
+ if n_probs:
+ if _vlm_generation_is_diffusion(self._model):
+ raise LogprobsUnsupported(
+ "logprobs is not supported on this model: it generates by diffusion rather "
+ "than one token at a time."
+ )
+ recorder, recorded = _make_mlx_raw_logprob_recorder()
+ token_logprobs = []
draft = self._speculative_draft
if (
presence_penalty
@@ -6513,6 +6608,7 @@ class MLXInferenceBackend:
or constraint is not None
or (_rep_active and draft is not None)
or penalty_window is not None
+ or recorded is not None
):
# These need custom processors: pass the full list (repetition + the rest) instead of the
# repetition_penalty shortcut so all apply.
@@ -6525,6 +6621,12 @@ class MLXInferenceBackend:
penalty_window = penalty_window,
penalty_prompt = prompt_ids,
)
+ if recorded is not None:
+ # Not history_only, so no draft runs: its verify pass would record rejected tokens.
+ vlm_kwargs["logits_processors"] = [
+ recorder,
+ *(vlm_kwargs["logits_processors"] or ()),
+ ]
elif _rep_active:
vlm_kwargs["repetition_penalty"] = float(repetition_penalty)
@@ -6678,6 +6780,15 @@ class MLXInferenceBackend:
):
# The streaming detokenizer trims the first leading space; decode the ids.
raw_ids.append(int(raw_id))
+ if recorded:
+ token_logprobs.append(
+ _mlx_token_logprob(
+ recorded.popleft(),
+ int(raw_id),
+ n_probs,
+ self._tokenizer,
+ )
+ )
decoded = self._tokenizer.decode(
raw_ids,
skip_special_tokens = True,
@@ -6769,6 +6880,8 @@ class MLXInferenceBackend:
if draft is None
else (draft.draft_n, draft.draft_n_accepted),
)
+ if token_logprobs is not None:
+ self.last_generation_stats["token_logprobs"] = token_logprobs
if document_only or prompt_ids is not None:
# The normalizer would rewrite marker text inside the document.
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index cf9600e9459..e42822157d3 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -1470,6 +1470,7 @@ class InferenceOrchestrator:
prompt_ids: Optional[list] = None,
penalty_window: Optional[int] = None,
grammar: Optional[str] = None,
+ n_probs: int = 0,
) -> dict:
"""Build the 'generate' command shared by the locked and dispatched paths."""
cmd = {
@@ -1524,6 +1525,8 @@ class InferenceOrchestrator:
cmd["penalty_window"] = penalty_window
if grammar is not None:
cmd["grammar"] = grammar
+ if n_probs:
+ cmd["n_probs"] = int(n_probs)
return cmd
def _consume_token_stream(
@@ -1775,6 +1778,7 @@ class InferenceOrchestrator:
prompt_ids: Optional[list] = None,
penalty_window: Optional[int] = None,
grammar: Optional[str] = None,
+ n_probs: int = 0,
) -> Generator[Any, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
@@ -1835,6 +1839,7 @@ class InferenceOrchestrator:
prompt_ids = prompt_ids,
penalty_window = penalty_window,
grammar = grammar,
+ n_probs = n_probs,
)
mailbox = _WorkerMailbox(self._proc)
@@ -3595,6 +3600,7 @@ class InferenceOrchestrator:
prompt_ids: Optional[list] = None,
penalty_window: Optional[int] = None,
grammar: Optional[str] = None,
+ n_probs: int = 0,
) -> Generator[Any, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
@@ -3602,7 +3608,7 @@ class InferenceOrchestrator:
managed = getattr(self, "_managed_engine", None)
if managed is not None:
kwargs = dict(locals())
- for key in ("self", "managed", "kwargs", "diffusion_frames"):
+ for key in ("self", "managed", "kwargs", "diffusion_frames", "n_probs"):
kwargs.pop(key, None)
from .engine_transport import EngineHTTPError
@@ -3661,6 +3667,7 @@ class InferenceOrchestrator:
prompt_ids = prompt_ids,
penalty_window = penalty_window,
grammar = grammar,
+ n_probs = n_probs,
)
return
@@ -3706,6 +3713,7 @@ class InferenceOrchestrator:
prompt_ids = prompt_ids,
penalty_window = penalty_window,
grammar = grammar,
+ n_probs = n_probs,
)
read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index a663a746b73..ad0212de98b 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1195,7 +1195,7 @@ def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
raise RuntimeError("The loaded backend does not read video.")
gen_kwargs["video"] = cmd["video_base64"]
# Only a raw completion carries these; its batch rows are planned from them.
- for raw in ("prompt_ids", "penalty_window", "grammar"):
+ for raw in ("prompt_ids", "penalty_window", "grammar", "n_probs"):
if raw in cmd:
gen_kwargs[raw] = cmd[raw]
@@ -1279,6 +1279,7 @@ _COMPLETION_OPTIONS = (
"stop",
"penalty_window",
"grammar",
+ "n_probs",
)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index d1a2674455f..455a55ada9c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -39056,13 +39056,14 @@ _LLAMA_DEFAULT_SAMPLERS = (
# (tools/server/server-schema.cpp) plus OpenAI's. A field llama-server reads only while another
# is on is off with it.
_MLX_COMPLETION_FIELDS_OFF = {
- "logprobs": lambda v, b: v == 0,
- "n_probs": lambda v, b: v == 0,
# best_of equal to the choice count asks for no hidden candidates; llama-server ignores it.
"best_of": lambda v, b: v in (1, _llama_value(b, "n_cmpl")),
"suffix": lambda v, b: v == "",
# A lazy grammar waits for a trigger; this one constrains from the first token.
"grammar_lazy": lambda v, b: v is False or _llama_grammar_off(b),
+ # Ours are pre-sampler; llama-server's post_sampling_probs flips that and renames fields,
+ # but only with n_probs, so refuse only together.
+ "post_sampling_probs": lambda v, b: v is False or not _llama_value(b, "n_probs"),
"timings_per_token": lambda v, b: v is False,
"return_tokens": lambda v, b: v is False,
"return_progress": lambda v, b: v is False,
@@ -39215,6 +39216,17 @@ def _mlx_completions_refusal(body: dict) -> Optional[str]:
)
if body.get("stream") is True and len(prompts) * (_llama_value(body, "n_cmpl") or 1) > 1:
return "Streaming more than one completion is not supported for MLX models."
+ n_probs = _llama_value(body, "n_probs")
+ if n_probs not in (None, False) and not (_is_token_id(n_probs) and n_probs >= 0):
+ return f"Field 'n_probs': Value must be a non-negative integer, but got {n_probs}"
+ if n_probs:
+ # Only the reported tokens: a prompt's own logprobs (echo) are not computed.
+ if body.get("stream") is True:
+ return "'logprobs' is not supported with stream for MLX models."
+ if body.get("echo") is True:
+ return "'logprobs' is not supported with echo for MLX models."
+ if len(prompts) * (_llama_value(body, "n_cmpl") or 1) > 1:
+ return "'logprobs' is supported for one completion per request on MLX models."
try:
_llama_logit_bias(body.get("logit_bias"))
_mlx_completion_grammar(body)
@@ -39373,6 +39385,8 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusa
subject = current_subject,
)
+ n_probs = _llama_value(body, "n_probs") or 0
+
def generate(prompt_ids, choice = 0):
return backend.generate_completion(
prompt_ids,
@@ -39380,6 +39394,7 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusa
stats_holder = stats_holder,
max_new_tokens = budget,
**{**sampling, "seed": _choice_seed(sampling["seed"], choice)},
+ **({"n_probs": n_probs} if n_probs else {}),
)
def choice_end(prompt_ids, stats):
@@ -39563,8 +39578,15 @@ async def _mlx_completions(request, current_subject, backend, body: dict, refusa
cached_tokens = (cached_tokens or 0) + cached
completion_tokens += usage["completion_tokens"]
prefix = resolved[prompt_index]["text"] if echo else ""
+ token_logprobs = (stats or {}).get("token_logprobs") if n_probs else None
choices.append(
- {"text": prefix + text, "index": index, "logprobs": None, "finish_reason": reason}
+ {
+ "text": prefix + text,
+ "index": index,
+ # llama-server's shape, null when nothing was generated.
+ "logprobs": {"content": token_logprobs} if token_logprobs else None,
+ "finish_reason": reason,
+ }
)
replies.append(text)
except asyncio.CancelledError:
diff --git a/studio/backend/routes/llama_compat.py b/studio/backend/routes/llama_compat.py
index 0cc0fad4fcd..387d3d0c9f3 100644
--- a/studio/backend/routes/llama_compat.py
+++ b/studio/backend/routes/llama_compat.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import asyncio
import functools
+import threading
from typing import Any, Optional
import httpx
@@ -234,7 +235,11 @@ def _upstream_client() -> httpx.AsyncClient:
def _engine_error(status_code: int, message: str) -> JSONResponse:
- error_type = "unavailable_error" if status_code == 503 else "not_found_error"
+ # llama-server: not_found_error is 404 only; a rejected body is invalid_request_error.
+ if status_code == 400:
+ error_type = "invalid_request_error"
+ else:
+ error_type = "unavailable_error" if status_code == 503 else "not_found_error"
return JSONResponse(
status_code = status_code,
content = {"error": {"code": status_code, "message": message, "type": error_type}},
@@ -249,6 +254,9 @@ async def _forward_to_llama_server(request: Request, path: str) -> Response:
# Unload flips is_loaded before clearing _port; "127.0.0.1:None" raises InvalidURL, not HTTPError.
port = getattr(llama_backend, "_port", None)
if not getattr(llama_backend, "is_loaded", False) or not port:
+ mlx_backend = _inference()._active_mlx_backend() if path == "tokenize" else None
+ if mlx_backend is not None:
+ return await _mlx_tokenize(request, mlx_backend)
return _engine_error(503, "No llama.cpp model is loaded")
url = f"http://127.0.0.1:{port}/{path}"
headers = dict(llama_backend._auth_headers or {})
@@ -275,6 +283,53 @@ async def _forward_to_llama_server(request: Request, path: str) -> Response:
)
+async def _mlx_tokenize(request: Request, backend) -> Response:
+ """llama-server's ``POST /tokenize`` for a resident MLX model, which lm-eval's gguf backend
+ calls before scoring with ``/v1/completions`` logprobs: specials parsed, BOS only with
+ ``add_special``."""
+ try:
+ body = await request.json()
+ except ValueError:
+ body = None
+ content = body.get("content") if isinstance(body, dict) else None
+ # llama-server's tokenize_mixed accepts mixed arrays of strings and ids.
+ if not isinstance(content, (str, list)):
+ return _engine_error(400, "'content' must be a string or an array")
+ if body.get("with_pieces"):
+ return _engine_error(400, "'with_pieces' is not supported for MLX models")
+ # Specials are always parsed, so refuse rather than ignore: GGUF would tokenize differently.
+ if body.get("parse_special") is False:
+ return _engine_error(400, "'parse_special': false is not supported for MLX models")
+
+ add_special = body.get("add_special") is True
+ pieces = [content] if isinstance(content, str) else list(content)
+
+ # Resolution polls this event; without it an abandoned request pins a worker thread.
+ cancel = threading.Event()
+
+ def tokenize():
+ # Only a leading string gets specials, so a leading id (dropped after) tokenizes without.
+ prompt = pieces if add_special else [0, *pieces]
+ ids = backend.resolve_completion_prompt(prompt, cancel_event = cancel)["ids"]
+ return ids if add_special else ids[1:]
+
+ watcher = asyncio.create_task(_inference()._await_disconnect_then_cancel(request, cancel))
+ try:
+ tokens = await asyncio.to_thread(tokenize)
+ except ValueError as exc:
+ if any(piece != "" for piece in pieces):
+ return _engine_error(400, str(exc))
+ tokens = [] # llama-server answers empty content with an empty list
+ except RuntimeError:
+ # Unloaded since selection: an unavailable engine, not a server fault.
+ return _engine_error(503, "No llama.cpp model is loaded")
+ except InterruptedError:
+ return _engine_error(499, "Request cancelled.")
+ finally:
+ await _inference()._stop_local_disconnect_cancel_watcher(watcher)
+ return JSONResponse({"tokens": tokens})
+
+
def _make_passthrough(path: str):
async def passthrough(request: Request, current_subject: str = Depends(get_current_subject)):
return await _forward_to_llama_server(request, path)
diff --git a/studio/backend/tests/test_llama_compat_routes.py b/studio/backend/tests/test_llama_compat_routes.py
index 11e251e08c1..339f829cc75 100644
--- a/studio/backend/tests/test_llama_compat_routes.py
+++ b/studio/backend/tests/test_llama_compat_routes.py
@@ -16,6 +16,7 @@ on the body. The module is loaded standalone with a double for routes.inference.
from __future__ import annotations
+import asyncio
import importlib.util
import sys
import time
@@ -82,6 +83,7 @@ def _inference_double(
backend,
catalog,
orchestrator = None,
+ mlx = None,
):
"""Stand-in for routes.inference. Deliberately not a sys.modules entry: stubbing it
globally broke two unrelated route tests when the suite ran in one process."""
@@ -90,12 +92,27 @@ def _inference_double(
inference._llama_public_model_id = lambda b, fallback = None: b._openai_advertised_id
inference._peek_inference_backend = lambda: orchestrator
inference._orchestrator_public_model_id = lambda b: b.active_model_name
+ inference._active_mlx_backend = lambda: mlx
async def _objects(quant_pins = False):
return catalog
inference._openai_catalog_objects = _objects
inference.model_slots = types.SimpleNamespace(visible_loading = lambda: None)
+
+ # Inert: is_disconnected() never returns under TestClient; one test supplies its own watcher.
+ async def _await_disconnect_then_cancel(request, cancel_event):
+ try:
+ await asyncio.Event().wait()
+ except asyncio.CancelledError:
+ return
+
+ async def _stop_watcher(watcher, timeout_s = 1.0):
+ watcher.cancel()
+ await asyncio.wait({watcher}, timeout = timeout_s)
+
+ inference._await_disconnect_then_cancel = _await_disconnect_then_cancel
+ inference._stop_local_disconnect_cancel_watcher = _stop_watcher
return inference
@@ -106,6 +123,7 @@ def _load(
backend = None,
catalog = None,
orchestrator = None,
+ mlx = None,
):
"""The real routes/llama_compat.py, with routes.inference replaced by a double."""
global _MOD
@@ -120,6 +138,7 @@ def _load(
backend if backend is not None else _Backend(),
CATALOG if catalog is None else catalog,
orchestrator,
+ mlx,
)
_MOD._inference = lambda: double
return _MOD
@@ -735,6 +754,45 @@ def test_nothing_loaded_is_503_in_llama_servers_error_shape(monkeypatch):
assert upstream.calls == []
+class _MLXTokenizer:
+ """resolve_completion_prompt's rules: only a leading string gets BOS (1), empty raises."""
+
+ def resolve_completion_prompt(
+ self,
+ prompt,
+ cancel_event = None,
+ ):
+ pieces = [prompt] if isinstance(prompt, str) else prompt
+ ids = []
+ for index, piece in enumerate(pieces):
+ if isinstance(piece, int):
+ ids.append(piece)
+ elif isinstance(piece, str):
+ if piece:
+ ids += ([1] if index == 0 else []) + [40 + i for i in range(len(piece))]
+ else:
+ raise ValueError("Invalid prompt.")
+ if not ids:
+ raise ValueError("The prompt is empty.")
+ return {"ids": ids, "text": ""}
+
+
+def test_tokenize_answers_from_a_resident_mlx_model(monkeypatch):
+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())
+ upstream = _Upstream()
+ _with_upstream(monkeypatch, mod, upstream)
+ with _client(mod) as c:
+ special = c.post("/tokenize", json = {"content": "ab", "add_special": True})
+ plain = c.post("/tokenize", json = {"content": "ab"})
+ empty = c.post("/tokenize", json = {"content": ""})
+ bad = c.post("/tokenize", json = {"content": 3})
+ pieces = c.post("/tokenize", json = {"content": "ab", "with_pieces": True})
+ assert special.json() == {"tokens": [1, 40, 41]} and plain.json() == {"tokens": [40, 41]}
+ assert empty.json() == {"tokens": []}
+ assert bad.status_code == 400 and pieces.status_code == 400
+ assert upstream.calls == []
+
+
def test_an_unload_racing_the_request_is_503_not_500(monkeypatch):
"""unload_model flips is_loaded before it clears _port."""
backend = _Backend()
@@ -1592,3 +1650,87 @@ def test_management_unload_routes_a_kept_slot_quant_past_the_primary(monkeypatch
with _client(mod) as c:
assert c.post("/models/unload", json = {"model": pin["id"]}).status_code == 200
assert calls == [("unload", pin["id"])]
+
+
+def test_tokenize_takes_llama_server_mixed_content_on_mlx(monkeypatch):
+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())
+ upstream = _Upstream()
+ _with_upstream(monkeypatch, mod, upstream)
+ with _client(mod) as c:
+ mixed = c.post("/tokenize", json = {"content": ["ab", 7, "c"]})
+ leading_special = c.post("/tokenize", json = {"content": ["ab", 7], "add_special": True})
+ bad = c.post("/tokenize", json = {"content": [{}]})
+ assert mixed.json() == {"tokens": [40, 41, 7, 40]}
+ assert leading_special.json() == {"tokens": [1, 40, 41, 7]}
+ assert bad.status_code == 400
+ assert upstream.calls == []
+
+
+def test_tokenize_refuses_parse_special_false_on_mlx(monkeypatch):
+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())
+ upstream = _Upstream()
+ _with_upstream(monkeypatch, mod, upstream)
+ with _client(mod) as c:
+ refused = c.post("/tokenize", json = {"content": "ab", "parse_special": False})
+ allowed = c.post("/tokenize", json = {"content": "ab", "parse_special": True})
+ assert refused.status_code == 400 and "parse_special" in refused.text
+ assert allowed.json() == {"tokens": [40, 41]}
+ assert upstream.calls == []
+
+
+def test_tokenize_reports_an_mlx_unload_race_as_an_unavailable_engine(monkeypatch):
+ class _Racing:
+ def resolve_completion_prompt(
+ self,
+ prompt,
+ cancel_event = None,
+ ):
+ raise RuntimeError("No active model")
+
+ mod = _load(backend = _Backend(loaded = False), mlx = _Racing())
+ upstream = _Upstream()
+ _with_upstream(monkeypatch, mod, upstream)
+ with _client(mod) as c:
+ raced = c.post("/tokenize", json = {"content": "ab"})
+ assert raced.status_code == 503
+ assert upstream.calls == []
+
+
+def test_tokenize_rejections_are_invalid_request_errors(monkeypatch):
+ mod = _load(backend = _Backend(loaded = False), mlx = _MLXTokenizer())
+ _with_upstream(monkeypatch, mod, _Upstream())
+ with _client(mod) as c:
+ bad = c.post("/tokenize", json = {"content": 3})
+ unavailable = c.post("/tokenize", json = {"content": "ab", "parse_special": False})
+ assert bad.status_code == 400 and bad.json()["error"]["type"] == "invalid_request_error"
+ assert unavailable.json()["error"]["type"] == "invalid_request_error"
+
+
+def test_tokenize_stops_waiting_for_the_generation_lock_when_the_client_goes(monkeypatch):
+ import threading
+
+ started = threading.Event()
+
+ class _Blocked:
+ def resolve_completion_prompt(
+ self,
+ prompt,
+ cancel_event = None,
+ ):
+ started.set()
+ # Event.wait, not a spin: a busy loop here starves the loop thread of the GIL.
+ assert cancel_event.wait(10), "the disconnect never reached the waiting resolve"
+ raise InterruptedError("Request cancelled.")
+
+ mod = _load(backend = _Backend(loaded = False), mlx = _Blocked())
+ _with_upstream(monkeypatch, mod, _Upstream())
+
+ async def disconnected(request, cancel_event):
+ while not started.is_set():
+ await asyncio.sleep(0.01)
+ cancel_event.set()
+
+ monkeypatch.setattr(mod._inference(), "_await_disconnect_then_cancel", disconnected)
+ with _client(mod) as c:
+ gone = c.post("/tokenize", json = {"content": "ab"})
+ assert gone.status_code == 499
diff --git a/studio/backend/tests/test_mlx_completions_apple.py b/studio/backend/tests/test_mlx_completions_apple.py
index deb9359bce1..a863c0b7ae6 100644
--- a/studio/backend/tests/test_mlx_completions_apple.py
+++ b/studio/backend/tests/test_mlx_completions_apple.py
@@ -114,3 +114,56 @@ def test_raw_vision_prompt_matches_the_decoded_sampled_ids():
tokens, skip_special_tokens = True, clean_up_tokenization_spaces = False
)
assert direct.strip() and _greedy(backend, ids) == direct
+
+
+def _direct_logprobs(backend, ids):
+ """mlx_lm's log_softmax for the first generated position with no processors, under the
+ scopes Studio decodes in: the logits llama-server reads ``n_probs`` from."""
+ from mlx_lm import stream_generate
+
+ with backend._int8_prefill_scope(), backend._generation_scopes():
+ (first,) = list(
+ stream_generate(backend._model, backend._tokenizer, prompt = ids, max_tokens = 1)
+ )
+ return first.logprobs
+
+
+# mlx_lm reduces its log_softmax in the model's dtype, so its reference carries a few ulps of
+# that dtype (bf16 for a 4-bit checkpoint); the recorder reads float32 like llama-server.
+_REF_EPS = {"bfloat16": 2.0**-8, "float16": 2.0**-11, "float32": 2.0**-24}
+
+
+def _ref_approx(direct, value):
+ eps = _REF_EPS.get(str(direct.dtype).rsplit(".", 1)[-1], 2.0**-8)
+ return pytest.approx(value, abs = max(1e-3, 4 * eps * abs(value)))
+
+
+def _scored(backend, ids, **kw):
+ for _ in backend.generate_completion(
+ ids, temperature = 0.0, top_p = 1.0, top_k = 0, min_p = 0.0, max_new_tokens = 1, **kw
+ ):
+ pass
+ return backend.last_generation_stats["token_logprobs"]
+
+
+def test_raw_prompt_logprobs_match_a_direct_forward_pass(backend):
+ ids = backend.resolve_completion_prompt(_PROMPT)["ids"]
+ direct = _direct_logprobs(backend, ids)
+ greedy = int(direct.argmax().item())
+ (entry,) = _scored(backend, ids, n_probs = 3)
+ assert entry["id"] == greedy
+ assert entry["logprob"] == _ref_approx(direct, direct[greedy].item())
+ assert entry["top_logprobs"][0]["id"] == greedy and len(entry["top_logprobs"]) == 3
+
+
+def test_raw_prompt_logprobs_report_a_forced_token_unbiased(backend):
+ import mlx.core as mx
+
+ ids = backend.resolve_completion_prompt(_PROMPT)["ids"]
+ direct = _direct_logprobs(backend, ids)
+ forced = int(mx.argsort(-direct)[100].item())
+ (entry,) = _scored(backend, ids, n_probs = 2, logit_bias = {forced: 100.0})
+ assert entry["id"] == forced
+ assert entry["logprob"] == _ref_approx(direct, direct[forced].item())
+ assert entry["logprob"] < -1.0
+ assert entry["top_logprobs"][0]["id"] == int(direct.argmax().item())
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 1020300d9d5..fd41f91092c 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -2855,6 +2855,172 @@ def test_mlx_vlm_completion_prompt_runs_untemplated_from_its_ids(monkeypatch):
assert list(backend.generate_completion([5, 6], max_new_tokens = 3))[-1] == " Paris!"
+def _numpy_mx(monkeypatch):
+ np = pytest.importorskip("numpy")
+
+ class _At:
+ def __init__(self, array):
+ self.array = array
+
+ def __getitem__(self, idx):
+ def add(values):
+ out = np.array(self.array)
+ np.add.at(out, np.asarray(idx), np.asarray(values))
+ return out
+
+ return SimpleNamespace(add = add)
+
+ class _Arr(np.ndarray):
+ @property
+ def at(self):
+ return _At(self)
+
+ def _logsumexp(
+ a,
+ axis = -1,
+ keepdims = False,
+ ):
+ peak = a.max(axis = axis, keepdims = True)
+ out = np.log(np.exp(a - peak).sum(axis = axis, keepdims = True)) + peak
+ return out if keepdims else out.squeeze(axis)
+
+ core = sys.modules["mlx.core"]
+ monkeypatch.setattr(core, "float32", np.float32, raising = False)
+ monkeypatch.setattr(core, "int32", np.int32, raising = False)
+ monkeypatch.setattr(
+ core, "array", lambda v, dtype = None: np.array(v, dtype = dtype), raising = False
+ )
+ monkeypatch.setattr(
+ core,
+ "zeros",
+ lambda shape, dtype = None: np.zeros(shape, dtype = dtype).view(_Arr),
+ raising = False,
+ )
+ monkeypatch.setattr(core, "logsumexp", _logsumexp, raising = False)
+ monkeypatch.setattr(core, "argpartition", lambda a, kth: np.argpartition(a, kth), raising = False)
+ monkeypatch.setattr(core, "argsort", np.argsort, raising = False)
+ return np
+
+
+# Probabilities the fake model gives every step: token 0 is the argmax, token 3 unlikely.
+_STEP_PROBS = [0.6, 0.3, 0.05, 0.04, 0.01]
+
+
+def _processed_steps(np, processors, prompt, steps):
+ """(token, response) per step as mlx_lm / mlx_vlm run processors: once per step, never
+ during prefill, step n + 1 computed before token n is yielded."""
+ raw = np.log(np.array([_STEP_PROBS], dtype = np.float32))
+ tokens = np.array(prompt)
+
+ def step(tokens):
+ logits = raw.copy()
+ for processor in processors or ():
+ logits = processor(tokens, logits)
+ return int(np.argmax(logits))
+
+ y = step(tokens)
+ for n in range(steps):
+ tokens = np.append(tokens, y)
+ upcoming = step(tokens) if n + 1 < steps else None
+ yield y, n + 1
+ y = upcoming
+
+
+def test_mlx_completion_logprobs_read_the_logits_before_logit_bias(monkeypatch):
+ import math
+
+ _install_fake_prompt_cache_api(monkeypatch)
+ captured = []
+ backend = _install_fake_text_stack(monkeypatch, {"generated": []}, captured)
+ np = _numpy_mx(monkeypatch)
+
+ def _stream_generate(_model, _tokenizer, **kw):
+ captured.append(kw)
+ steps = _processed_steps(np, kw.get("logits_processors"), kw["prompt"], kw["max_tokens"])
+ for token, n in steps:
+ yield SimpleNamespace(
+ token = token,
+ text = "",
+ prompt_tokens = len(kw["prompt"]),
+ prompt_tps = 1.0,
+ generation_tokens = n,
+ generation_tps = 1.0,
+ finish_reason = None,
+ )
+
+ monkeypatch.setattr(sys.modules["mlx_lm"], "stream_generate", _stream_generate)
+ sampling = dict(max_new_tokens = 2, temperature = 0.0, logit_bias = {3: 100.0})
+ out = list(backend.generate_completion([1, 2], n_probs = 2, **sampling))
+ entries = backend.last_generation_stats["token_logprobs"]
+ assert out[-1] == "33" and [e["id"] for e in entries] == [3, 3]
+ first = entries[0]
+ assert first["logprob"] == pytest.approx(math.log(0.04), abs = 1e-5)
+ assert first["token"] == "3" and first["bytes"] == [51]
+ assert [(t["id"], t["token"], t["bytes"]) for t in first["top_logprobs"]] == [
+ (0, "0", [48]),
+ (1, "1", [49]),
+ ]
+ assert first["top_logprobs"][0]["logprob"] == pytest.approx(math.log(0.6), abs = 1e-5)
+ list(backend.generate_completion([1, 2], max_new_tokens = 2))
+ assert "logits_processors" not in captured[-1]
+ assert "token_logprobs" not in backend.last_generation_stats
+
+
+def test_mlx_logprob_tokens_keep_a_sentencepiece_word_space(monkeypatch):
+ from core.inference import mlx_inference
+
+ _install_fake_mlx(monkeypatch)
+ pieces = {7: "\u2581the", 8: "Hand"}
+ tokenizer = SimpleNamespace(
+ decode = lambda ids, **_kw: "".join(pieces[i] for i in ids)
+ .replace("\u2581", " ")
+ .removeprefix(" ")
+ )
+ assert mlx_inference._token_bytes_text(tokenizer, 7) == (" the", [32, 116, 104, 101])
+ assert mlx_inference._token_bytes_text(tokenizer, 8) == ("Hand", [72, 97, 110, 100])
+
+
+def test_mlx_vlm_completion_logprobs_skip_the_draft_and_repeated_responses(monkeypatch):
+ from core.inference import mlx_inference
+
+ _install_fake_mlx(monkeypatch)
+ np = _numpy_mx(monkeypatch)
+ seen = {}
+
+ def _vlm_stream(*_a, **kw):
+ seen.update(kw)
+ steps = list(_processed_steps(np, kw["logits_processors"], kw["input_ids"][0], 2))
+ for token, n in steps:
+ yield SimpleNamespace(text = "", token = token, generation_tokens = n, prompt_tokens = 2)
+ # mlx-vlm repeats the last token in its closing response.
+ yield SimpleNamespace(text = "", token = token, generation_tokens = n, prompt_tokens = 2)
+
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.stream_generate = _vlm_stream
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setattr(
+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a: contextlib.nullcontext()
+ )
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._model, backend._is_vlm = SimpleNamespace(), True
+ backend._processor = SimpleNamespace()
+ backend._tokenizer = SimpleNamespace(
+ decode = lambda ids, skip_special_tokens, clean_up_tokenization_spaces: "".join(
+ str(i) for i in ids
+ )
+ )
+ monkeypatch.setattr(backend, "_vlm_prompt_cache_session", lambda *_a, **_k: None)
+ backend._speculative_draft = SimpleNamespace(drafter = None, draft_kind = "mtp")
+
+ out = list(backend.generate_completion([5, 6], max_new_tokens = 2, n_probs = 1))
+ entries = backend.last_generation_stats["token_logprobs"]
+ assert out[-1] == "00" and [e["id"] for e in entries] == [0, 0]
+ assert [t["id"] for t in entries[0]["top_logprobs"]] == [0]
+ assert "draft_model" not in seen
+ assert mlx_inference._request_batch_gap({"n_probs": 2})
+ assert mlx_inference._request_batch_gap({"n_probs": 0}) is None
+
+
def test_a_raw_vision_prompt_decodes_on_its_own_even_when_zoo_could_batch_it(monkeypatch):
from core.inference import mlx_inference
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index badf3936e30..b74589abec1 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -11943,11 +11943,11 @@ def test_mlx_completion_refusals_are_client_errors_as_in_chat(monkeypatch, stop,
_LLAMA_SERVED = (
"stream stream_options n_predict max_tokens max_completion_tokens top_k top_p min_p "
"temperature repeat_last_n repeat_penalty frequency_penalty presence_penalty seed "
- "logit_bias stop echo n n_cmpl grammar json_schema response_format"
+ "logit_bias stop echo n n_cmpl grammar json_schema response_format logprobs n_probs"
)
_LLAMA_IGNORED = (
"cache_prompt sse_ping_interval n_keep n_discard n_cache_reuse backend_sampling "
- "post_sampling_probs grammar_triggers chat_format "
+ "grammar_triggers chat_format "
"reasoning_format generation_prompt parse_tool_calls chat_parser continue_final_message "
"reasoning_control reasoning_budget_start_tag reasoning_budget_end_tags "
"reasoning_budget_end_tag reasoning_budget_message"
@@ -11955,7 +11955,7 @@ _LLAMA_IGNORED = (
# The rest change the text or the shape with no MLX equivalent: a value MLX refuses, and the one
# that turns the field off.
_LLAMA_UNSERVED = json.loads(
- '{"logprobs": [2, 0], "n_probs": [1, 0], "best_of": [2, 1], "suffix": ["x", ""],'
+ '{"best_of": [2, 1], "suffix": ["x", ""],'
' "timings_per_token": [true, false], "return_tokens": [true, false],'
' "return_progress": [true, false], "verbose": [true, false], "response_fields": [["id"], []], "lora":'
' [[{"id": 0, "scale": 1}], []], "n_indent": [2, 0], "t_max_predict_ms": [100, -1], "ignore_eos":'
@@ -11969,7 +11969,8 @@ _LLAMA_DEPENDENT = json.loads(
'{"xtc_threshold": "xtc_probability", "dynatemp_exponent": "dynatemp_range", "mirostat_tau":'
' "mirostat", "mirostat_eta": "mirostat", "dry_base": "dry_multiplier", "dry_allowed_length":'
' "dry_multiplier", "dry_penalty_last_n": "dry_multiplier", "dry_sequence_breakers":'
- ' "dry_multiplier", "adaptive_decay": "adaptive_target", "grammar_lazy": "grammar"}'
+ ' "dry_multiplier", "adaptive_decay": "adaptive_target", "grammar_lazy": "grammar",'
+ ' "post_sampling_probs": "n_probs"}'
)
@@ -12016,6 +12017,11 @@ def test_mlx_completion_refuses_only_what_mlx_cannot_reproduce(monkeypatch):
assert refusal({"prompt": "p", "xtc_probability": 0.5, "xtc_threshold": 0.6}) is None
assert refusal({"prompt": "p", "dry_multiplier": 0.8, "dry_penalty_last_n": 0}) is None
assert refusal({"prompt": "p", "n_cmpl": 1, "n": 2, "n_probs": 0, "logprobs": 5}) is None
+ # llama-server reads post_sampling_probs only with n_probs, and then renames the fields.
+ assert refusal({"prompt": "p", "post_sampling_probs": True}) is None
+ assert "'post_sampling_probs'" in refusal(
+ {"prompt": "p", "logprobs": 2, "post_sampling_probs": True}
+ )
assert refusal({"prompt": "p", "n": 2, "best_of": 2}) is None
assert refusal({"prompt": "p", "grammar": None, "json_schema": {}}) is None
for prompt in ([1, "a"], ["a", 1], [[1, 2]], {"prompt_string": [1, "a"]}):
@@ -12035,5 +12041,42 @@ def test_mlx_completion_refuses_only_what_mlx_cannot_reproduce(monkeypatch):
{"prompt": "p", "repeat_last_n": -1},
{"prompt": "p", "logit_bias": {"hi": 1}},
{"prompt": "p", "grammar": "root ::= nowhere"},
+ {"prompt": "p", "logprobs": 2, "stream": True},
+ {"prompt": "p", "logprobs": 2, "echo": True},
+ {"prompt": "p", "logprobs": 2, "n": 2},
+ {"prompt": ["a", "b"], "n_probs": 1},
+ {"prompt": "p", "n_probs": -1},
+ {"prompt": "p", "logprobs": True},
):
assert refusal(bad), bad
+
+
+def test_mlx_completion_reports_logprobs_in_llama_server_shape(monkeypatch):
+ entry = {
+ "id": 7,
+ "token": " a",
+ "bytes": [32, 97],
+ "logprob": -2.5,
+ "top_logprobs": [
+ {"id": 3, "token": " b", "bytes": [32, 98], "logprob": -0.1},
+ {"id": 7, "token": " a", "bytes": [32, 97], "logprob": -2.5},
+ ],
+ }
+ real = _MLXCompletionBackend.generate_completion
+
+ def generate_completion(self, prompt_ids, cancel_event, stats_holder, **kw):
+ yield from real(self, prompt_ids, cancel_event, stats_holder, **kw)
+ if kw.get("n_probs"):
+ stats_holder["stats"]["token_logprobs"] = [entry]
+
+ monkeypatch.setattr(_MLXCompletionBackend, "generate_completion", generate_completion)
+ body = {"prompt": [1, 2], "max_tokens": 1, "temperature": 0, "logprobs": 2}
+ backend, out = _run_mlx_completion(monkeypatch, {**body, "logit_bias": [[7, 100]]})
+ assert backend.call[1]["n_probs"] == 2 and backend.call[1]["logit_bias"] == {7: 100.0}
+ assert out["choices"][0]["logprobs"] == {"content": [entry]}
+ # n_probs is llama-server's own name and wins over the alias.
+ backend, _ = _run_mlx_completion(monkeypatch, {**body, "n_probs": 4})
+ assert backend.call[1]["n_probs"] == 4
+ for off in ({}, {"logprobs": 0}, {"logprobs": False}):
+ plain, out = _run_mlx_completion(monkeypatch, {"prompt": [1, 2], **off})
+ assert "n_probs" not in plain.call[1] and out["choices"][0]["logprobs"] is None
diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.py
index d479603cd31..cadc53016c3 100644
--- a/studio/backend/tests/test_presence_penalty.py
+++ b/studio/backend/tests/test_presence_penalty.py
@@ -495,10 +495,14 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serve
_ for _ in ()
).throw(_Built())
with pytest.raises(_Built):
- list(o.generate_completion([5, 6], max_new_tokens = 3, penalty_window = 0, grammar = "G"))
+ list(
+ o.generate_completion(
+ [5, 6], max_new_tokens = 3, penalty_window = 0, grammar = "G", n_probs = 2
+ )
+ )
cmd = built[0]
assert (cmd["messages"], cmd["prompt_ids"], cmd["penalty_window"]) == ([], [5, 6], 0)
- assert cmd["grammar"] == "G"
+ assert cmd["grammar"] == "G" and cmd["n_probs"] == 2
with pytest.raises(_Built):
list(o.generate_completion_batch([{"prompt_ids": [5]}], penalty_window = 1))
assert (built[1]["messages"], built[1]["rows"], built[1]["penalty_window"]) == (
@@ -506,6 +510,7 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serve
[{"prompt_ids": [5]}],
1,
)
+ assert "n_probs" not in built[1]
class _MLX:
last_generation_stats = None
@@ -521,7 +526,7 @@ def test_a_raw_completion_prompt_crosses_the_worker_only_to_a_backend_that_serve
mlx, queue = _MLX(), _Queue()
_handle_generate(mlx, cmd, queue, threading.Event())
assert mlx.received[0] == [5, 6] and mlx.received[1]["penalty_window"] == 0
- assert mlx.received[1]["grammar"] == "G"
+ assert mlx.received[1]["grammar"] == "G" and mlx.received[1]["n_probs"] == 2
assert [item["type"] for item in queue] == ["token", "gen_done"]
queue.clear()
_handle_generate(chat, cmd, queue, threading.Event())