Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: honor response_format on the MLX backend with grammar-constrained decoding (#10180)
Long Yixinga00fcbb33d68
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.pyindex 4dc24338af5..684d6c8e6de 100644--- a/studio/backend/core/inference/chat_template_helpers.py+++ b/studio/backend/core/inference/chat_template_helpers.py@@ -2603,9 +2603,9 @@ def detect_think_prefill(unclosed block that swallows the answer; in that case return ``""`` and fall back to plain text.``preserves_think_close`` says the stream keeps that closer anyway, as- ``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block. The- special-token list then says nothing, and skipping the opener is the same bug mirrored: a stray- ``</think>``.+ ``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block, and as+ a path streaming the detokenizer's own text does. The special-token list then says nothing, and+ skipping the opener is the same bug mirrored: a stray ``</think>``."""if not prompt:return ""diff --git a/studio/backend/core/inference/grammar_constraint.py b/studio/backend/core/inference/grammar_constraint.pynew file mode 100644index 00000000000..cb80f09092a--- /dev/null+++ b/studio/backend/core/inference/grammar_constraint.py@@ -0,0 +1,491 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Grammar-constrained decoding for the MLX paths, through llguidance token bitmasks."""++import hashlib+import json+import re+import threading+from collections import OrderedDict+from dataclasses import dataclass+from typing import Optional++try:+ import llguidance as _llg+ import llguidance.hf as _llg_hf+ import llguidance.mlx as _llg_mlx++ LLGUIDANCE_AVAILABLE = True+ LLGUIDANCE_VERSION = str(_llg.get_version())+except Exception:+ _llg = None+ _llg_hf = None+ _llg_mlx = None+ LLGUIDANCE_AVAILABLE = False+ LLGUIDANCE_VERSION = None++SUPPORTED_RESPONSE_FORMAT_TYPES = ("text", "json_object", "json_schema")++MISSING_ENGINE_MESSAGE = (+ "response_format needs a grammar engine, and llguidance did not load. Install or repair "+ "it with `pip install llguidance==1.8.0`, or load a GGUF model to use llama.cpp's."+)++# json_object promises an object, not any JSON value; spelled canonically to share a grammar.+_JSON_OBJECT_SCHEMA = '{"type":"object"}'++_PRELUDE_MAX_TOKENS = 4000++_THINK_MARKERS = ("<think>", "</think>")+++class ResponseFormatError(ValueError):+ """Invalid or unhonorable ``response_format``; the message is client-safe."""++ public = True+ openai_param = "response_format"+++class GrammarDesyncError(RuntimeError):+ """A token the mask never offered was committed; no change to the request fixes it."""+++@dataclass(frozen = True)+class ConstraintSpec:+ grammar: str+ schema_json: Optional[str] = None+ reasoning_close: Optional[str] = None+ reasoning_close_marker: Optional[str] = None++ def build(+ self,+ tokenizer,+ *,+ in_reasoning: bool = False,+ strips_special_tokens: bool = False,+ ) -> "GrammarConstraint":+ if in_reasoning:+ if self.reasoning_close is None:+ raise ResponseFormatError(+ "response_format cannot be honored on this model: the prompt starts "+ "inside an open reasoning block, and its closing marker "+ f"{self.reasoning_close_marker!r} cannot both be written by a grammar "+ "and survive into the reply, so nothing can close the block before the "+ "document. Send the request with thinking disabled, or load a GGUF "+ "build of this model."+ )+ grammar = _cached_grammar(self.schema_json, prelude_close = self.reasoning_close)+ else:+ grammar = self.grammar+ return GrammarConstraint(+ grammar,+ tokenizer,+ allows_reasoning = in_reasoning,+ strips_special_tokens = strips_special_tokens,+ )+++def build_constraint(+ response_format,+ tokenizer,+ prompt,+ *,+ reasoning_markers = None,+ tools = None,+ reasoning_is_extracted: bool = False,+ reply_keeps_special_tokens: bool = False,+) -> Optional["GrammarConstraint"]:+ markers = _reasoning_markers(tokenizer, reasoning_markers, tools)+ spec = constraint_spec_from_response_format(+ response_format,+ tokenizer,+ reasoning_markers = markers,+ reply_keeps_special_tokens = reply_keeps_special_tokens,+ )+ if spec is None:+ return None+ from core.inference.chat_template_helpers import prompt_opens_reasoning_channel++ return spec.build(+ tokenizer,+ in_reasoning = (reasoning_is_extracted and prompt_opens_reasoning_channel(prompt, markers)),+ strips_special_tokens = not reply_keeps_special_tokens,+ )+++def constraint_spec_from_response_format(+ response_format,+ tokenizer = None,+ *,+ reasoning_markers = None,+ reply_keeps_special_tokens: bool = False,+) -> Optional[ConstraintSpec]:+ if response_format is None:+ return None+ if not isinstance(response_format, dict):+ raise ResponseFormatError("response_format must be an object with a 'type' field")+ format_type = response_format.get("type")+ if format_type not in SUPPORTED_RESPONSE_FORMAT_TYPES:+ raise ResponseFormatError(+ f"unsupported response_format type {format_type!r}; supported: "+ + ", ".join(SUPPORTED_RESPONSE_FORMAT_TYPES)+ )+ if format_type == "text":+ if response_format != {"type": "text"}:+ raise ResponseFormatError(+ "unsupported members on a text response_format: "+ + ", ".join(sorted(k for k in response_format if k != "type"))+ )+ return None+ if not LLGUIDANCE_AVAILABLE:+ raise ResponseFormatError(MISSING_ENGINE_MESSAGE)++ if format_type == "json_object":+ supplied = response_format.get("schema")+ if supplied is not None:+ supplied = _require_schema_object(+ supplied,+ "response_format type 'json_object' requires a 'schema' member to be a "+ "JSON Schema object",+ )+ # Absent, null and empty mean this format's own object promise, not llguidance's any-JSON `{}`.+ schema_json = _JSON_OBJECT_SCHEMA if not supplied else _canonical_schema_json(supplied)+ else:+ schema_json = _canonical_schema_json(_json_schema_from_response_format(response_format))++ markers = tuple(reasoning_markers) if reasoning_markers else None+ return ConstraintSpec(+ grammar = _cached_grammar(schema_json, prelude_close = None),+ schema_json = schema_json,+ reasoning_close = _marker_reference(+ tokenizer, markers[1], reply_keeps_special_tokens = reply_keeps_special_tokens+ )+ if markers+ else None,+ reasoning_close_marker = markers[1] if markers else None,+ )+++def _json_schema_from_response_format(response_format: dict) -> dict:+ wrapper = response_format.get("json_schema")+ if isinstance(wrapper, dict):+ schema = wrapper.get("schema")+ elif wrapper is None:+ schema = response_format.get("schema")+ else:+ schema = None+ return _require_schema_object(+ schema,+ "response_format type 'json_schema' requires json_schema.schema to be a "+ "JSON Schema object",+ )+++def _require_schema_object(schema, message: str) -> dict:+ if not isinstance(schema, dict):+ raise ResponseFormatError(message)+ return schema+++def _reserved_token_text(tokenizer, token_id: int) -> str:+ try:+ return str(_unwrap_hf_tokenizer(tokenizer).convert_ids_to_tokens(int(token_id)))+ except Exception:+ return f"token {int(token_id)}"+++def _reasoning_markers(+ tokenizer,+ markers = None,+ tools = None,+) -> Optional[tuple]:+ if markers:+ return tuple(markers)+ if tokenizer is None:+ return None+ from core.inference.chat_template_helpers import detect_reasoning_channel_markers++ try:+ return detect_reasoning_channel_markers(tokenizer, tools = tools) or _THINK_MARKERS+ except Exception:+ return None+++def _marker_reference(tokenizer, marker: str, *, reply_keeps_special_tokens: bool) -> Optional[str]:+ if tokenizer is None:+ return None+ inner = _unwrap_hf_tokenizer(tokenizer)+ try:+ ids = inner.encode(marker, add_special_tokens = False)+ if len(ids) != 1:+ return None+ survives = reply_keeps_special_tokens or (+ inner.decode(ids, skip_special_tokens = True) == marker+ )+ except Exception:+ return None+ return marker if survives else None+++class GrammarConstraint:+ def __init__(+ self,+ grammar: str,+ tokenizer,+ *,+ allows_reasoning: bool = False,+ strips_special_tokens: bool = False,+ ):+ self._grammar = grammar+ self._tokenizer = tokenizer+ self._matcher = None+ self._bitmask = None+ self._stop_ids = ()+ self._stripped_ids = ()+ self._strips_special_tokens = strips_special_tokens+ self._dropped_ids = None+ self._runtime_stops = None+ self.allows_reasoning = allows_reasoning++ def _bind(self, n_vocab: int) -> None:+ self._stop_ids = (+ tuple(sorted({int(i) for i in self._runtime_stops if 0 <= int(i) < int(n_vocab)}))+ if self._runtime_stops+ else _runtime_stop_ids(self._tokenizer, n_vocab) or ()+ )+ if self._strips_special_tokens:+ # llguidance may spell a literal with a special id that this decode would delete.+ self._stripped_ids = _stripped_special_ids(+ self._tokenizer, n_vocab, self._stop_ids, self._dropped_ids+ )+ ll_tokenizer = _cached_ll_tokenizer(self._tokenizer, n_vocab, self._stop_ids or None)+ matcher = _llg.LLMatcher(ll_tokenizer, self._grammar)+ error = matcher.get_error()+ if error:+ raise ResponseFormatError(f"response_format grammar rejected: {error}")+ self._matcher = matcher+ self._bitmask = _llg_mlx.allocate_token_bitmask(1, n_vocab)++ def mask_logits(self, logits):+ if self._matcher is None:+ self._bind(int(logits.shape[-1]))+ _llg_mlx.fill_next_token_bitmask(self._matcher, self._bitmask)+ forced_stop = self._stop_offered_as_text()+ if forced_stop is not None:+ raise ResponseFormatError(self._desync_message(forced_stop))+ self._forbid_stripped_ids()+ masked = _llg_mlx.apply_token_bitmask(logits.reshape(1, -1), self._bitmask)+ return masked.reshape(logits.shape)++ def stops_on(self, ids) -> None:+ """The ids the decode loop really ends on, when the tokenizer alone cannot say."""+ self._runtime_stops = tuple(ids or ()) or None++ def decoded_dropping(self, ids = None) -> None:+ """The special ids the reply's decoder removes; None means every special id."""+ self._strips_special_tokens = True+ self._dropped_ids = None if ids is None else frozenset(int(i) for i in ids)++ def _forbid_stripped_ids(self) -> None:+ offered = [i for i in self._stripped_ids if self._mask_allowed(i)]+ if not offered:+ return+ import numpy as np++ words = self._bitmask.view(np.uint32)+ for token_id in offered:+ words[0, token_id // 32] &= np.uint32(~(1 << (token_id % 32)) & 0xFFFFFFFF)+ if not words[0].any():+ raise ResponseFormatError(self._desync_message(offered[0]))++ def _stop_offered_as_text(self) -> Optional[int]:+ if not self._stop_ids or self._matcher.is_stopped() or self._matcher.is_accepting():+ return None+ return next((i for i in self._stop_ids if self._mask_allowed(i)), None)++ def advance(self, token_id: int) -> None:+ if self._matcher is None:+ raise GrammarDesyncError("guided decoding advanced before the grammar was bound")+ if self._matcher.is_stopped():+ return+ self._matcher.consume_token(int(token_id))+ error = self._matcher.get_error()+ if error:+ if self._mask_allowed(int(token_id)):+ raise ResponseFormatError(self._desync_message(int(token_id)))+ raise GrammarDesyncError(f"guided decoding desynced on token {int(token_id)}: {error}")++ def _mask_allowed(self, token_id: int) -> bool:+ try:+ import numpy as np+ word = int(np.asarray(self._bitmask)[0, token_id // 32])+ except Exception:+ return False+ return bool(word >> (token_id % 32) & 1)++ def _desync_message(self, token_id: int) -> str:+ reserved = _reserved_token_text(self._tokenizer, token_id)+ return (+ f"response_format could not be honored: the reply reached a point where the "+ f"schema allows only the text {reserved!r}, which this model reserves as a "+ "control token and no grammar can spell as ordinary text. Remove that value "+ "from the schema."+ )+++def make_grammar_logits_processor(constraint: GrammarConstraint):+ state = {"seen": None}++ def _processor(tokens, logits):+ seen = state["seen"]+ if seen is None:+ state["seen"] = int(tokens.shape[0])+ else:+ for token_id in tokens[seen:].tolist():+ constraint.advance(int(token_id))+ state["seen"] = int(tokens.shape[0])+ return constraint.mask_logits(logits)++ return _processor+++# Bounded, strong-reference caches, so a recycled id() cannot alias a live entry.++_GRAMMAR_CACHE: "OrderedDict[str, str]" = OrderedDict()+_GRAMMAR_CACHE_MAX = 64+_GRAMMAR_CACHE_MAX_BYTES = 8 << 20+_TOKENIZER_CACHE: "OrderedDict[tuple, tuple]" = OrderedDict()+_TOKENIZER_CACHE_MAX = 4+_CACHE_LOCK = threading.Lock()+++def _canonical_schema_json(schema: dict) -> str:+ try:+ return json.dumps(schema, sort_keys = True, separators = (",", ":"))+ except (TypeError, ValueError) as exc:+ raise ResponseFormatError(f"response_format schema is not JSON-serializable: {exc}")+++def _cached_grammar(schema_json: str, *, prelude_close: Optional[str]) -> str:+ digest = hashlib.sha256(schema_json.encode("utf-8")).hexdigest()+ key = f"{LLGUIDANCE_VERSION}:{prelude_close or ''}:" f"{_PRELUDE_MAX_TOKENS}:{digest}"+ with _CACHE_LOCK:+ cached = _GRAMMAR_CACHE.get(key)+ if cached is not None:+ _GRAMMAR_CACHE.move_to_end(key)+ return cached+ grammar = _compile_grammar(schema_json, prelude_close)+ with _CACHE_LOCK:+ _GRAMMAR_CACHE[key] = grammar+ cached_bytes = sum(len(value) for value in _GRAMMAR_CACHE.values())+ while _GRAMMAR_CACHE and (+ len(_GRAMMAR_CACHE) > _GRAMMAR_CACHE_MAX or cached_bytes > _GRAMMAR_CACHE_MAX_BYTES+ ):+ _, evicted = _GRAMMAR_CACHE.popitem(last = False)+ cached_bytes -= len(evicted)+ return grammar+++def _compile_grammar(schema_json: str, prelude_close: Optional[str]) -> str:+ try:+ if prelude_close is None:+ grammar = _llg.LLMatcher.grammar_from_json_schema(schema_json)+ else:+ # No textual closer in reasoning (the route splits there); a {0,N} bound here exhausts llguidance's lexer.+ closer_rx = re.escape(prelude_close).replace("/", "\\/")+ grammar = (+ "%llguidance {}\n"+ "start: prelude doc\n"+ f"prelude: prelude_text {prelude_close}\n"+ f"prelude_text[max_tokens={_PRELUDE_MAX_TOKENS}]: "+ f"/(.|\\n)*/ & ~/(.|\\n)*{closer_rx}(.|\\n)*/\n"+ f"doc: %json {schema_json}\n"+ )+ except Exception as exc:+ raise ResponseFormatError(f"unsupported JSON Schema: {exc}") from exc+ error = _llg.LLMatcher.validate_grammar(grammar)+ if error:+ raise ResponseFormatError(f"unsupported JSON Schema: {error}")+ return grammar+++def _unwrap_hf_tokenizer(tokenizer):+ import transformers++ if isinstance(tokenizer, transformers.PreTrainedTokenizerFast):+ return tokenizer+ inner = getattr(tokenizer, "_tokenizer", None)+ if isinstance(inner, transformers.PreTrainedTokenizerFast):+ return inner+ inner = getattr(tokenizer, "tokenizer", None)+ if isinstance(inner, transformers.PreTrainedTokenizerFast):+ return inner+ return tokenizer+++def _stripped_special_ids(+ tokenizer,+ n_vocab: int,+ stop_ids,+ dropped = None,+) -> tuple:+ inner = _unwrap_hf_tokenizer(tokenizer)+ if dropped is not None:+ ids = set(dropped)+ else:+ # skip_special_tokens drops every added special token, not just all_special_ids.+ try:+ ids = set(getattr(inner, "all_special_ids", None) or ())+ ids.update(+ i+ for i, t in (getattr(inner, "added_tokens_decoder", None) or {}).items()+ if t.special+ )+ except Exception:+ return ()+ # llguidance ends on the runtime stops, or on the tokenizer's own end token when there are none.+ keep = set(stop_ids) if stop_ids else {getattr(inner, "eos_token_id", None)}+ return tuple(sorted({int(i) for i in ids if 0 <= int(i) < int(n_vocab)} - keep))+++def _runtime_stop_ids(tokenizer, n_vocab: int) -> Optional[tuple]:+ for obj in (tokenizer, getattr(tokenizer, "tokenizer", None)):+ if obj is None:+ continue+ for ids in (+ getattr(obj, "eos_token_ids", None),+ getattr(getattr(obj, "stopping_criteria", None), "eos_token_ids", None),+ ):+ try:+ found = tuple(sorted({int(i) for i in ids if 0 <= int(i) < int(n_vocab)}))+ except Exception:+ continue+ if found:+ return found+ return None+++def _cached_ll_tokenizer(+ tokenizer,+ n_vocab: int,+ stop_ids = None,+):+ stop_ids = stop_ids or _runtime_stop_ids(tokenizer, n_vocab)+ tokenizer = _unwrap_hf_tokenizer(tokenizer)+ key = (id(tokenizer), int(n_vocab), stop_ids)+ with _CACHE_LOCK:+ entry = _TOKENIZER_CACHE.get(key)+ if entry is not None and entry[0] is tokenizer:+ _TOKENIZER_CACHE.move_to_end(key)+ return entry[1]+ ll_tokenizer = _llg_hf.from_tokenizer(+ tokenizer,+ n_vocab = int(n_vocab),+ eos_token = list(stop_ids) if stop_ids else None,+ )+ with _CACHE_LOCK:+ _TOKENIZER_CACHE[key] = (tokenizer, ll_tokenizer)+ while len(_TOKENIZER_CACHE) > _TOKENIZER_CACHE_MAX:+ _TOKENIZER_CACHE.popitem(last = False)+ return ll_tokenizerdiff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex ef1e5bf1d3a..20b3f560d48 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2311,12 +2311,52 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):return _processor+def _vlm_generation_is_diffusion(model) -> bool:+ try:+ from mlx_vlm.generate.diffusion import is_diffusion_model+ except Exception:+ return False+ try:+ return bool(is_diffusion_model(model))+ except Exception:+ return False+++def _build_grammar_constraint(+ response_format,+ tokenizer,+ prompt,+ *,+ reasoning_markers = None,+ tools = None,+ reasoning_is_extracted = False,+ reply_keeps_special_tokens = False,+):+ if response_format is None or response_format == {"type": "text"}:+ return None+ from core.inference.grammar_constraint import build_constraint++ constraint = build_constraint(+ response_format,+ tokenizer,+ prompt,+ reasoning_markers = reasoning_markers,+ tools = tools,+ reasoning_is_extracted = reasoning_is_extracted,+ reply_keeps_special_tokens = reply_keeps_special_tokens,+ )+ if constraint is not None:+ logger.info("Guided decoding active: response_format=%s", response_format.get("type"))+ return constraint++def _mlx_sampling_processors(*,repetition_penalty = None,presence_penalty: float = 0.0,frequency_penalty: float = 0.0,logit_bias = None,+ grammar_constraint = None,):"""Logits processors for the sampling knobs, or ``None`` when all are inert. Bias runs beforethe penalties, matching llama-server's sampler order. mlx_lm supplies only the repetition@@ -2324,6 +2364,9 @@ def _mlx_sampling_processors(prompt*, while the penalties below score the whole completion and exclude it, so using themwould make the same request sample differently depending on the backend."""processors = []+ if grammar_constraint is not None:+ from core.inference.grammar_constraint import make_grammar_logits_processor+ processors.append(make_grammar_logits_processor(grammar_constraint))if logit_bias:processors.append(_make_mlx_logit_bias_processor(logit_bias))if repetition_penalty is not None and float(repetition_penalty) not in (0.0, 1.0):@@ -3246,6 +3289,8 @@ class MLXInferenceBackend:frequency_penalty = 0.0,logit_bias = None,stop = None,+ response_format = None,+ reasoning_is_extracted = False,_adapter_state = None,# Unrestricted mode runs the tool protocol with an EMPTY tools list, so bool(tools)# cannot tell that the wrappers below still have to survive decoding.@@ -3336,6 +3381,8 @@ class MLXInferenceBackend:_adapter_state = _adapter_state,stop = stop,tool_protocol_active = tool_protocol_active,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,video = video,)else:@@ -3360,6 +3407,8 @@ class MLXInferenceBackend:_adapter_state = _adapter_state,stop = stop,tool_protocol_active = tool_protocol_active,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,)yield from stream@@ -3391,6 +3440,8 @@ class MLXInferenceBackend:_adapter_state = None,tool_protocol_active = None,stop = None,+ response_format = None,+ reasoning_is_extracted = False,):from mlx_lm import stream_generatefrom mlx_lm.sample_utils import make_sampler@@ -3410,20 +3461,32 @@ class MLXInferenceBackend:# Not the request flag: a later tool-loop pass keeps it but renders an ordinary post-tool prompt._resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))+ preserve_native_channels = reasoning_channel_markers is not None# An open <think> prefilled by the template lives in the prompt, not the generated tokens; re-emit it so the# frontend renders the block.+ # Matches native_token_decoder below: when it runs </think> survives, so re-emit the opener.+ think_close_survives = (+ bool(tools) or tool_protocol_active or reasoning_channel_markers is not None+ ) and decoder_preserves_token(+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)+ )think_prefix = detect_think_prefill(prompt,getattr(self._tokenizer, "all_special_tokens", None),- # Matches native_token_decoder below: when it runs </think> survives, so the- # prefilled opener has to be re-emitted with it.- preserves_think_close = (- bool(tools) or tool_protocol_active or reasoning_channel_markers is not None- )- and decoder_preserves_token(- self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)- ),+ preserves_think_close = think_close_survives,)+ constraint = _build_grammar_constraint(+ response_format,+ self._tokenizer,+ prompt,+ reasoning_markers = reasoning_channel_markers,+ reasoning_is_extracted = reasoning_is_extracted,+ # Must match the prefill's answer, or a closer it re-emits would be refused.+ reply_keeps_special_tokens = preserve_native_channels or think_close_survives,+ )+ if constraint is not None and not constraint.allows_reasoning:+ think_prefix = ""+ preserve_native_channels = Falseif seed is None:sampler = make_sampler(temp = temperature,@@ -3445,9 +3508,9 @@ class MLXInferenceBackend:presence_penalty = presence_penalty,frequency_penalty = frequency_penalty,logit_bias = logit_bias,+ grammar_constraint = constraint,)- preserve_native_channels = reasoning_channel_markers is not Nonenative_token_decoder = (NativeToolTokenDecoder(self._tokenizer,@@ -3456,6 +3519,10 @@ class MLXInferenceBackend:if tools or preserve_native_channels or tool_protocol_activeelse None)+ if constraint is not None:+ constraint.decoded_dropping(+ native_token_decoder.dropped_ids() if native_token_decoder is not None else None+ )# Consulted per token on the reasoning path below, so resolved once here.stop_token_ids = (_mlx_stop_token_ids(self._tokenizer, self._model)@@ -3480,7 +3547,7 @@ class MLXInferenceBackend:prompt, reasoning_channel_markers, _resumed_partial),)- if reasoning_channel_markers is not None+ if preserve_native_channelselse None)# Sequences match the sampled text, ahead of the prefill this path restores and the <think> rewriting below:@@ -3579,6 +3646,12 @@ class MLXInferenceBackend:sampled = self._tokenizer.decode(token_ids,skip_special_tokens = True,+ # Cleanup would rewrite " ." inside a grammar-approved string.+ **(+ {"clean_up_tokenization_spaces": False}+ if constraint is not None+ else {}+ ),)if not sequences:yield think_prefix + sampled@@ -3811,6 +3884,8 @@ class MLXInferenceBackend:tool_protocol_active = None,stop = None,video = None,+ response_format = None,+ reasoning_is_extracted = False,):from mlx_vlm import stream_generate as vlm_stream@@ -3830,7 +3905,6 @@ class MLXInferenceBackend:# Detected once: the decoder keeps the delimiters the normalizer below consumes.vlm_reasoning_markers = detect_reasoning_channel_markers(chat_target, tools = tools)- # Re-emit an open <think> prefill from the prompt (see _generate_text).prefill = detect_think_prefill(prompt,getattr(chat_target, "all_special_tokens", None),@@ -3885,7 +3959,30 @@ class MLXInferenceBackend:0.0,1.0,)- if presence_penalty or frequency_penalty or logit_bias:+ constraint = _build_grammar_constraint(+ response_format,+ chat_target,+ prompt,+ reasoning_markers = vlm_reasoning_markers,+ tools = tools,+ reasoning_is_extracted = reasoning_is_extracted,+ reply_keeps_special_tokens = True,+ )+ document_only = constraint is not None and not constraint.allows_reasoning+ if document_only:+ prefill = ""+ elif constraint is not None:+ # The grammar lets a kept </think> close the block: re-emit the opener or it all reads as content.+ prefill = detect_think_prefill(prompt, preserves_think_close = True)+ if constraint is not None and _vlm_generation_is_diffusion(self._model):+ from core.inference.grammar_constraint import ResponseFormatError+ raise ResponseFormatError(+ "response_format is not supported on this model: it generates by "+ "diffusion rather than one token at a time, so no grammar can "+ "constrain the next token. Load an autoregressive model to use "+ "guided decoding."+ )+ if presence_penalty or frequency_penalty or logit_bias or constraint is not None:# These need custom processors: pass the full list (repetition + the rest) instead of the# repetition_penalty shortcut so all apply.vlm_kwargs["logits_processors"] = _mlx_sampling_processors(@@ -3893,6 +3990,7 @@ class MLXInferenceBackend:presence_penalty = presence_penalty,frequency_penalty = frequency_penalty,logit_bias = logit_bias,+ grammar_constraint = constraint,)elif _rep_active:vlm_kwargs["repetition_penalty"] = float(repetition_penalty)@@ -3909,6 +4007,11 @@ class MLXInferenceBackend:and self._tokenizerelse None)+ if constraint is not None:+ # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.+ constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))+ if constraint is not None and vlm_token_decoder is not None:+ constraint.decoded_dropping(vlm_token_decoder.dropped_ids())# The runtime EOS can itself be an allowlisted control, and this path appends every# decoded token to the snapshot, so it would trail each answer. As in _generate_text.vlm_stop_ids = (@@ -4059,16 +4162,20 @@ class MLXInferenceBackend:),)- yield from normalize_reasoning_snapshots(- _stream_vlm_snapshots(),- chat_target,- cancel_event,- markers = vlm_reasoning_markers,- tools = tools,- prompt = prompt,- continued = vlm_continued,- ended = lambda: stopped,- )+ if document_only:+ # The normalizer would rewrite marker text inside the document.+ yield from _stream_vlm_snapshots()+ else:+ yield from normalize_reasoning_snapshots(+ _stream_vlm_snapshots(),+ chat_target,+ cancel_event,+ markers = vlm_reasoning_markers,+ tools = tools,+ prompt = prompt,+ continued = vlm_continued,+ ended = lambda: stopped,+ )if stopped:self._mark_stopped()diff --git a/studio/backend/core/inference/native_tool_tokens.py b/studio/backend/core/inference/native_tool_tokens.pyindex acea3498911..a5117bc6155 100644--- a/studio/backend/core/inference/native_tool_tokens.py+++ b/studio/backend/core/inference/native_tool_tokens.py@@ -258,6 +258,10 @@ class NativeToolTokenDecoder:self._tokenizer = tokenizerself._special_ids, self._tool_ids = _special_token_sets(tokenizer, preserved_tokens)+ def dropped_ids(self) -> "frozenset[int] | None":+ """The special ids this decoder removes, or None when it falls back to dropping all."""+ return (self._special_ids - self._tool_ids) if self._special_ids else None+def preserves(self, token: str) -> bool:"""Whether this decoder really keeps ``token``, which the allowlist does not settle:with no usable ``all_special_ids`` every decode falls back todiff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex bf5366210bc..815532208be 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -258,32 +258,42 @@ class GenStreamError(str):from model output whose visible text starts with "Error:" by checking isinstance(chunk,GenStreamError)."""- __slots__ = ("public",)+ __slots__ = ("public", "openai_param")def __new__(cls,value,*,public: bool = False,+ openai_param: Optional[str] = None,):obj = str.__new__(cls, value)obj.public = bool(public)+ # Set for a refusal about one request field, so the caller answers 400 not 500.+ obj.openai_param = openai_paramreturn objclass GenStreamErrorRaised(RuntimeError):"""Internal exception form of ``GenStreamError`` for generator boundaries."""- __slots__ = ("public",)+ __slots__ = ("public", "openai_param")def __init__(self,value,*,public: bool = False,+ openai_param: Optional[str] = None,):super().__init__(value)self.public = bool(public)+ self.openai_param = openai_param++ @classmethod+ def from_chunk(cls, chunk: "GenStreamError") -> "GenStreamErrorRaised":+ """Keeps public/openai_param so a refusal is not answered 500."""+ return cls(str(chunk), public = chunk.public, openai_param = chunk.openai_param)def _summed_tool_loop_stats(total, turn):@@ -1302,6 +1312,8 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video_b64: Optional[str] = None,+ response_format: Optional[dict] = None,+ reasoning_is_extracted: bool = False,) -> dict:"""Build the 'generate' command shared by the locked and dispatched paths."""cmd = {@@ -1326,6 +1338,9 @@ class InferenceOrchestrator:cmd["seed"] = seedif stop:cmd["stop"] = stop+ if response_format is not None:+ cmd["response_format"] = response_format+ cmd["reasoning_is_extracted"] = bool(reasoning_is_extracted)if video_b64:cmd["video_base64"] = video_b64if use_adapter is not None:@@ -1410,7 +1425,11 @@ class InferenceOrchestrator:stats_holder["stats"] = resp.get("stats")returnelif rtype == "gen_error":- yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")+ yield GenStreamError(+ f"Error: {resp.get('error', 'Unknown error')}",+ public = bool(resp.get("public", False)),+ openai_param = resp.get("openai_param"),+ )returndef _start_dispatcher(self) -> bool:@@ -1541,6 +1560,8 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,+ response_format: Optional[dict] = None,+ reasoning_is_extracted: bool = False,) -> Generator[str, None, None]:"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-requestmailbox for tokens so two compare-mode requests can be queued at once. The subprocess@@ -1596,6 +1617,8 @@ class InferenceOrchestrator:frequency_penalty = frequency_penalty,logit_bias = logit_bias,stop = stop,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,use_adapter = use_adapter,tools = tools,enable_thinking = enable_thinking,@@ -2385,6 +2408,8 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,+ response_format: Optional[dict] = None,+ reasoning_is_extracted: bool = False,) -> Generator[str, None, None]:"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render@@ -2418,6 +2443,8 @@ class InferenceOrchestrator:frequency_penalty = frequency_penalty,logit_bias = logit_bias,stop = stop,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,video = video,)@@ -2530,7 +2557,7 @@ class InferenceOrchestrator:for chunk in stream:if isinstance(chunk, GenStreamError):close_stream = True- raise GenStreamErrorRaised(str(chunk), public = chunk.public)+ raise GenStreamErrorRaised.from_chunk(chunk)yield chunkfinally:if close_stream:@@ -2621,9 +2648,7 @@ class InferenceOrchestrator:try:for chunk in stream:if isinstance(chunk, GenStreamError):- # Preserve the public/operational flag so the route can surface the real message (e.g. "model is- # being unloaded") instead of a generic error. Mirrors the safetensors tool loop's _single_turn.- raise GenStreamErrorRaised(str(chunk), public = chunk.public)+ raise GenStreamErrorRaised.from_chunk(chunk)yield chunkfinally:close = getattr(stream, "close", None)@@ -2658,6 +2683,8 @@ class InferenceOrchestrator:logit_bias: Optional[dict] = None,stop: Optional[list] = None,video: Optional[str] = None,+ response_format: Optional[dict] = None,+ reasoning_is_extracted: bool = False,) -> Generator[str, None, None]:"""Inner generation logic: sends the command to the subprocess and yields tokens. Serializedby _gen_lock (one generation at a time) so concurrent readers don't consume each other's@@ -2701,6 +2728,8 @@ class InferenceOrchestrator:frequency_penalty = frequency_penalty,logit_bias = logit_bias,stop = stop,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,use_adapter = use_adapter,tools = tools,enable_thinking = enable_thinking,diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.pyindex 20c6275b4bc..281d790a893 100644--- a/studio/backend/core/inference/passthrough_healing.py+++ b/studio/backend/core/inference/passthrough_healing.py@@ -18,8 +18,9 @@ promotes calls whose function name exactly matches a declared tool. Promotionremoves EXACTLY the promoted calls' markup spans (the parser reports them):undeclared calls, unparseable blocks, and suppressed alternate formats keepevery byte and relay as text, so healing can never silently delete model-output. Responses without a tool signal, requests without tools, and Unsloth's-own enable-tools loop are untouched. Per-request opt-out:+output. Responses without a tool signal and requests without tools are untouched, as is+any request whose ``response_format`` constrains decoding: a grammar has already fixed what+the reply means. Per-request opt-out:``auto_heal_tool_calls: false``. Process kill-switch:``UNSLOTH_DISABLE_TOOL_CALL_HEALING=1``."""@@ -59,7 +60,13 @@ _HEALING_DISABLED = os.environ.get("UNSLOTH_DISABLE_TOOL_CALL_HEALING", "0") ==_NUDGE_DEFAULT = os.environ.get("UNSLOTH_TOOL_CALL_NUDGE", "0") == "1"-def nudge_enabled(request_flag: Optional[bool]) -> bool:+def response_format_constrains_decoding(response_format: Any) -> bool:+ return response_format is not None and response_format != {"type": "text"}+++def nudge_enabled(request_flag: Optional[bool], *, response_format: Any = None) -> bool:+ if response_format_constrains_decoding(response_format):+ return Falsereturn _NUDGE_DEFAULT if request_flag is None else bool(request_flag)@@ -73,6 +80,8 @@ def heal_gate(auto_heal: Optional[bool],tools: Optional[list],tool_choice: Any = None,+ *,+ response_format: Any = None,) -> Optional[set]:"""Return the declared client-tool name set when healing applies, else None.@@ -86,11 +95,15 @@ def heal_gate(markup stays text), and a forced ``{"type": "function", "function":{"name": N}}`` narrows promotion to that one function. ``"auto"`` /``"required"`` / absent keep the full declared set.++ A constraining ``response_format`` disables healing: schema strings resembling call markup would be promoted away."""if _HEALING_DISABLED or auto_heal is False:return Noneif tool_choice == "none":return None+ if response_format_constrains_decoding(response_format):+ return Nonenames = set()for tool in tools or []:if not isinstance(tool, dict):diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 9a61c6c54b4..f45dd68e249 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -753,7 +753,15 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:# backend's documented "ignores them" behavior into a TypeError.# ``tool_protocol_active`` rides here rather than above: MLX declares no such# parameter and takes no **kwargs, so an unconditional forward would raise.- for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):+ for gated in (+ "seed",+ "frequency_penalty",+ "logit_bias",+ "stop",+ "tool_protocol_active",+ "response_format",+ "reasoning_is_extracted",+ ):if gated in cmd and _backend_declares(backend, gated):gen_kwargs[gated] = cmd[gated]# A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.@@ -813,6 +821,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:"type": "gen_error","request_id": request_id,"error": str(exc),+ # Client-safe refusals would otherwise reach the caller as a generic 500.+ "public": bool(getattr(exc, "public", False)),+ "openai_param": getattr(exc, "openai_param", None),"stack": traceback.format_exc(limit = 20),},)@@ -1054,6 +1065,8 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve"type": "gen_error","request_id": request_id,"error": str(exc),+ "public": bool(getattr(exc, "public", False)),+ "openai_param": getattr(exc, "openai_param", None),"stack": traceback.format_exc(limit = 20),},)diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.pyindex 97645da309d..84c907edbeb 100644--- a/studio/backend/core/research_runs.py+++ b/studio/backend/core/research_runs.py@@ -126,7 +126,8 @@ _MODEL_WAIT_POLL_SECONDS = 2.0# A model that keeps disappearing would re-send forever, so cap how many times one call may wait._MAX_MODEL_WAITS = 3_NO_MODEL_LOADED_DETAIL = "No model loaded"-_NO_GRAMMAR_ENGINE_DETAIL = "needs the llama.cpp grammar engine"+# Every "no grammar engine here" refusal says this (routes, grammar_constraint); a re-send fixes those only.+_NO_GRAMMAR_ENGINE_DETAIL = "needs a grammar engine"# routes.inference reports the same unloaded state this way when auto-switch finds no local match._MODEL_NOT_FOUND_CODE = "model_not_found"# routes.inference 503s with this while an auto-switch to the run's model is still loading.diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 7ffcb492a9f..095091e7a60 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -237,6 +237,11 @@ _install_httpcore_asyncgen_silencer()_STATUS_PROBE_EXECUTOR = ThreadPoolExecutor(max_workers = 2, thread_name_prefix = "inference-status")+_SCHEMA_VALIDATION_EXECUTOR = ThreadPoolExecutor(+ max_workers = 2, thread_name_prefix = "inference-response-format"+)++# Lease waiters must not consume default workers needed by the current holder._OLLAMA_LEASE_EXECUTOR = ThreadPoolExecutor(max_workers = 2, thread_name_prefix = "inference-ollama-lease"@@ -340,6 +345,11 @@ def _friendly_gen_stream_error(value) -> str:return safe_error_detail(RuntimeError(text), fallback = "An internal error occurred.")+def _refused_parameter(value) -> Optional[str]:+ param = getattr(value, "openai_param", None)+ return param if getattr(value, "public", False) and param else None++def _friendly_upstream_error(text: str) -> str:"""Rewrite a raw llama-server error body into an actionable message where we can.@@ -25524,11 +25534,7 @@ async def produce_openai_chat_completions(or bool(payload.openai_code_exec_container_id)or bool(payload.anthropic_code_exec_container_id)# A JSON-schema response_format is guided-decoding structured output the- # router forwards to the llama-server passthrough, not Unsloth's tool- # loop, so a --enable-tools policy must not 400 it as a local-confirm- # request under ask/auto. Read with the predicate the router itself- # uses, so this gate cannot admit a request the router then rejects- # after a model switch, nor reject one the router would have served.+ # router answers with a grammar engine; read with the router's own predicate.or _response_format_constrains_decoding(payload))# permission_mode only implies the confirm gate for that local loop.@@ -25805,15 +25811,38 @@ async def produce_openai_chat_completions(_raise_unsupported_n("streaming chat completions")model_info = backend.models.get(backend.active_model_name, {})if _response_format_constrains_decoding(payload):- if model_info.get("is_audio") and model_info.get("audio_type") != "whisper":+ _audio_reply = model_info.get("is_audio") and model_info.get("audio_type") != "whisper"+ _transcribes = model_info.get("audio_type") == "whisper"+ _listens = payload.audio_base64 and model_info.get("has_audio_input")+ if _audio_reply or _transcribes or _listens:_raise_unsupported_openai_parameter("response_format",- "response_format cannot be honored by an audio reply; send the request to a text model "- "to use guided decoding.",+ "response_format is not supported when the reply comes from an audio "+ "route: a waveform admits no grammar, and transcription and audio-input "+ "answers are decoded without one. Send the request to a text model.",)- _raise_unsupported_openai_parameter(- "response_format",- "response_format needs the llama.cpp grammar engine; load a GGUF model to use it.",+ if not model_info.get("is_mlx"):+ _raise_unsupported_openai_parameter(+ "response_format",+ "response_format needs a grammar engine, and the transformers backend "+ "has none; load an MLX or GGUF model to use it.",+ )+ if payload.tool_choice not in (None, "auto", "none"):+ _raise_unsupported_openai_parameter(+ "tool_choice",+ "response_format cannot be combined with a tool_choice that requires a "+ "call: guided decoding admits only the document, so no tool would be "+ "called. Use tool_choice 'auto' or 'none'.",+ )+ if _continue_final_message(payload):+ _raise_unsupported_openai_parameter(+ "response_format",+ "response_format cannot resume a partial assistant message: guided "+ "decoding starts a new document rather than continuing the text before "+ "it. Send the turn without continue_final_message.",+ )+ await asyncio.get_running_loop().run_in_executor(+ _SCHEMA_VALIDATION_EXECUTOR, _reject_unhonorable_response_format, payload)# ── Audio TTS path: auto-route to audio generation ────@@ -28287,6 +28316,10 @@ async def produce_openai_chat_completions(# request as before.if _sf_tools_on and not _launcher_tool_default_applies(payload, _ui_events):_sf_tools_on = False+ if _response_format_constrains_decoding(payload) and not _explicit_studio_tool_loop_requested(+ payload+ ):+ _sf_tools_on = False# tool_choice: "none" withdraws the catalogue outright, as the GGUF loop does at its# controller. Nothing below reads the field (not _sf_use_tools, not# _select_request_tools, and generate_chat_completion_with_tools is never passed it), so@@ -28424,6 +28457,8 @@ async def produce_openai_chat_completions(_sf_continue = _continue_final_message(payload)_sf_continued_turn = [_sf_continue]+ _sf_single_block = _response_format_constrains_decoding(payload)+def _new_sf_reasoning_extractor():prefilled = _sf_reasoning_prefilledif _sf_continued_turn[0]:@@ -28432,6 +28467,7 @@ async def produce_openai_chat_completions(return _ResponsesReasoningExtractor(parse_think_markers = _sf_parse_think,reasoning_prefilled = prefilled,+ single_block = _sf_single_block,)cancel_event = _chat_cancel_event(request)@@ -28494,6 +28530,19 @@ async def produce_openai_chat_completions(if _sf_use_tools and _wants_multiple_choices(payload):_raise_unsupported_n("non-GGUF tool chat completions", monitor_id)+ if _sf_use_tools and _response_format_constrains_decoding(payload):+ raise _reject(+ 400,+ openai_error_body(+ "response_format is not supported with Unsloth tool execution; "+ "send the request without enable_tools or mcp_enabled to use "+ "guided decoding.",+ status = 400,+ code = "unsupported_parameter",+ param = "response_format",+ ),+ )+if _sf_use_tools:# permission_mode ask/auto require the confirm gate for Unsloth's own tool# loop; when a CLI policy (--enable-tools) forces the loop on without a@@ -29042,6 +29091,7 @@ async def produce_openai_chat_completions(frequency_penalty = payload.frequency_penalty,logit_bias = payload.logit_bias,stop = normalized_stop,+ response_format = _extract_response_format(payload),)if _video_clip is not None:gen_kwargs["video"] = _video_clip@@ -29127,7 +29177,12 @@ async def produce_openai_chat_completions(else None)_sf_heal = (- heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice)+ heal_gate(+ payload.auto_heal_tool_calls,+ _sf_healing_tools,+ payload.tool_choice,+ response_format = _extract_response_format(payload),+ )if _sf_client_toolselse None)@@ -29236,6 +29291,8 @@ async def produce_openai_chat_completions(template = _sf_image_tpl,prefer_tool_use = _sf_image_tpl is None,)+ if gen_kwargs.get("response_format") is not None:+ gen_kwargs["reasoning_is_extracted"] = bool(_sf_parse_think)# Request-scoped usage/timings receptacle (filled at gen_done).stats_holder: dict = {}@@ -29328,8 +29385,16 @@ async def produce_openai_chat_completions(backend.reset_generation_state(cancel_event)_msg = _friendly_gen_stream_error(cumulative)api_monitor.fail(monitor_id, _msg)+ _refused = _refused_parameter(cumulative)yield _openai_stream_error_sse(- {"error": {"message": _msg, "type": "server_error"}}+ openai_error_body(+ _msg,+ status = 400,+ code = "unsupported_parameter",+ param = _refused,+ )+ if _refused+ else {"error": {"message": _msg, "type": "server_error"}})returnif await request.is_disconnected():@@ -29468,7 +29533,14 @@ async def produce_openai_chat_completions(backend.reset_generation_state(cancel_event)_msg = _friendly_gen_stream_error(exc)api_monitor.fail(monitor_id, _msg)- yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}})+ _refused = _refused_parameter(exc)+ yield _openai_stream_error_sse(+ openai_error_body(+ _msg, status = 400, code = "unsupported_parameter", param = _refused+ )+ if _refused+ else {"error": {"message": _msg, "type": "server_error"}}+ )except Exception as e:backend.reset_generation_state(cancel_event)logger.error(f"Error during OpenAI streaming: {e}", exc_info = True)@@ -29550,6 +29622,17 @@ async def produce_openai_chat_completions(backend.reset_generation_state(cancel_event)_msg = _friendly_gen_stream_error(full_text)api_monitor.fail(monitor_id, _msg)+ _refused = _refused_parameter(full_text)+ if _refused:+ raise _reject(+ 400,+ openai_error_body(+ _msg,+ status = 400,+ code = "unsupported_parameter",+ param = _refused,+ ),+ )raise HTTPException(status_code = 500, detail = _msg)# Split prefilled <think> reasoning (GGUF parity); also covers MLX via@@ -29559,6 +29642,7 @@ async def produce_openai_chat_completions(full_text,parse_think_markers = _sf_parse_think,reasoning_prefilled = _sf_reasoning_prefilled and not _sf_continue,+ single_block = _sf_single_block,)# Client-tool passthrough: promote text-form calls; opt-in single# nudge retry on unparseable tool markup.@@ -29575,7 +29659,10 @@ async def produce_openai_chat_completions(if _sf_heal:if heal_openai_message(_msg, _sf_heal, _sf_healing_tools):_finish = "tool_calls"- elif nudge_enabled(payload.nudge_tool_calls):+ elif nudge_enabled(+ payload.nudge_tool_calls,+ response_format = _extract_response_format(payload),+ ):_data = {"choices": [{"message": {"role": "assistant", "content": _visible_text}}@@ -29753,10 +29840,18 @@ async def produce_openai_chat_completions(raiseexcept GenStreamErrorRaised as exc:# Adapter-controlled (compare-mode) backend failure. Honor the public- # flag so operational errors surface their real message.+ # flag, and the refused parameter so a contract refusal stays a client error.backend.reset_generation_state(cancel_event)_msg = _friendly_gen_stream_error(exc)api_monitor.fail(monitor_id, _msg)+ _refused = _refused_parameter(exc)+ if _refused:+ raise _reject(+ 400,+ openai_error_body(+ _msg, status = 400, code = "unsupported_parameter", param = _refused+ ),+ )raise HTTPException(status_code = 500, detail = _msg)except Exception as e:backend.reset_generation_state(cancel_event)@@ -32152,13 +32247,15 @@ class _ResponsesReasoningExtractor:*,parse_think_markers: bool = False,reasoning_prefilled: bool = False,+ single_block: bool = False,) -> None:self._buffer = ""# reasoning_prefilled: the template inserts an unclosed <think>, so output begins inside- # the block; start in reasoning until the first close tag. Existing callers pass False.- self._in_reasoning = reasoning_prefilled+ # the block; start in reasoning until the first close tag.+ self._in_reasoning = reasoning_prefilled and not single_block# Splitting requires marker parsing; a prefilled open implies it.self._parse_think_markers = parse_think_markers or reasoning_prefilled+ self._single_block = single_blockdef feed(self,@@ -32199,6 +32296,18 @@ class _ResponsesReasoningExtractor:self._buffer = self._buffer[-keep:] if keep else ""break+ if self._single_block:+ if self._buffer.startswith(_RESPONSES_THINK_OPEN):+ self._buffer = self._buffer[len(_RESPONSES_THINK_OPEN) :]+ self._in_reasoning = True+ continue+ if _RESPONSES_THINK_OPEN.startswith(self._buffer):+ break # still short of the opener; it may yet arrive+ self._parse_think_markers = False+ visible_parts.append(self._buffer)+ self._buffer = ""+ break+open_idx = self._buffer.find(_RESPONSES_THINK_OPEN)close_idx = self._buffer.find(_RESPONSES_THINK_CLOSE)if close_idx != -1 and (open_idx == -1 or close_idx < open_idx):@@ -32242,10 +32351,12 @@ def _extract_responses_reasoning(*,parse_think_markers: bool = False,reasoning_prefilled: bool = False,+ single_block: bool = False,) -> tuple[str, str]:extractor = _ResponsesReasoningExtractor(parse_think_markers = parse_think_markers,reasoning_prefilled = reasoning_prefilled,+ single_block = single_block,)reasoning, visible = extractor.feed(text, reasoning_content)final_reasoning, final_visible = extractor.finish()@@ -32822,6 +32933,7 @@ async def _responses_non_streaming(raw_text,msg.get("reasoning_content"),parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend),+ single_block = _response_format_constrains_decoding(chat_req),)tool_calls = msg.get("tool_calls") or []@@ -33062,7 +33174,8 @@ async def _responses_stream(# From the chat chunks; applied once before finish, so chunk order does not matter.stream_finish_reason: Optional[str] = Noneextractor = _ResponsesReasoningExtractor(- parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend)+ parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend),+ single_block = _response_format_constrains_decoding(chat_req),)reasoning_state: dict[str, Any] = {"output_index": None, "item_id": None, "opened": False}message_state: dict[str, Any] = {@@ -33086,6 +33199,7 @@ async def _responses_stream(getattr(chat_req, "auto_heal_tool_calls", None),body.get("tools"),body.get("tool_choice"),+ response_format = body.get("response_format"),)healer = StreamToolCallHealer(_allowed_tools, body.get("tools")) if _allowed_tools else Nonehealed_tc_index = 0@@ -37486,7 +37600,12 @@ async def _anthropic_passthrough_stream(# was already sent as "auto", and gating on the stale name would intersect the safe# names with a removed one and disable healing outright. "none" survives# reconciliation, so it still forbids promotion (#7066).- _allowed_tools = heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))+ _allowed_tools = heal_gate(+ auto_heal_tool_calls,+ body.get("tools"),+ body.get("tool_choice"),+ response_format = body.get("response_format"),+ )if _allowed_tools:emitter.enable_healing(_allowed_tools,@@ -37815,13 +37934,18 @@ async def _anthropic_passthrough_non_streaming(# was already sent as "auto", and gating on the stale name would intersect the safe# names with a removed one and disable healing outright. "none" survives# reconciliation, so it still forbids promotion (#7066).- _allowed_tools = heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))+ _allowed_tools = heal_gate(+ auto_heal_tool_calls,+ body.get("tools"),+ body.get("tool_choice"),+ response_format = body.get("response_format"),+ )# Opt-in single-retry nudge (mirrors the OpenAI passthrough): the tool call came out# unusable; re-ask with the prompt prefix intact so the KV cache is reused.if (_allowed_tools- and nudge_enabled(nudge_tool_calls)+ and nudge_enabled(nudge_tool_calls, response_format = body.get("response_format"))and nudge_should_retry(data, _allowed_tools, _healing_tools)):first_data = data@@ -38401,6 +38525,21 @@ def _extract_response_format(payload):return rf if isinstance(rf, dict) else None+def _response_format_for_llama_server(response_format):+ """llama-server reads a json_schema schema only from ``json_schema.schema`` (tools/server/server-common.cpp:1170)."""+ if (+ isinstance(response_format, dict)+ and response_format.get("type") == "json_schema"+ # Absent means missing or null, the reading the MLX path takes, so both agree.+ and response_format.get("json_schema") is None+ and isinstance(response_format.get("schema"), dict)+ ):+ wrapped = {key: value for key, value in response_format.items() if key != "schema"}+ wrapped["json_schema"] = {"schema": response_format["schema"]}+ return wrapped+ return response_format++def _response_format_constrains_decoding(payload) -> bool:"""Whether the request's ``response_format`` needs a grammar engine.@@ -38411,8 +38550,19 @@ def _response_format_constrains_decoding(payload) -> bool:members it does not know -- is a contract the caller expects to be kept, so itgoes where such a contract can be answered or rejected rather than dropped."""- rf = _extract_response_format(payload)- return isinstance(rf, dict) and rf != {"type": "text"}+ from core.inference.passthrough_healing import response_format_constrains_decoding+ return response_format_constrains_decoding(_extract_response_format(payload))+++def _reject_unhonorable_response_format(payload) -> None:+ from core.inference.grammar_constraint import (+ ResponseFormatError,+ constraint_spec_from_response_format,+ )+ try:+ constraint_spec_from_response_format(_extract_response_format(payload))+ except ResponseFormatError as exc:+ _raise_unsupported_openai_parameter("response_format", str(exc))def _build_openai_passthrough_body(@@ -38463,7 +38613,7 @@ def _build_openai_passthrough_body(frequency_penalty = payload.frequency_penalty,logit_bias = payload.logit_bias,tool_choice = tool_choice,- response_format = _extract_response_format(payload),+ response_format = _response_format_for_llama_server(_extract_response_format(payload)),chat_template_kwargs = tpl_kwargs,backend_ctx = backend_ctx,seed = payload.seed,@@ -38762,7 +38912,10 @@ async def _openai_passthrough_stream_admitted(# auto_heal_tool_calls=false keep the unhealed relay. tool_choice constrains# the allowlist ("none" disables, a forced function narrows to it)._allowed_tools = heal_gate(- payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")+ payload.auto_heal_tool_calls,+ body.get("tools"),+ body.get("tool_choice"),+ response_format = body.get("response_format"),)# Keep the pre-header window short so accepted SSE clients receive@@ -39801,7 +39954,10 @@ async def _openai_passthrough_non_streaming_upstream(_do_fence = _guided_fence and _extract_response_format(payload) is not None_cap_parallel = payload.parallel_tool_calls is False_allowed_tools = heal_gate(- payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")+ payload.auto_heal_tool_calls,+ body.get("tools"),+ body.get("tool_choice"),+ response_format = body.get("response_format"),)try:@@ -39823,7 +39979,7 @@ async def _openai_passthrough_non_streaming_upstream(usage_aggregated = Falseif (_allowed_tools- and nudge_enabled(payload.nudge_tool_calls)+ and nudge_enabled(payload.nudge_tool_calls, response_format = body.get("response_format"))and nudge_should_retry(data, _allowed_tools, body.get("tools"))):first_data = datadiff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.pyindex dcc1b6bdf83..7228454ae5e 100644--- a/studio/backend/tests/test_control_markup_neutralize_7066.py+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py@@ -3743,10 +3743,9 @@ def test_anthropic_healing_is_gated_on_the_sanitized_catalog():assert "heal_gate(auto_heal_tool_calls, openai_tools, tool_choice)" not in source# The third argument is the reconciled choice the body carries, not the caller's: see# test_healing_is_gated_on_the_tool_choice_actually_sent.- assert (- source.count('heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))')- == 2- )+ packed = " ".join(source.split()).replace("( ", "(")+ gate = 'heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")'+ assert packed.count(gate + ",") + packed.count(gate + ")") == 2assert "nudge_should_retry(data, _allowed_tools, openai_tools)" not in source@@ -4803,9 +4802,9 @@ def test_safetensors_healing_is_gated_on_the_sanitized_catalog():"heal_gate(payload.auto_heal_tool_calls, payload.tools, payload.tool_choice)" not in source)assert "_sf_renderable_tools," in source and "asyncio.to_thread(" in source- assert (- "heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice)" in source- )+ packed = " ".join(source.split()).replace("( ", "(")+ gate = "heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice"+ assert packed.count(gate + ",") + packed.count(gate + ")") == 1for call in ("StreamToolCallHealer(_sf_heal, _sf_healing_tools)","heal_openai_message(_msg, _sf_heal, _sf_healing_tools)",diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 21e0b91e1c2..72034ea6559 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -4903,6 +4903,320 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(assert guard < snapshot, "the guard must be held across the snapshot, not after it"+import numpy as np # noqa: E402++from core.inference import grammar_constraint as gc+from core.inference.grammar_constraint import (+ ResponseFormatError,+ build_constraint,+ constraint_spec_from_response_format,+)++SCHEMA = {+ "type": "object",+ "properties": {"a": {"type": "integer"}},+ "required": ["a"],+ "additionalProperties": False,+}++JSON_SCHEMA_FORMAT = {"type": "json_schema", "schema": SCHEMA}+++def _char_tokenizer(+ *extra_specials,+ markers_are_special = False,+ extra_chars = "",+):+ if not gc.LLGUIDANCE_AVAILABLE:+ pytest.skip("llguidance.mlx did not import")+ tk = pytest.importorskip("tokenizers")+ transformers = pytest.importorskip("transformers")++ chars = list('{}"[]:,0123456789abcdefghijklmnopqrstuvwxyz \n.-' + extra_chars)+ backend = tk.Tokenizer(tk.models.BPE(vocab = {c: i for i, c in enumerate(chars)}, merges = []))+ backend.pre_tokenizer = tk.pre_tokenizers.ByteLevel(add_prefix_space = False, use_regex = False)+ backend.decoder = tk.decoders.ByteLevel()+ backend.add_special_tokens([tk.AddedToken("<eos>", special = True)])+ markers = [tk.AddedToken(t, special = markers_are_special) for t in ("</think>", *extra_specials)]+ if markers_are_special:+ backend.add_special_tokens(markers)+ else:+ backend.add_tokens(markers)+ tokenizer = transformers.PreTrainedTokenizerFast(tokenizer_object = backend, eos_token = "<eos>")+ tokenizer.chat_template = "{{ messages[0].content }}<think>"+ return tokenizer++[email protected](scope = "module")+def tiny_tokenizer():+ return _char_tokenizer()+++def _metal_mx():+ mx = pytest.importorskip("mlx.core")+ if not mx.metal.is_available():+ pytest.skip("llguidance.mlx masks with a Metal kernel")+ return mx+++def _allowed(constraint, width):+ mx = _metal_mx()+ masked = np.asarray(constraint.mask_logits(mx.zeros((width,), dtype = mx.float32)))+ return set(np.nonzero(~np.isneginf(masked))[0].tolist())+++def test_every_spelling_of_a_supplied_schema_agrees_and_a_non_schema_is_refused(tiny_tokenizer):+ wrapped = constraint_spec_from_response_format(+ {"type": "json_schema", "json_schema": {"name": "city", "schema": SCHEMA}}+ ).grammar+ for spelling in (+ JSON_SCHEMA_FORMAT,+ {"type": "json_schema", "schema": dict(reversed(list(SCHEMA.items())))},+ {"type": "json_object", "schema": SCHEMA},+ ):+ assert constraint_spec_from_response_format(spelling).grammar == wrapped+ for supplied in (False, True, "nonsense", [1, 2]):+ with pytest.raises(ResponseFormatError):+ constraint_spec_from_response_format({"type": "json_object", "schema": supplied})+ plain = constraint_spec_from_response_format({"type": "json_object"}).grammar+ for spelling in (+ {"type": "json_object", "schema": None},+ {"type": "json_object", "schema": {}},+ {"type": "json_schema", "schema": {"type": "object"}},+ ):+ assert constraint_spec_from_response_format(spelling).grammar == plain+ digit = tiny_tokenizer.encode("4", add_special_tokens = False)[0]+ width = len(tiny_tokenizer)+ any_value = build_constraint({"type": "json_schema", "schema": {}}, tiny_tokenizer, "p")+ assert digit in _allowed(any_value, width)+ assert digit not in _allowed(+ build_constraint({"type": "json_object"}, tiny_tokenizer, "p"), width+ )+++def test_the_schema_itself_constrains_not_merely_json_syntax(tiny_tokenizer):+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")+ _allowed(constraint, len(tiny_tokenizer))+ for token_id in tiny_tokenizer.encode('{"', add_special_tokens = False):+ constraint.advance(int(token_id))+ assert _allowed(constraint, len(tiny_tokenizer)) == set(+ tiny_tokenizer.encode("a", add_special_tokens = False)+ )+ for token_id in tiny_tokenizer.encode('a":', add_special_tokens = False):+ constraint.advance(int(token_id))+ quote = tiny_tokenizer.encode('"', add_special_tokens = False)[0]+ assert quote not in _allowed(constraint, len(tiny_tokenizer))+++def test_advance_raises_on_a_token_the_grammar_rejects(tiny_tokenizer):+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")+ _allowed(constraint, len(tiny_tokenizer))+ illegal = tiny_tokenizer.encode("]", add_special_tokens = False)[0]+ with pytest.raises(gc.GrammarDesyncError, match = "desynced on token") as excinfo:+ constraint.advance(int(illegal))+ assert not isinstance(excinfo.value, ResponseFormatError)+++def test_binding_waits_for_the_logits_width_a_padded_lm_head_widens(tiny_tokenizer):+ gc._TOKENIZER_CACHE.clear()+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")+ padded = len(tiny_tokenizer) + 16+ legal = _allowed(constraint, padded)+ assert legal == set(tiny_tokenizer.encode("{", add_special_tokens = False))+ assert max(legal) < len(tiny_tokenizer)+++def test_prelude_is_bounded_so_an_unclosed_think_block_cannot_eat_the_budget(+ tiny_tokenizer, monkeypatch+):+ monkeypatch.setattr(gc, "_PRELUDE_MAX_TOKENS", 6)+ constraint = build_constraint(+ JSON_SCHEMA_FORMAT, tiny_tokenizer, "chat\n<think>\n", reasoning_is_extracted = True+ )+ _allowed(constraint, len(tiny_tokenizer))+ for token_id in tiny_tokenizer.encode("abcdef", add_special_tokens = False):+ constraint.advance(int(token_id))+ close_id = tiny_tokenizer.encode("</think>", add_special_tokens = False)[0]+ assert _allowed(constraint, len(tiny_tokenizer)) == {close_id}+ constraint.advance(int(close_id))+ assert _allowed(constraint, len(tiny_tokenizer)) == set(+ tiny_tokenizer.encode("{", add_special_tokens = False)+ )+++def test_reasoning_cannot_spell_the_closer_as_text_and_forge_the_document_boundary():+ tokenizer = _char_tokenizer(extra_chars = "</>")++ def _fresh():+ constraint = build_constraint(+ JSON_SCHEMA_FORMAT, tokenizer, "chat\n<think>", reasoning_is_extracted = True+ )+ constraint._bind(len(tokenizer))+ return constraint++ def _step(constraint, token_id):+ # The numpy half of mask_logits, so the desync check reads a real mask without Metal.+ gc._llg_mlx.fill_next_token_bitmask(constraint._matcher, constraint._bitmask)+ constraint.advance(int(token_id))++ def _chars(text):+ return [tokenizer.convert_tokens_to_ids(c) for c in text]++ constraint = _fresh()+ for token_id in _chars("a<b</think"):+ _step(constraint, token_id)+ with pytest.raises(gc.GrammarDesyncError):+ _step(constraint, tokenizer.convert_tokens_to_ids(">"))++ close_id = tokenizer.encode("</think>", add_special_tokens = False)+ assert len(close_id) == 1+ constraint = _fresh()+ for token_id in (+ _chars("a<b</th") + close_id + tokenizer.encode('{"a":1}', add_special_tokens = False)+ ):+ _step(constraint, token_id)++[email protected]("reply_keeps_special_tokens", [False, True])+def test_a_close_marker_is_refused_only_where_the_reply_would_lose_it(reply_keeps_special_tokens):+ tokenizer = _char_tokenizer(markers_are_special = True)+ ids = tokenizer.encode("</think>", add_special_tokens = False)+ assert (+ tokenizer.decode(ids, skip_special_tokens = True) != "</think>"+ ), "fixture no longer models a marker a skip_special_tokens decode strips"++ def _build():+ return build_constraint(+ JSON_SCHEMA_FORMAT,+ tokenizer,+ "chat\n<think>",+ reasoning_is_extracted = True,+ reply_keeps_special_tokens = reply_keeps_special_tokens,+ )++ if not reply_keeps_special_tokens:+ with pytest.raises(ResponseFormatError):+ _build()+ return+ constraint = _build()+ assert constraint.allows_reasoning+ for token_id in tokenizer.encode('x</think>{"a":1}', add_special_tokens = False):+ assert int(token_id) in _allowed(constraint, len(tokenizer))+ constraint.advance(int(token_id))+++class _Recorder:+ def __init__(self):+ self.advanced = []++ def mask_logits(self, logits):+ return logits++ def advance(self, token_id):+ self.advanced.append(int(token_id))+++def test_the_grammar_mask_runs_first_and_its_inf_survives_every_knob():+ mx = pytest.importorskip("mlx.core")+ pytest.importorskip("mlx_lm")+ from core.inference.mlx_inference import _mlx_sampling_processors++ class _MaskOdd(_Recorder):+ def mask_logits(self, logits):+ keep = mx.array([[1.0, 0.0, 1.0, 0.0]])+ return mx.where(keep > 0, logits, mx.array(float("-inf")))++ grammar = _MaskOdd()+ processors = _mlx_sampling_processors(+ repetition_penalty = 1.3,+ presence_penalty = 0.5,+ frequency_penalty = 0.5,+ logit_bias = {1: 100.0},+ grammar_constraint = grammar,+ )+ zeros = mx.zeros((1, 4), dtype = mx.float32)+ tokens = mx.array([0, 1])+ leading = np.asarray(processors[0](tokens, zeros))[0]+ assert np.isneginf(leading[1]) and np.isneginf(leading[3])++ logits = zeros+ for processor in processors:+ logits = processor(tokens, logits)+ out = np.asarray(logits)[0]+ assert np.isneginf(out[1]) and np.isneginf(out[3])+ assert np.isfinite(out[0]) and np.isfinite(out[2])++ assert grammar.advanced == []+ processors[0](mx.array([0, 1, 2, 3]), zeros)+ processors[0](mx.array([0, 1, 2, 3, 4]), zeros)+ assert grammar.advanced == [2, 3, 4]+++def _two_stop_tokenizer():+ tokenizer = _char_tokenizer("<end_of_text>", markers_are_special = True)+ tokenizer.chat_template = "{{ messages[0].content }}"+ runtime_stop = tokenizer.convert_tokens_to_ids("<end_of_text>")+ assert runtime_stop != tokenizer.eos_token_id, "fixture no longer models a disagreement"+ return tokenizer, runtime_stop+++def _stop_offered_after_the_document(target, tokenizer):+ constraint = build_constraint(JSON_SCHEMA_FORMAT, target, "prompt")+ for token_id in tokenizer.encode('{"a":1}', add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token_id))+ assert constraint._matcher.is_stopped(), "the document did not complete"+ return _allowed(constraint, len(tokenizer))+++def test_the_grammar_stops_on_the_token_the_decode_loop_stops_on():+ mlx_lm = pytest.importorskip("mlx_lm")+ from mlx_lm.tokenizer_utils import TokenizerWrapper++ tokenizer, runtime_stop = _two_stop_tokenizer()+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])+ assert _stop_offered_after_the_document(wrapped, tokenizer) == {runtime_stop}+ assert _stop_offered_after_the_document(tokenizer, tokenizer) == {tokenizer.eos_token_id}+++def test_a_schema_forced_through_a_stop_token_is_refused_while_it_can_still_be_reported():+ pytest.importorskip("mlx_lm")+ from mlx_lm.tokenizer_utils import TokenizerWrapper++ tokenizer, runtime_stop = _two_stop_tokenizer()+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])++ def replay(schema, text):+ constraint = build_constraint({"type": "json_schema", "schema": schema}, wrapped, "p")+ for token in tokenizer.encode(text, add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token))+ return _allowed(constraint, len(tokenizer))++ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):+ replay({"const": tokenizer.convert_ids_to_tokens(runtime_stop)}, '"')+ assert runtime_stop in replay({"type": "integer"}, "42")+ assert runtime_stop in replay(SCHEMA, '{"a":1}')+++def test_a_scalar_end_token_does_not_hide_the_runtime_stop_set():+ pytest.importorskip("mlx_vlm")+ from mlx_vlm.utils import StoppingCriteria++ tokenizer, runtime_stop = _two_stop_tokenizer()+ assert getattr(tokenizer, "eos_token_ids", None) == tokenizer.eos_token_id, "not a scalar"+ criteria = StoppingCriteria([runtime_stop], tokenizer = tokenizer)+ tokenizer.stopping_criteria = criteria+ for target in (tokenizer, SimpleNamespace(tokenizer = tokenizer)):+ assert _stop_offered_after_the_document(target, tokenizer) == set(criteria.eos_token_ids)+++def test_advance_before_the_first_mask_raises_rather_than_dropping_the_token(tiny_tokenizer):+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")+ with pytest.raises(gc.GrammarDesyncError, match = "before the grammar was bound"):+ constraint.advance(tiny_tokenizer.encode("{", add_special_tokens = False)[0])++def _uncopyable_naive_detokenizer(detokenizers):"""mlx-vlm's naive detokenizer as it behaves BELOW 0.6.0, which is where ``__copy__`` arrived.@@ -6739,3 +7053,351 @@ def test_single_image_replay_notes_describe_only_retained_pixels(monkeypatch, meelse:assert calls[-1]["prompt"].count(IMAGE_TURN_TEXT) == 1assert len(image_marker_parts(history)) == 2+++def test_a_random_model_decoding_through_mlx_lm_emits_only_the_schema():+ """Real mlx_lm pipelining (the next step is masked before this token is yielded) with a+ random model: only the grammar can make the reply a valid document."""+ mx = _metal_mx()+ pytest.importorskip("mlx_lm")+ from mlx_lm import stream_generate+ from mlx_lm.models import llama+ from mlx_lm.sample_utils import make_sampler+ from mlx_lm.tokenizer_utils import TokenizerWrapper++ from core.inference.grammar_constraint import make_grammar_logits_processor++ tokenizer = _char_tokenizer()+ tokenizer.chat_template = "{{ messages[0].content }}"+ mx.random.seed(0)+ model = llama.Model(+ llama.ModelArgs(+ model_type = "llama",+ hidden_size = 32,+ num_hidden_layers = 1,+ intermediate_size = 64,+ num_attention_heads = 2,+ num_key_value_heads = 2,+ rms_norm_eps = 1e-5,+ vocab_size = len(tokenizer),+ )+ )+ schema = {+ "type": "object",+ "properties": {"a": {"enum": ["x", "y"]}, "b": {"type": "boolean"}},+ "required": ["a", "b"],+ "additionalProperties": False,+ }+ wrapped = TokenizerWrapper(tokenizer)+ constraint = build_constraint({"type": "json_schema", "schema": schema}, wrapped, "p")+ text = "".join(+ chunk.text+ for chunk in stream_generate(+ model,+ wrapped,+ tokenizer.encode("go", add_special_tokens = False),+ max_tokens = 64,+ sampler = make_sampler(temp = 1.0),+ logits_processors = [make_grammar_logits_processor(constraint)],+ )+ )+ document = json.loads(text)+ assert document["a"] in ("x", "y") and isinstance(document["b"], bool)+ assert set(document) == {"a", "b"}+++def _constrained_vlm_reply(+ monkeypatch,+ pieces,+ *,+ allows_reasoning,+ markers,+ specials = (),+):+ """Snapshots ``_generate_vlm`` yields for scripted text under a stand-in grammar."""+ from core.inference import mlx_inference+ from core.inference.mlx_inference import MLXInferenceBackend++ mlx_vlm = types.ModuleType("mlx_vlm")+ mlx_vlm.prompt_utils = SimpleNamespace(MODEL_CONFIG = {}, apply_chat_template = None)+ mlx_vlm.stream_generate = lambda *_a, **_k: iter(+ [SimpleNamespace(text = piece, prompt_tokens = 1, generation_tokens = 1) for piece in pieces]+ )+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)+ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda *_a, **_k: "user: hi\nassistant: <think>\n",+ )+ monkeypatch.setattr(+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a, **_k: contextlib.nullcontext()+ )+ monkeypatch.setattr(+ mlx_inference, "detect_reasoning_channel_markers", lambda *_a, **_k: markers+ )+ monkeypatch.setattr(+ mlx_inference,+ "_build_grammar_constraint",+ lambda *_a, **_k: SimpleNamespace(+ allows_reasoning = allows_reasoning,+ stops_on = lambda _ids: None,+ decoded_dropping = lambda _ids = None: None,+ ),+ )+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)+ monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda _model: False)++ backend = MLXInferenceBackend()+ backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})+ backend._processor = SimpleNamespace(chat_template = "template", all_special_tokens = specials)+ args = ([{"role": "user", "content": "hi"}], [], 0.7, 0.9, 40, 0.01, 64, 1.0, None)+ return list(backend._generate_vlm(*args, response_format = {"type": "json_object"}))+++def test_a_document_only_vlm_reply_is_not_rewritten_by_the_reasoning_normalizer(monkeypatch):+ document = '{"city":"<think>Paris</think>"}'+ snapshots = _constrained_vlm_reply(+ monkeypatch,+ [document[:12], document[12:]],+ allows_reasoning = False,+ markers = ("<think>", "</think>"),+ )+ assert snapshots[-1] == document+++def test_an_allowed_vlm_reasoning_block_keeps_its_opener_when_the_closer_is_special(monkeypatch):+ snapshots = _constrained_vlm_reply(+ monkeypatch,+ ["x", "</think>", '{"a":1}'],+ allows_reasoning = True,+ markers = None,+ specials = ("</think>",),+ )+ assert snapshots[-1] == '<think>\nx</think>{"a":1}'+++def test_the_text_grammar_is_told_a_closer_survives_whenever_the_prefill_is(monkeypatch):+ """Client tools run the native decoder, which keeps a special </think>: the prefill then+ re-emits the opener, so the grammar must allow the close rather than refuse the request."""+ from core.inference import mlx_inference++ mlx_lm = types.ModuleType("mlx_lm")+ sample_utils = types.ModuleType("mlx_lm.sample_utils")+ sample_utils.make_sampler = lambda **_k: object()+ mlx_lm.sample_utils = sample_utils+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)+ monkeypatch.setattr(mlx_inference, "decoder_preserves_token", lambda *_a, **_k: True)+ seen = {}++ def _capture(*_a, **kwargs):+ seen.update(kwargs)+ return None++ monkeypatch.setattr(mlx_inference, "_build_grammar_constraint", _capture)+ backend = _budget_backend(monkeypatch)+ monkeypatch.setattr(+ backend,+ "_render_text_prompt",+ lambda *_a, **_k: SimpleNamespace(prompt = "P<think>", reasoning_channel_markers = None),+ raising = False,+ )+ list(+ backend._generate_text(+ _TEXT_TURN,+ 0.0,+ 1.0,+ 0,+ 0.0,+ 8,+ 1.0,+ None,+ tools = [{"type": "function", "function": {"name": "f"}}],+ response_format = {"type": "json_object"},+ reasoning_is_extracted = True,+ )+ )+ assert seen["reply_keeps_special_tokens"] is True++[email protected]("constrained", [False, True])+def test_a_constrained_plain_reply_is_decoded_without_space_cleanup(monkeypatch, constrained):+ from core.inference import mlx_inference++ mlx_lm = types.ModuleType("mlx_lm")+ sample_utils = types.ModuleType("mlx_lm.sample_utils")+ sample_utils.make_sampler = lambda **_k: object()+ mlx_lm.sample_utils = sample_utils+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)+ monkeypatch.setattr(+ mlx_inference,+ "_build_grammar_constraint",+ lambda *_a, **_k: SimpleNamespace(+ allows_reasoning = False, decoded_dropping = lambda _ids: None+ )+ if constrained+ else None,+ )+ backend = _budget_backend(monkeypatch)+ decodes = []++ def _decode(_ids, **kwargs):+ decodes.append(kwargs)+ return "ok"++ backend._tokenizer.decode = _decode+ monkeypatch.setattr(+ backend,+ "_render_text_prompt",+ lambda *_a, **_k: SimpleNamespace(prompt = "P", reasoning_channel_markers = None),+ raising = False,+ )+ list(+ backend._generate_text(+ _TEXT_TURN,+ 0.0,+ 1.0,+ 0,+ 0.0,+ 8,+ 1.0,+ None,+ response_format = {"type": "json_object"} if constrained else None,+ )+ )+ assert decodes+ # " ." inside a grammar-approved string must reach the client unchanged.+ for kwargs in decodes:+ assert kwargs.get("clean_up_tokenization_spaces") is (False if constrained else None)+++def test_a_special_token_the_reply_would_drop_is_refused_not_silently_deleted():+ """llguidance spells a forced literal with a special id when one matches, and a decode that+ skips special tokens would delete it from a reply reported as valid."""+ tokenizer = _char_tokenizer("ab", markers_are_special = True)+ special = tokenizer.convert_tokens_to_ids("ab")+ fmt = {"type": "json_schema", "schema": {"const": "ab"}}++ def _after_quote(constraint):+ for token in tokenizer.encode('"', add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token))+ return _allowed(constraint, len(tokenizer))++ assert special in _after_quote(+ build_constraint(fmt, tokenizer, "p", reply_keeps_special_tokens = True)+ )+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):+ _after_quote(build_constraint(fmt, tokenizer, "p"))+++def test_only_the_specials_the_decoder_drops_are_withheld_from_the_document():+ """A native decoder keeps its allowlist and drops every other special id, so the grammar is+ held to exactly that set, not to all specials or none."""+ tokenizer = _char_tokenizer("ab", "cd", markers_are_special = True)+ dropped, kept = (tokenizer.convert_tokens_to_ids(t) for t in ("ab", "cd"))++ def _after_quote(literal):+ constraint = build_constraint(+ {"type": "json_schema", "schema": {"const": literal}},+ tokenizer,+ "p",+ reply_keeps_special_tokens = True,+ )+ constraint.decoded_dropping({dropped})+ for token in tokenizer.encode('"', add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token))+ return _allowed(constraint, len(tokenizer))++ assert kept in _after_quote("cd")+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):+ _after_quote("ab")+++def test_the_text_path_hands_the_constraint_its_decoders_dropped_ids(monkeypatch):+ from core.inference import mlx_inference++ mlx_lm = types.ModuleType("mlx_lm")+ sample_utils = types.ModuleType("mlx_lm.sample_utils")+ sample_utils.make_sampler = lambda **_k: object()+ mlx_lm.sample_utils = sample_utils+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)+ seen = []+ constraint = SimpleNamespace(allows_reasoning = False, decoded_dropping = seen.append)+ monkeypatch.setattr(mlx_inference, "_build_grammar_constraint", lambda *_a, **_k: constraint)++ class _Decoder:+ def __init__(self, *_a, **_k):+ pass++ def dropped_ids(self):+ return frozenset({7})++ monkeypatch.setattr(mlx_inference, "NativeToolTokenDecoder", _Decoder)+ backend = _budget_backend(monkeypatch)+ monkeypatch.setattr(+ backend,+ "_render_text_prompt",+ lambda *_a, **_k: SimpleNamespace(prompt = "P", reasoning_channel_markers = None),+ raising = False,+ )+ run = lambda **kw: list(+ backend._generate_text(+ _TEXT_TURN,+ 0.0,+ 1.0,+ 0,+ 0.0,+ 8,+ 1.0,+ None,+ response_format = {"type": "json_object"},+ **kw,+ )+ )+ try:+ run(tools = [{"type": "function", "function": {"name": "f"}}])+ except Exception:+ pass # the stand-in decoder cannot decode; the handoff happened before generation+ assert seen and seen[0] == frozenset({7})+ seen.clear()+ run()+ assert seen == [None], "a plain skip_special_tokens decode drops every special id"+++def test_a_tokenizer_end_token_the_runtime_does_not_stop_on_is_withheld_too():+ """llguidance ends only on the runtime stops, so the tokenizer's own end token is just+ another special id the decode drops, and a literal spelled only by it is refused."""+ pytest.importorskip("mlx_lm")+ from mlx_lm.tokenizer_utils import TokenizerWrapper++ tokenizer, runtime_stop = _two_stop_tokenizer()+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])+ constraint = build_constraint(+ {"type": "json_schema", "schema": {"const": "<eos>"}}, wrapped, "p"+ )+ for token in tokenizer.encode('"', add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token))+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):+ _allowed(constraint, len(tokenizer))+++def test_the_grammar_ends_on_the_stop_the_decode_loop_names():+ """mlx-vlm stops on config ids the tokenizer may not carry; told them, the finished+ document is closed by that id rather than the tokenizer's end token."""+ tokenizer, runtime_stop = _two_stop_tokenizer()+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tokenizer, "p")+ constraint.stops_on([runtime_stop])+ for token_id in tokenizer.encode('{"a":1}', add_special_tokens = False):+ _allowed(constraint, len(tokenizer))+ constraint.advance(int(token_id))+ assert _allowed(constraint, len(tokenizer)) == {runtime_stop}diff --git a/studio/backend/tests/test_mlx_stack_blockers.py b/studio/backend/tests/test_mlx_stack_blockers.pyindex 531890da2cc..bc40d5d1bdc 100644--- a/studio/backend/tests/test_mlx_stack_blockers.py+++ b/studio/backend/tests/test_mlx_stack_blockers.py@@ -382,3 +382,10 @@ def test_uv_missing_never_marks_the_environment(monkeypatch):monkeypatch.setattr(mr, "_transformers_constraint_args", lambda: ([], None))assert mr.attempt_mlx_repair() is Falseassert mr._environment_mutated is False+++def test_a_missing_grammar_engine_names_the_package_instead_of_ignoring_the_schema(monkeypatch):+ from core.inference import grammar_constraint as gc+ monkeypatch.setattr(gc, "LLGUIDANCE_AVAILABLE", False)+ with pytest.raises(gc.ResponseFormatError, match = "llguidance"):+ gc.constraint_spec_from_response_format({"type": "json_object"})diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 1cacbb6daff..e925f213463 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -10753,6 +10753,30 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():assert payload["cache_prompt"] is False, (seed, value)+def test_a_lenient_schema_reaches_llama_server_where_it_reads_one():+ from models.inference import ChatCompletionRequest+ from routes.inference import _response_format_for_llama_server as for_llama++ schema = {"type": "integer"}+ wrapped = {"type": "json_schema", "json_schema": {"schema": schema}}+ lenient = {"type": "json_schema", "schema": schema}+ assert for_llama(lenient) == wrapped+ assert for_llama({"type": "json_schema", "json_schema": None, "schema": schema}) == wrapped+ for already in (+ {"type": "json_schema", "json_schema": {"name": "x", "schema": schema}},+ {"type": "json_schema", "json_schema": {"schema": schema}, "schema": {}},+ {"type": "json_object", "schema": schema},+ {"type": "json_object"},+ {"type": "text"},+ None,+ ):+ assert for_llama(already) is already+ request = ChatCompletionRequest(+ model = "m", messages = [{"role": "user", "content": "hi"}], response_format = lenient+ )+ assert _build_openai_passthrough_body(request)["response_format"] == wrapped++class TestPassthroughImageNormalization:@staticmethoddef _data_url(fmt: str) -> str:diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.pyindex 01d9ac6bebe..c38f8fe22ba 100644--- a/studio/backend/tests/test_passthrough_healing.py+++ b/studio/backend/tests/test_passthrough_healing.py@@ -24,6 +24,7 @@ from core.inference.passthrough_healing import ( # noqa: E402StreamToolCallHealer,heal_gate,heal_openai_message,+ nudge_enabled,nudge_messages,nudge_should_retry,response_has_promotable_calls,@@ -1630,3 +1631,40 @@ class TestClientToolSchemaTyping:)assert _healed_arguments(call % ("WebFetch", "url")) == {"url": "x", "timeout": "30"}assert _healed_arguments(call % ("Grep", "pattern")) == {"pattern": "x", "timeout": 30}+++_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}+_CONTRACT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _SCHEMA}}+++def test_a_contract_withdraws_both_post_decode_reinterpreters():+ tools = [{"type": "function", "function": {"name": "lookup"}}]+ assert heal_gate(True, tools) == {"lookup"}+ assert heal_gate(True, tools, response_format = {"type": "text"}) == {"lookup"}+ assert nudge_enabled(True, response_format = {"type": "text"}) is True+ for constraining in (_CONTRACT, {"type": "json_object"}, {"type": "text", "x": 1}, True, "x"):+ assert heal_gate(True, tools, response_format = constraining) is None, constraining+ assert nudge_enabled(True, response_format = constraining) is False, constraining+++def test_every_post_decode_reinterpreter_is_handed_the_request_s_own_contract():+ import ast+ from pathlib import Path++ names = ("heal_gate", "nudge_enabled")+ forwards = ('body.get("response_format")', "_extract_response_format(payload)")+ source = (Path(__file__).resolve().parents[1] / "routes/inference.py").read_text(+ encoding = "utf-8"+ )+ calls, missing = [], []+ for node in ast.walk(ast.parse(source)):+ if not isinstance(node, ast.Call):+ continue+ if (getattr(node.func, "attr", None) or getattr(node.func, "id", None)) not in names:+ continue+ calls.append(node)+ kwargs = {k.arg: ast.get_source_segment(source, k.value) or "" for k in node.keywords}+ if kwargs.get("response_format") not in forwards:+ missing.append(node.lineno)+ assert len(calls) >= 9, f"expected 9 {names} sites, found {len(calls)}"+ assert not missing, f"heal_gate/nudge_enabled not handed the contract at {missing}"diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.pyindex 5c7b433f2eb..d48bb3bf078 100644--- a/studio/backend/tests/test_presence_penalty.py+++ b/studio/backend/tests/test_presence_penalty.py@@ -215,6 +215,10 @@ def test_orchestrator_cmd_carries_all_sampling_params():)for key, val in _SAMPLING.items():assert cmd[key] == val, f"{key} dropped/altered in orchestrator cmd"+ rf = {"response_format": {"type": "json_object"}, "reasoning_is_extracted": True}+ with_rf = o._build_generate_cmd("req1", None, messages = [], max_new_tokens = 8, **rf)+ assert {key: with_rf[key] for key in rf} == rf+ assert "reasoning_is_extracted" not in cmddef test_worker_forwards_all_sampling_params_to_backend():@@ -226,8 +230,14 @@ def test_worker_forwards_all_sampling_params_to_backend():def __init__(self):self.received = None- def generate_chat_response(self, **kwargs):- self.received = kwargs+ def generate_chat_response(+ self,+ *,+ response_format = None,+ reasoning_is_extracted = None,+ **kwargs,+ ):+ self.received = dict(kwargs, rf = response_format, rie = reasoning_is_extracted)return iter(()) # empty stream -> loop exits, gen_done is sentclass _FakeQueue:@@ -242,6 +252,8 @@ def test_worker_forwards_all_sampling_params_to_backend():"request_id": "r","messages": [{"role": "user", "content": "hi"}],"max_new_tokens": 128,+ "response_format": {"type": "json_object"},+ "reasoning_is_extracted": True,**_SAMPLING,}backend = _RecordingBackend()@@ -250,6 +262,8 @@ def test_worker_forwards_all_sampling_params_to_backend():assert backend.received is not Nonefor key, val in _SAMPLING.items():assert backend.received[key] == val, f"{key} dropped/altered in worker gen_kwargs"+ assert backend.received["rf"] == {"type": "json_object"}+ assert backend.received["rie"] is Truedef test_orchestrator_cmd_carries_the_tool_protocol_flag():@@ -323,9 +337,10 @@ def test_the_mlx_think_prefill_predicate_matches_the_decoder_it_describes():from core.inference.mlx_inference import MLXInferenceBackendsrc = inspect.getsource(MLXInferenceBackend._generate_text)- after = src[src.index("preserves_think_close") :]+ after = src[src.index("think_close_survives = (") :]predicate = after[: after.index("decoder_preserves_token")]assert "tool_protocol_active" in predicate, "the prefill predicate ignores unrestricted mode"+ assert "preserves_think_close = think_close_survives" in srcdef test_the_mlx_vlm_decoder_survives_a_reasoning_only_request():diff --git a/studio/backend/tests/test_research_json_fallback.py b/studio/backend/tests/test_research_json_fallback.pyindex d7498ae1314..0444deec371 100644--- a/studio/backend/tests/test_research_json_fallback.py+++ b/studio/backend/tests/test_research_json_fallback.py@@ -23,7 +23,8 @@ from .test_sf_client_tools_passthrough import _ScriptedBackend, _fixed, _install_NO_GRAMMAR_ENGINE = (- "response_format needs the llama.cpp grammar engine; load a GGUF model to use it."+ "response_format needs a grammar engine, and the transformers backend has none; load an "+ "MLX or GGUF model to use it.")# Same code and param, different cause. The real-route cases keep these strings honest._AUDIO_REFUSAL_MESSAGE = (@@ -107,6 +108,10 @@ def research_call(monkeypatch):def test_local_json_research_recovers_through_the_real_route(monkeypatch, research_call, is_mlx, phase, forced_tools):+ from core.inference import grammar_constraint++ # MLX refuses only without its engine; pinned so a Mac with llguidance tests this too.+ monkeypatch.setattr(grammar_constraint, "LLGUIDANCE_AVAILABLE", False)backend = _ScriptedBackend(_fixed('{"ok": true}'))backend.models[backend.active_model_name]["is_mlx"] = is_mlx_install(monkeypatch, backend, supports_tools = forced_tools)diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex c97d5abd7b3..68d5cdd5e3f 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -130,6 +130,7 @@ def _install(backend,*,supports_tools = True,+ supports_reasoning = False,):import routes.inference as inffrom state.tool_policy import reset_tool_policy@@ -142,7 +143,10 @@ def _install(monkeypatch.setattr(inf,"_detect_safetensors_features",- lambda *a, **k: {"supports_tools": supports_tools},+ lambda *a, **k: {+ "supports_tools": supports_tools,+ "supports_reasoning": supports_reasoning,+ },)return monitor@@ -1369,6 +1373,55 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):assert backend.calls[0]["image"] is not None+_RF_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}+_RF_FORMAT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _RF_SCHEMA}}+_MARKUP_DOC = '{"city":"<think>Paris</think>"}'++[email protected]("chunked", [False, True], ids = ["whole", "streamed"])+ "prefix, prefilled, reasoning",+ [+ ("", False, ""),+ ("<think>weighing</think>", False, "weighing"),+ ("<think>weighing</think>", True, "weighing"),+ ("", True, ""),+ ],+)+def test_markers_inside_a_constrained_document_stay_in_it(chunked, prefix, prefilled, reasoning):+ from routes.inference import _ResponsesReasoningExtractor++ extractor = _ResponsesReasoningExtractor(+ parse_think_markers = True, reasoning_prefilled = prefilled, single_block = True+ )+ text = prefix + _MARKUP_DOC+ got_reasoning = got_content = ""+ for piece in text if chunked else [text]:+ delta_reasoning, delta_content = extractor.feed(piece)+ got_reasoning += delta_reasoning+ got_content += delta_content+ delta_reasoning, delta_content = extractor.finish()++ assert got_content + delta_content == _MARKUP_DOC+ assert got_reasoning + delta_reasoning == reasoning+ assert json.loads(_MARKUP_DOC)["city"] == "<think>Paris</think>"++[email protected]("reasoning", [False, True], ids = ["plain", "reasoning"])+def test_the_contract_reaches_the_backend_and_its_reply_comes_back_whole(monkeypatch, reasoning):+ pytest.importorskip("llguidance.mlx")+ backend = _ScriptedBackend(_fixed(_MARKUP_DOC))+ backend.models["sf-model"]["is_mlx"] = True+ caps = {"supports_tools": False, "supports_reasoning": reasoning}+ reply = _call(_request(response_format = _RF_FORMAT), monkeypatch, backend, **caps)+ body = _json_body(reply)+ assert backend.calls[0]["response_format"] == _RF_FORMAT+ assert backend.calls[0]["reasoning_is_extracted"] is reasoning+ message = body["choices"][0]["message"]+ assert message["content"] == _MARKUP_DOC+ assert not message.get("reasoning_content")++class _VisionToolLoopBackend(_ToolLoopBackend):def __init__(self, responder, **kwargs):super().__init__(responder, **kwargs)diff --git a/studio/backend/tests/test_think_prefill_reemit.py b/studio/backend/tests/test_think_prefill_reemit.pyindex 3ad32472aef..d05809ee974 100644--- a/studio/backend/tests/test_think_prefill_reemit.py+++ b/studio/backend/tests/test_think_prefill_reemit.py@@ -106,6 +106,14 @@ def test_guard_suppresses_when_close_tag_is_special():assert detect_think_prefill(QWEN_PROMPT + "<think>\n", specials) == ""+def test_guard_follows_the_path_s_rendering_not_the_token_metadata():+ specials = ["<|im_end|>", "<think>", "</think>"]+ assert (+ detect_think_prefill(QWEN_PROMPT + "<think>\n", specials, preserves_think_close = True)+ == "<think>\n"+ )++def test_guard_emits_when_think_not_special():specials = ["<|im_end|>", "<|endoftext|>"]assert detect_think_prefill(QWEN_PROMPT + "<think>\n", specials) == "<think>\n"diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex 10d69d618b6..581a1b1cd83 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -9745,6 +9745,8 @@ def _has_working_git() -> bool:# _MLX_INSTALL_SPECS._MLX_PINS: tuple[str, ...] = ("mlx==0.32.2", "mlx-metal==0.32.2", "mlx-lm==0.31.3")_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.1"+# Exact: llguidance.mlx / llguidance.hf are the API grammar_constraint.py binds to.+_LLGUIDANCE_PIN = "llguidance==1.8.0"_MLX_NAMES: tuple[str, ...] = tuple(spec.partition("==")[0] for spec in _MLX_PINS) + ("mlx-vlm",)@@ -11383,6 +11385,8 @@ def install_python_stack() -> int:# declared mlx-vlm range the step above honoured. Same gate, so the slot is spent on# every Apple Silicon run with torch, including the no-wheel branch.base_total += 1 # MLX stack re-resolve+ if IS_MAC_ARM:+ base_total += 1 # MLX grammar engine (step 11d), same gate as the step itselfif NO_TORCH and not skip_base:# no-torch runtime deps, which this build announces on its own slot inside the core# step rather than folding into it. Same gate as the step itself.@@ -11903,6 +11907,21 @@ def install_python_stack() -> int:# rather than no install._diffusers_main_step()+ # 11d. Apple Silicon grammar engine, outside skip_base (install.sh always skips base); failure only loses MLX response_format.+ if IS_MAC_ARM:+ if not _full_deps_requested() and _exact_distribution_spec_is_installed(_LLGUIDANCE_PIN):+ _progress("MLX grammar engine (satisfied, skipped)")+ else:+ _progress("MLX grammar engine")+ try:+ pip_install(+ "Installing the MLX grammar engine (llguidance)",+ "--no-cache-dir",+ _LLGUIDANCE_PIN,+ )+ except SystemExit:+ _note(f"{_LLGUIDANCE_PIN} failed to install; MLX response_format stays unavailable")+# 12. Patch metadata for single-env compatibility_finalize_ran = _dd_deps_ran or _dd_ran or _patch_metadata_is_pending()_progress("finalizing" if _finalize_ran else "finalizing (satisfied, skipped)")diff --git a/tests/studio/install/test_dependency_pass_skips.py b/tests/studio/install/test_dependency_pass_skips.pyindex 6888848fd71..a8eeca55f6e 100644--- a/tests/studio/install/test_dependency_pass_skips.py+++ b/tests/studio/install/test_dependency_pass_skips.py@@ -2311,3 +2311,14 @@ def test_a_temp_copy_that_cannot_be_unlinked_does_not_end_the_pass(audited, monk# The gate's own cleanup is the same shape, one step earlier: it must answer, not raise.stack._PASS_EVIDENCE = {"pass_inputs": {}, "step_results": {}}assert stack._requirements_satisfied(audited / "studio.txt", no_deps = False) is False+++def test_the_mlx_grammar_engine_is_pinned_skippable_and_never_fatal() -> None:+ """An unpinned --upgrade re-ran on every update, and its SystemExit aborted the install."""+ assert re.fullmatch(r"llguidance==\d+\.\d+\.\d+", stack._LLGUIDANCE_PIN)+ source = STACK_PATH.read_text(encoding = "utf-8")+ step = source[source.index("# 11d.") : source.index("# 12. Patch metadata")]+ assert "_exact_distribution_spec_is_installed(_LLGUIDANCE_PIN)" in step+ assert "not _full_deps_requested() and" in step+ assert "except SystemExit:" in step+ assert "--upgrade" not in step