Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: honor response_format on the MLX backend with grammar-constrained decoding (#10180)

Long Yixinga00fcbb33d68
diff --git a/studio/backend/core/inference/chat_template_helpers.py b/studio/backend/core/inference/chat_template_helpers.py
index 4dc24338af5..684d6c8e6de 100644
--- a/studio/backend/core/inference/chat_template_helpers.py
+++ b/studio/backend/core/inference/chat_template_helpers.py
@@ -2603,9 +2603,9 @@ def detect_think_prefill(
unclosed block that swallows the answer; in that case return ``""`` and fall back to plain text.
``preserves_think_close`` says the stream keeps that closer anyway, as
- ``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block. The
- special-token list then says nothing, and skipping the opener is the same bug mirrored: a stray
- ``</think>``.
+ ``NativeToolTokenDecoder`` does so the parser can see a call rehearsed inside the block, and as
+ a path streaming the detokenizer's own text does. The special-token list then says nothing, and
+ skipping the opener is the same bug mirrored: a stray ``</think>``.
"""
if not prompt:
return ""
diff --git a/studio/backend/core/inference/grammar_constraint.py b/studio/backend/core/inference/grammar_constraint.py
new file mode 100644
index 00000000000..cb80f09092a
--- /dev/null
+++ b/studio/backend/core/inference/grammar_constraint.py
@@ -0,0 +1,491 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Grammar-constrained decoding for the MLX paths, through llguidance token bitmasks."""
+
+import hashlib
+import json
+import re
+import threading
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import Optional
+
+try:
+ import llguidance as _llg
+ import llguidance.hf as _llg_hf
+ import llguidance.mlx as _llg_mlx
+
+ LLGUIDANCE_AVAILABLE = True
+ LLGUIDANCE_VERSION = str(_llg.get_version())
+except Exception:
+ _llg = None
+ _llg_hf = None
+ _llg_mlx = None
+ LLGUIDANCE_AVAILABLE = False
+ LLGUIDANCE_VERSION = None
+
+SUPPORTED_RESPONSE_FORMAT_TYPES = ("text", "json_object", "json_schema")
+
+MISSING_ENGINE_MESSAGE = (
+ "response_format needs a grammar engine, and llguidance did not load. Install or repair "
+ "it with `pip install llguidance==1.8.0`, or load a GGUF model to use llama.cpp's."
+)
+
+# json_object promises an object, not any JSON value; spelled canonically to share a grammar.
+_JSON_OBJECT_SCHEMA = '{"type":"object"}'
+
+_PRELUDE_MAX_TOKENS = 4000
+
+_THINK_MARKERS = ("<think>", "</think>")
+
+
+class ResponseFormatError(ValueError):
+ """Invalid or unhonorable ``response_format``; the message is client-safe."""
+
+ public = True
+ openai_param = "response_format"
+
+
+class GrammarDesyncError(RuntimeError):
+ """A token the mask never offered was committed; no change to the request fixes it."""
+
+
+@dataclass(frozen = True)
+class ConstraintSpec:
+ grammar: str
+ schema_json: Optional[str] = None
+ reasoning_close: Optional[str] = None
+ reasoning_close_marker: Optional[str] = None
+
+ def build(
+ self,
+ tokenizer,
+ *,
+ in_reasoning: bool = False,
+ strips_special_tokens: bool = False,
+ ) -> "GrammarConstraint":
+ if in_reasoning:
+ if self.reasoning_close is None:
+ raise ResponseFormatError(
+ "response_format cannot be honored on this model: the prompt starts "
+ "inside an open reasoning block, and its closing marker "
+ f"{self.reasoning_close_marker!r} cannot both be written by a grammar "
+ "and survive into the reply, so nothing can close the block before the "
+ "document. Send the request with thinking disabled, or load a GGUF "
+ "build of this model."
+ )
+ grammar = _cached_grammar(self.schema_json, prelude_close = self.reasoning_close)
+ else:
+ grammar = self.grammar
+ return GrammarConstraint(
+ grammar,
+ tokenizer,
+ allows_reasoning = in_reasoning,
+ strips_special_tokens = strips_special_tokens,
+ )
+
+
+def build_constraint(
+ response_format,
+ tokenizer,
+ prompt,
+ *,
+ reasoning_markers = None,
+ tools = None,
+ reasoning_is_extracted: bool = False,
+ reply_keeps_special_tokens: bool = False,
+) -> Optional["GrammarConstraint"]:
+ markers = _reasoning_markers(tokenizer, reasoning_markers, tools)
+ spec = constraint_spec_from_response_format(
+ response_format,
+ tokenizer,
+ reasoning_markers = markers,
+ reply_keeps_special_tokens = reply_keeps_special_tokens,
+ )
+ if spec is None:
+ return None
+ from core.inference.chat_template_helpers import prompt_opens_reasoning_channel
+
+ return spec.build(
+ tokenizer,
+ in_reasoning = (reasoning_is_extracted and prompt_opens_reasoning_channel(prompt, markers)),
+ strips_special_tokens = not reply_keeps_special_tokens,
+ )
+
+
+def constraint_spec_from_response_format(
+ response_format,
+ tokenizer = None,
+ *,
+ reasoning_markers = None,
+ reply_keeps_special_tokens: bool = False,
+) -> Optional[ConstraintSpec]:
+ if response_format is None:
+ return None
+ if not isinstance(response_format, dict):
+ raise ResponseFormatError("response_format must be an object with a 'type' field")
+ format_type = response_format.get("type")
+ if format_type not in SUPPORTED_RESPONSE_FORMAT_TYPES:
+ raise ResponseFormatError(
+ f"unsupported response_format type {format_type!r}; supported: "
+ + ", ".join(SUPPORTED_RESPONSE_FORMAT_TYPES)
+ )
+ if format_type == "text":
+ if response_format != {"type": "text"}:
+ raise ResponseFormatError(
+ "unsupported members on a text response_format: "
+ + ", ".join(sorted(k for k in response_format if k != "type"))
+ )
+ return None
+ if not LLGUIDANCE_AVAILABLE:
+ raise ResponseFormatError(MISSING_ENGINE_MESSAGE)
+
+ if format_type == "json_object":
+ supplied = response_format.get("schema")
+ if supplied is not None:
+ supplied = _require_schema_object(
+ supplied,
+ "response_format type 'json_object' requires a 'schema' member to be a "
+ "JSON Schema object",
+ )
+ # Absent, null and empty mean this format's own object promise, not llguidance's any-JSON `{}`.
+ schema_json = _JSON_OBJECT_SCHEMA if not supplied else _canonical_schema_json(supplied)
+ else:
+ schema_json = _canonical_schema_json(_json_schema_from_response_format(response_format))
+
+ markers = tuple(reasoning_markers) if reasoning_markers else None
+ return ConstraintSpec(
+ grammar = _cached_grammar(schema_json, prelude_close = None),
+ schema_json = schema_json,
+ reasoning_close = _marker_reference(
+ tokenizer, markers[1], reply_keeps_special_tokens = reply_keeps_special_tokens
+ )
+ if markers
+ else None,
+ reasoning_close_marker = markers[1] if markers else None,
+ )
+
+
+def _json_schema_from_response_format(response_format: dict) -> dict:
+ wrapper = response_format.get("json_schema")
+ if isinstance(wrapper, dict):
+ schema = wrapper.get("schema")
+ elif wrapper is None:
+ schema = response_format.get("schema")
+ else:
+ schema = None
+ return _require_schema_object(
+ schema,
+ "response_format type 'json_schema' requires json_schema.schema to be a "
+ "JSON Schema object",
+ )
+
+
+def _require_schema_object(schema, message: str) -> dict:
+ if not isinstance(schema, dict):
+ raise ResponseFormatError(message)
+ return schema
+
+
+def _reserved_token_text(tokenizer, token_id: int) -> str:
+ try:
+ return str(_unwrap_hf_tokenizer(tokenizer).convert_ids_to_tokens(int(token_id)))
+ except Exception:
+ return f"token {int(token_id)}"
+
+
+def _reasoning_markers(
+ tokenizer,
+ markers = None,
+ tools = None,
+) -> Optional[tuple]:
+ if markers:
+ return tuple(markers)
+ if tokenizer is None:
+ return None
+ from core.inference.chat_template_helpers import detect_reasoning_channel_markers
+
+ try:
+ return detect_reasoning_channel_markers(tokenizer, tools = tools) or _THINK_MARKERS
+ except Exception:
+ return None
+
+
+def _marker_reference(tokenizer, marker: str, *, reply_keeps_special_tokens: bool) -> Optional[str]:
+ if tokenizer is None:
+ return None
+ inner = _unwrap_hf_tokenizer(tokenizer)
+ try:
+ ids = inner.encode(marker, add_special_tokens = False)
+ if len(ids) != 1:
+ return None
+ survives = reply_keeps_special_tokens or (
+ inner.decode(ids, skip_special_tokens = True) == marker
+ )
+ except Exception:
+ return None
+ return marker if survives else None
+
+
+class GrammarConstraint:
+ def __init__(
+ self,
+ grammar: str,
+ tokenizer,
+ *,
+ allows_reasoning: bool = False,
+ strips_special_tokens: bool = False,
+ ):
+ self._grammar = grammar
+ self._tokenizer = tokenizer
+ self._matcher = None
+ self._bitmask = None
+ self._stop_ids = ()
+ self._stripped_ids = ()
+ self._strips_special_tokens = strips_special_tokens
+ self._dropped_ids = None
+ self._runtime_stops = None
+ self.allows_reasoning = allows_reasoning
+
+ def _bind(self, n_vocab: int) -> None:
+ self._stop_ids = (
+ tuple(sorted({int(i) for i in self._runtime_stops if 0 <= int(i) < int(n_vocab)}))
+ if self._runtime_stops
+ else _runtime_stop_ids(self._tokenizer, n_vocab) or ()
+ )
+ if self._strips_special_tokens:
+ # llguidance may spell a literal with a special id that this decode would delete.
+ self._stripped_ids = _stripped_special_ids(
+ self._tokenizer, n_vocab, self._stop_ids, self._dropped_ids
+ )
+ ll_tokenizer = _cached_ll_tokenizer(self._tokenizer, n_vocab, self._stop_ids or None)
+ matcher = _llg.LLMatcher(ll_tokenizer, self._grammar)
+ error = matcher.get_error()
+ if error:
+ raise ResponseFormatError(f"response_format grammar rejected: {error}")
+ self._matcher = matcher
+ self._bitmask = _llg_mlx.allocate_token_bitmask(1, n_vocab)
+
+ def mask_logits(self, logits):
+ if self._matcher is None:
+ self._bind(int(logits.shape[-1]))
+ _llg_mlx.fill_next_token_bitmask(self._matcher, self._bitmask)
+ forced_stop = self._stop_offered_as_text()
+ if forced_stop is not None:
+ raise ResponseFormatError(self._desync_message(forced_stop))
+ self._forbid_stripped_ids()
+ masked = _llg_mlx.apply_token_bitmask(logits.reshape(1, -1), self._bitmask)
+ return masked.reshape(logits.shape)
+
+ def stops_on(self, ids) -> None:
+ """The ids the decode loop really ends on, when the tokenizer alone cannot say."""
+ self._runtime_stops = tuple(ids or ()) or None
+
+ def decoded_dropping(self, ids = None) -> None:
+ """The special ids the reply's decoder removes; None means every special id."""
+ self._strips_special_tokens = True
+ self._dropped_ids = None if ids is None else frozenset(int(i) for i in ids)
+
+ def _forbid_stripped_ids(self) -> None:
+ offered = [i for i in self._stripped_ids if self._mask_allowed(i)]
+ if not offered:
+ return
+ import numpy as np
+
+ words = self._bitmask.view(np.uint32)
+ for token_id in offered:
+ words[0, token_id // 32] &= np.uint32(~(1 << (token_id % 32)) & 0xFFFFFFFF)
+ if not words[0].any():
+ raise ResponseFormatError(self._desync_message(offered[0]))
+
+ def _stop_offered_as_text(self) -> Optional[int]:
+ if not self._stop_ids or self._matcher.is_stopped() or self._matcher.is_accepting():
+ return None
+ return next((i for i in self._stop_ids if self._mask_allowed(i)), None)
+
+ def advance(self, token_id: int) -> None:
+ if self._matcher is None:
+ raise GrammarDesyncError("guided decoding advanced before the grammar was bound")
+ if self._matcher.is_stopped():
+ return
+ self._matcher.consume_token(int(token_id))
+ error = self._matcher.get_error()
+ if error:
+ if self._mask_allowed(int(token_id)):
+ raise ResponseFormatError(self._desync_message(int(token_id)))
+ raise GrammarDesyncError(f"guided decoding desynced on token {int(token_id)}: {error}")
+
+ def _mask_allowed(self, token_id: int) -> bool:
+ try:
+ import numpy as np
+ word = int(np.asarray(self._bitmask)[0, token_id // 32])
+ except Exception:
+ return False
+ return bool(word >> (token_id % 32) & 1)
+
+ def _desync_message(self, token_id: int) -> str:
+ reserved = _reserved_token_text(self._tokenizer, token_id)
+ return (
+ f"response_format could not be honored: the reply reached a point where the "
+ f"schema allows only the text {reserved!r}, which this model reserves as a "
+ "control token and no grammar can spell as ordinary text. Remove that value "
+ "from the schema."
+ )
+
+
+def make_grammar_logits_processor(constraint: GrammarConstraint):
+ state = {"seen": None}
+
+ def _processor(tokens, logits):
+ seen = state["seen"]
+ if seen is None:
+ state["seen"] = int(tokens.shape[0])
+ else:
+ for token_id in tokens[seen:].tolist():
+ constraint.advance(int(token_id))
+ state["seen"] = int(tokens.shape[0])
+ return constraint.mask_logits(logits)
+
+ return _processor
+
+
+# Bounded, strong-reference caches, so a recycled id() cannot alias a live entry.
+
+_GRAMMAR_CACHE: "OrderedDict[str, str]" = OrderedDict()
+_GRAMMAR_CACHE_MAX = 64
+_GRAMMAR_CACHE_MAX_BYTES = 8 << 20
+_TOKENIZER_CACHE: "OrderedDict[tuple, tuple]" = OrderedDict()
+_TOKENIZER_CACHE_MAX = 4
+_CACHE_LOCK = threading.Lock()
+
+
+def _canonical_schema_json(schema: dict) -> str:
+ try:
+ return json.dumps(schema, sort_keys = True, separators = (",", ":"))
+ except (TypeError, ValueError) as exc:
+ raise ResponseFormatError(f"response_format schema is not JSON-serializable: {exc}")
+
+
+def _cached_grammar(schema_json: str, *, prelude_close: Optional[str]) -> str:
+ digest = hashlib.sha256(schema_json.encode("utf-8")).hexdigest()
+ key = f"{LLGUIDANCE_VERSION}:{prelude_close or ''}:" f"{_PRELUDE_MAX_TOKENS}:{digest}"
+ with _CACHE_LOCK:
+ cached = _GRAMMAR_CACHE.get(key)
+ if cached is not None:
+ _GRAMMAR_CACHE.move_to_end(key)
+ return cached
+ grammar = _compile_grammar(schema_json, prelude_close)
+ with _CACHE_LOCK:
+ _GRAMMAR_CACHE[key] = grammar
+ cached_bytes = sum(len(value) for value in _GRAMMAR_CACHE.values())
+ while _GRAMMAR_CACHE and (
+ len(_GRAMMAR_CACHE) > _GRAMMAR_CACHE_MAX or cached_bytes > _GRAMMAR_CACHE_MAX_BYTES
+ ):
+ _, evicted = _GRAMMAR_CACHE.popitem(last = False)
+ cached_bytes -= len(evicted)
+ return grammar
+
+
+def _compile_grammar(schema_json: str, prelude_close: Optional[str]) -> str:
+ try:
+ if prelude_close is None:
+ grammar = _llg.LLMatcher.grammar_from_json_schema(schema_json)
+ else:
+ # No textual closer in reasoning (the route splits there); a {0,N} bound here exhausts llguidance's lexer.
+ closer_rx = re.escape(prelude_close).replace("/", "\\/")
+ grammar = (
+ "%llguidance {}\n"
+ "start: prelude doc\n"
+ f"prelude: prelude_text {prelude_close}\n"
+ f"prelude_text[max_tokens={_PRELUDE_MAX_TOKENS}]: "
+ f"/(.|\\n)*/ & ~/(.|\\n)*{closer_rx}(.|\\n)*/\n"
+ f"doc: %json {schema_json}\n"
+ )
+ except Exception as exc:
+ raise ResponseFormatError(f"unsupported JSON Schema: {exc}") from exc
+ error = _llg.LLMatcher.validate_grammar(grammar)
+ if error:
+ raise ResponseFormatError(f"unsupported JSON Schema: {error}")
+ return grammar
+
+
+def _unwrap_hf_tokenizer(tokenizer):
+ import transformers
+
+ if isinstance(tokenizer, transformers.PreTrainedTokenizerFast):
+ return tokenizer
+ inner = getattr(tokenizer, "_tokenizer", None)
+ if isinstance(inner, transformers.PreTrainedTokenizerFast):
+ return inner
+ inner = getattr(tokenizer, "tokenizer", None)
+ if isinstance(inner, transformers.PreTrainedTokenizerFast):
+ return inner
+ return tokenizer
+
+
+def _stripped_special_ids(
+ tokenizer,
+ n_vocab: int,
+ stop_ids,
+ dropped = None,
+) -> tuple:
+ inner = _unwrap_hf_tokenizer(tokenizer)
+ if dropped is not None:
+ ids = set(dropped)
+ else:
+ # skip_special_tokens drops every added special token, not just all_special_ids.
+ try:
+ ids = set(getattr(inner, "all_special_ids", None) or ())
+ ids.update(
+ i
+ for i, t in (getattr(inner, "added_tokens_decoder", None) or {}).items()
+ if t.special
+ )
+ except Exception:
+ return ()
+ # llguidance ends on the runtime stops, or on the tokenizer's own end token when there are none.
+ keep = set(stop_ids) if stop_ids else {getattr(inner, "eos_token_id", None)}
+ return tuple(sorted({int(i) for i in ids if 0 <= int(i) < int(n_vocab)} - keep))
+
+
+def _runtime_stop_ids(tokenizer, n_vocab: int) -> Optional[tuple]:
+ for obj in (tokenizer, getattr(tokenizer, "tokenizer", None)):
+ if obj is None:
+ continue
+ for ids in (
+ getattr(obj, "eos_token_ids", None),
+ getattr(getattr(obj, "stopping_criteria", None), "eos_token_ids", None),
+ ):
+ try:
+ found = tuple(sorted({int(i) for i in ids if 0 <= int(i) < int(n_vocab)}))
+ except Exception:
+ continue
+ if found:
+ return found
+ return None
+
+
+def _cached_ll_tokenizer(
+ tokenizer,
+ n_vocab: int,
+ stop_ids = None,
+):
+ stop_ids = stop_ids or _runtime_stop_ids(tokenizer, n_vocab)
+ tokenizer = _unwrap_hf_tokenizer(tokenizer)
+ key = (id(tokenizer), int(n_vocab), stop_ids)
+ with _CACHE_LOCK:
+ entry = _TOKENIZER_CACHE.get(key)
+ if entry is not None and entry[0] is tokenizer:
+ _TOKENIZER_CACHE.move_to_end(key)
+ return entry[1]
+ ll_tokenizer = _llg_hf.from_tokenizer(
+ tokenizer,
+ n_vocab = int(n_vocab),
+ eos_token = list(stop_ids) if stop_ids else None,
+ )
+ with _CACHE_LOCK:
+ _TOKENIZER_CACHE[key] = (tokenizer, ll_tokenizer)
+ while len(_TOKENIZER_CACHE) > _TOKENIZER_CACHE_MAX:
+ _TOKENIZER_CACHE.popitem(last = False)
+ return ll_tokenizer
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index ef1e5bf1d3a..20b3f560d48 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2311,12 +2311,52 @@ def _make_mlx_logit_bias_processor(logit_bias: dict):
return _processor
+def _vlm_generation_is_diffusion(model) -> bool:
+ try:
+ from mlx_vlm.generate.diffusion import is_diffusion_model
+ except Exception:
+ return False
+ try:
+ return bool(is_diffusion_model(model))
+ except Exception:
+ return False
+
+
+def _build_grammar_constraint(
+ response_format,
+ tokenizer,
+ prompt,
+ *,
+ reasoning_markers = None,
+ tools = None,
+ reasoning_is_extracted = False,
+ reply_keeps_special_tokens = False,
+):
+ if response_format is None or response_format == {"type": "text"}:
+ return None
+ from core.inference.grammar_constraint import build_constraint
+
+ constraint = build_constraint(
+ response_format,
+ tokenizer,
+ prompt,
+ reasoning_markers = reasoning_markers,
+ tools = tools,
+ reasoning_is_extracted = reasoning_is_extracted,
+ reply_keeps_special_tokens = reply_keeps_special_tokens,
+ )
+ if constraint is not None:
+ logger.info("Guided decoding active: response_format=%s", response_format.get("type"))
+ return constraint
+
+
def _mlx_sampling_processors(
*,
repetition_penalty = None,
presence_penalty: float = 0.0,
frequency_penalty: float = 0.0,
logit_bias = None,
+ grammar_constraint = None,
):
"""Logits processors for the sampling knobs, or ``None`` when all are inert. Bias runs before
the penalties, matching llama-server's sampler order. mlx_lm supplies only the repetition
@@ -2324,6 +2364,9 @@ def _mlx_sampling_processors(
prompt*, while the penalties below score the whole completion and exclude it, so using them
would make the same request sample differently depending on the backend."""
processors = []
+ if grammar_constraint is not None:
+ from core.inference.grammar_constraint import make_grammar_logits_processor
+ processors.append(make_grammar_logits_processor(grammar_constraint))
if logit_bias:
processors.append(_make_mlx_logit_bias_processor(logit_bias))
if repetition_penalty is not None and float(repetition_penalty) not in (0.0, 1.0):
@@ -3246,6 +3289,8 @@ class MLXInferenceBackend:
frequency_penalty = 0.0,
logit_bias = None,
stop = None,
+ response_format = None,
+ reasoning_is_extracted = False,
_adapter_state = None,
# Unrestricted mode runs the tool protocol with an EMPTY tools list, so bool(tools)
# cannot tell that the wrappers below still have to survive decoding.
@@ -3336,6 +3381,8 @@ class MLXInferenceBackend:
_adapter_state = _adapter_state,
stop = stop,
tool_protocol_active = tool_protocol_active,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
video = video,
)
else:
@@ -3360,6 +3407,8 @@ class MLXInferenceBackend:
_adapter_state = _adapter_state,
stop = stop,
tool_protocol_active = tool_protocol_active,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
)
yield from stream
@@ -3391,6 +3440,8 @@ class MLXInferenceBackend:
_adapter_state = None,
tool_protocol_active = None,
stop = None,
+ response_format = None,
+ reasoning_is_extracted = False,
):
from mlx_lm import stream_generate
from mlx_lm.sample_utils import make_sampler
@@ -3410,20 +3461,32 @@ class MLXInferenceBackend:
# Not the request flag: a later tool-loop pass keeps it but renders an ordinary post-tool prompt.
_resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))
+ preserve_native_channels = reasoning_channel_markers is not None
# An open <think> prefilled by the template lives in the prompt, not the generated tokens; re-emit it so the
# frontend renders the block.
+ # Matches native_token_decoder below: when it runs </think> survives, so re-emit the opener.
+ think_close_survives = (
+ bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
+ ) and decoder_preserves_token(
+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
+ )
think_prefix = detect_think_prefill(
prompt,
getattr(self._tokenizer, "all_special_tokens", None),
- # Matches native_token_decoder below: when it runs </think> survives, so the
- # prefilled opener has to be re-emitted with it.
- preserves_think_close = (
- bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
- )
- and decoder_preserves_token(
- self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
- ),
+ preserves_think_close = think_close_survives,
)
+ constraint = _build_grammar_constraint(
+ response_format,
+ self._tokenizer,
+ prompt,
+ reasoning_markers = reasoning_channel_markers,
+ reasoning_is_extracted = reasoning_is_extracted,
+ # Must match the prefill's answer, or a closer it re-emits would be refused.
+ reply_keeps_special_tokens = preserve_native_channels or think_close_survives,
+ )
+ if constraint is not None and not constraint.allows_reasoning:
+ think_prefix = ""
+ preserve_native_channels = False
if seed is None:
sampler = make_sampler(
temp = temperature,
@@ -3445,9 +3508,9 @@ class MLXInferenceBackend:
presence_penalty = presence_penalty,
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
+ grammar_constraint = constraint,
)
- preserve_native_channels = reasoning_channel_markers is not None
native_token_decoder = (
NativeToolTokenDecoder(
self._tokenizer,
@@ -3456,6 +3519,10 @@ class MLXInferenceBackend:
if tools or preserve_native_channels or tool_protocol_active
else None
)
+ if constraint is not None:
+ constraint.decoded_dropping(
+ native_token_decoder.dropped_ids() if native_token_decoder is not None else None
+ )
# Consulted per token on the reasoning path below, so resolved once here.
stop_token_ids = (
_mlx_stop_token_ids(self._tokenizer, self._model)
@@ -3480,7 +3547,7 @@ class MLXInferenceBackend:
prompt, reasoning_channel_markers, _resumed_partial
),
)
- if reasoning_channel_markers is not None
+ if preserve_native_channels
else None
)
# Sequences match the sampled text, ahead of the prefill this path restores and the <think> rewriting below:
@@ -3579,6 +3646,12 @@ class MLXInferenceBackend:
sampled = self._tokenizer.decode(
token_ids,
skip_special_tokens = True,
+ # Cleanup would rewrite " ." inside a grammar-approved string.
+ **(
+ {"clean_up_tokenization_spaces": False}
+ if constraint is not None
+ else {}
+ ),
)
if not sequences:
yield think_prefix + sampled
@@ -3811,6 +3884,8 @@ class MLXInferenceBackend:
tool_protocol_active = None,
stop = None,
video = None,
+ response_format = None,
+ reasoning_is_extracted = False,
):
from mlx_vlm import stream_generate as vlm_stream
@@ -3830,7 +3905,6 @@ class MLXInferenceBackend:
# Detected once: the decoder keeps the delimiters the normalizer below consumes.
vlm_reasoning_markers = detect_reasoning_channel_markers(chat_target, tools = tools)
- # Re-emit an open <think> prefill from the prompt (see _generate_text).
prefill = detect_think_prefill(
prompt,
getattr(chat_target, "all_special_tokens", None),
@@ -3885,7 +3959,30 @@ class MLXInferenceBackend:
0.0,
1.0,
)
- if presence_penalty or frequency_penalty or logit_bias:
+ constraint = _build_grammar_constraint(
+ response_format,
+ chat_target,
+ prompt,
+ reasoning_markers = vlm_reasoning_markers,
+ tools = tools,
+ reasoning_is_extracted = reasoning_is_extracted,
+ reply_keeps_special_tokens = True,
+ )
+ document_only = constraint is not None and not constraint.allows_reasoning
+ if document_only:
+ prefill = ""
+ elif constraint is not None:
+ # The grammar lets a kept </think> close the block: re-emit the opener or it all reads as content.
+ prefill = detect_think_prefill(prompt, preserves_think_close = True)
+ if constraint is not None and _vlm_generation_is_diffusion(self._model):
+ from core.inference.grammar_constraint import ResponseFormatError
+ raise ResponseFormatError(
+ "response_format is not supported on this model: it generates by "
+ "diffusion rather than one token at a time, so no grammar can "
+ "constrain the next token. Load an autoregressive model to use "
+ "guided decoding."
+ )
+ if presence_penalty or frequency_penalty or logit_bias or constraint is not None:
# These need custom processors: pass the full list (repetition + the rest) instead of the
# repetition_penalty shortcut so all apply.
vlm_kwargs["logits_processors"] = _mlx_sampling_processors(
@@ -3893,6 +3990,7 @@ class MLXInferenceBackend:
presence_penalty = presence_penalty,
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
+ grammar_constraint = constraint,
)
elif _rep_active:
vlm_kwargs["repetition_penalty"] = float(repetition_penalty)
@@ -3909,6 +4007,11 @@ class MLXInferenceBackend:
and self._tokenizer
else None
)
+ if constraint is not None:
+ # mlx-vlm stops on the config's ids, which some repos set apart from the tokenizer's.
+ constraint.stops_on(_mlx_stop_token_ids(self._tokenizer, self._model))
+ if constraint is not None and vlm_token_decoder is not None:
+ constraint.decoded_dropping(vlm_token_decoder.dropped_ids())
# The runtime EOS can itself be an allowlisted control, and this path appends every
# decoded token to the snapshot, so it would trail each answer. As in _generate_text.
vlm_stop_ids = (
@@ -4059,16 +4162,20 @@ class MLXInferenceBackend:
),
)
- yield from normalize_reasoning_snapshots(
- _stream_vlm_snapshots(),
- chat_target,
- cancel_event,
- markers = vlm_reasoning_markers,
- tools = tools,
- prompt = prompt,
- continued = vlm_continued,
- ended = lambda: stopped,
- )
+ if document_only:
+ # The normalizer would rewrite marker text inside the document.
+ yield from _stream_vlm_snapshots()
+ else:
+ yield from normalize_reasoning_snapshots(
+ _stream_vlm_snapshots(),
+ chat_target,
+ cancel_event,
+ markers = vlm_reasoning_markers,
+ tools = tools,
+ prompt = prompt,
+ continued = vlm_continued,
+ ended = lambda: stopped,
+ )
if stopped:
self._mark_stopped()
diff --git a/studio/backend/core/inference/native_tool_tokens.py b/studio/backend/core/inference/native_tool_tokens.py
index acea3498911..a5117bc6155 100644
--- a/studio/backend/core/inference/native_tool_tokens.py
+++ b/studio/backend/core/inference/native_tool_tokens.py
@@ -258,6 +258,10 @@ class NativeToolTokenDecoder:
self._tokenizer = tokenizer
self._special_ids, self._tool_ids = _special_token_sets(tokenizer, preserved_tokens)
+ def dropped_ids(self) -> "frozenset[int] | None":
+ """The special ids this decoder removes, or None when it falls back to dropping all."""
+ return (self._special_ids - self._tool_ids) if self._special_ids else None
+
def preserves(self, token: str) -> bool:
"""Whether this decoder really keeps ``token``, which the allowlist does not settle:
with no usable ``all_special_ids`` every decode falls back to
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index bf5366210bc..815532208be 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -258,32 +258,42 @@ class GenStreamError(str):
from model output whose visible text starts with "Error:" by checking isinstance(chunk,
GenStreamError)."""
- __slots__ = ("public",)
+ __slots__ = ("public", "openai_param")
def __new__(
cls,
value,
*,
public: bool = False,
+ openai_param: Optional[str] = None,
):
obj = str.__new__(cls, value)
obj.public = bool(public)
+ # Set for a refusal about one request field, so the caller answers 400 not 500.
+ obj.openai_param = openai_param
return obj
class GenStreamErrorRaised(RuntimeError):
"""Internal exception form of ``GenStreamError`` for generator boundaries."""
- __slots__ = ("public",)
+ __slots__ = ("public", "openai_param")
def __init__(
self,
value,
*,
public: bool = False,
+ openai_param: Optional[str] = None,
):
super().__init__(value)
self.public = bool(public)
+ self.openai_param = openai_param
+
+ @classmethod
+ def from_chunk(cls, chunk: "GenStreamError") -> "GenStreamErrorRaised":
+ """Keeps public/openai_param so a refusal is not answered 500."""
+ return cls(str(chunk), public = chunk.public, openai_param = chunk.openai_param)
def _summed_tool_loop_stats(total, turn):
@@ -1302,6 +1312,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video_b64: Optional[str] = None,
+ response_format: Optional[dict] = None,
+ reasoning_is_extracted: bool = False,
) -> dict:
"""Build the 'generate' command shared by the locked and dispatched paths."""
cmd = {
@@ -1326,6 +1338,9 @@ class InferenceOrchestrator:
cmd["seed"] = seed
if stop:
cmd["stop"] = stop
+ if response_format is not None:
+ cmd["response_format"] = response_format
+ cmd["reasoning_is_extracted"] = bool(reasoning_is_extracted)
if video_b64:
cmd["video_base64"] = video_b64
if use_adapter is not None:
@@ -1410,7 +1425,11 @@ class InferenceOrchestrator:
stats_holder["stats"] = resp.get("stats")
return
elif rtype == "gen_error":
- yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")
+ yield GenStreamError(
+ f"Error: {resp.get('error', 'Unknown error')}",
+ public = bool(resp.get("public", False)),
+ openai_param = resp.get("openai_param"),
+ )
return
def _start_dispatcher(self) -> bool:
@@ -1541,6 +1560,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ response_format: Optional[dict] = None,
+ reasoning_is_extracted: bool = False,
) -> Generator[str, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
@@ -1596,6 +1617,8 @@ class InferenceOrchestrator:
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
stop = stop,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
use_adapter = use_adapter,
tools = tools,
enable_thinking = enable_thinking,
@@ -2385,6 +2408,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ response_format: Optional[dict] = None,
+ reasoning_is_extracted: bool = False,
) -> Generator[str, None, None]:
"""Generate response, streaming tokens from subprocess. ``tools`` / ``enable_thinking`` /
``reasoning_effort`` / ``preserve_thinking`` are forwarded so the template can render
@@ -2418,6 +2443,8 @@ class InferenceOrchestrator:
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
stop = stop,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
video = video,
)
@@ -2530,7 +2557,7 @@ class InferenceOrchestrator:
for chunk in stream:
if isinstance(chunk, GenStreamError):
close_stream = True
- raise GenStreamErrorRaised(str(chunk), public = chunk.public)
+ raise GenStreamErrorRaised.from_chunk(chunk)
yield chunk
finally:
if close_stream:
@@ -2621,9 +2648,7 @@ class InferenceOrchestrator:
try:
for chunk in stream:
if isinstance(chunk, GenStreamError):
- # Preserve the public/operational flag so the route can surface the real message (e.g. "model is
- # being unloaded") instead of a generic error. Mirrors the safetensors tool loop's _single_turn.
- raise GenStreamErrorRaised(str(chunk), public = chunk.public)
+ raise GenStreamErrorRaised.from_chunk(chunk)
yield chunk
finally:
close = getattr(stream, "close", None)
@@ -2658,6 +2683,8 @@ class InferenceOrchestrator:
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
video: Optional[str] = None,
+ response_format: Optional[dict] = None,
+ reasoning_is_extracted: bool = False,
) -> Generator[str, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
@@ -2701,6 +2728,8 @@ class InferenceOrchestrator:
frequency_penalty = frequency_penalty,
logit_bias = logit_bias,
stop = stop,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
use_adapter = use_adapter,
tools = tools,
enable_thinking = enable_thinking,
diff --git a/studio/backend/core/inference/passthrough_healing.py b/studio/backend/core/inference/passthrough_healing.py
index 20c6275b4bc..281d790a893 100644
--- a/studio/backend/core/inference/passthrough_healing.py
+++ b/studio/backend/core/inference/passthrough_healing.py
@@ -18,8 +18,9 @@ promotes calls whose function name exactly matches a declared tool. Promotion
removes EXACTLY the promoted calls' markup spans (the parser reports them):
undeclared calls, unparseable blocks, and suppressed alternate formats keep
every byte and relay as text, so healing can never silently delete model
-output. Responses without a tool signal, requests without tools, and Unsloth's
-own enable-tools loop are untouched. Per-request opt-out:
+output. Responses without a tool signal and requests without tools are untouched, as is
+any request whose ``response_format`` constrains decoding: a grammar has already fixed what
+the reply means. Per-request opt-out:
``auto_heal_tool_calls: false``. Process kill-switch:
``UNSLOTH_DISABLE_TOOL_CALL_HEALING=1``.
"""
@@ -59,7 +60,13 @@ _HEALING_DISABLED = os.environ.get("UNSLOTH_DISABLE_TOOL_CALL_HEALING", "0") ==
_NUDGE_DEFAULT = os.environ.get("UNSLOTH_TOOL_CALL_NUDGE", "0") == "1"
-def nudge_enabled(request_flag: Optional[bool]) -> bool:
+def response_format_constrains_decoding(response_format: Any) -> bool:
+ return response_format is not None and response_format != {"type": "text"}
+
+
+def nudge_enabled(request_flag: Optional[bool], *, response_format: Any = None) -> bool:
+ if response_format_constrains_decoding(response_format):
+ return False
return _NUDGE_DEFAULT if request_flag is None else bool(request_flag)
@@ -73,6 +80,8 @@ def heal_gate(
auto_heal: Optional[bool],
tools: Optional[list],
tool_choice: Any = None,
+ *,
+ response_format: Any = None,
) -> Optional[set]:
"""Return the declared client-tool name set when healing applies, else None.
@@ -86,11 +95,15 @@ def heal_gate(
markup stays text), and a forced ``{"type": "function", "function":
{"name": N}}`` narrows promotion to that one function. ``"auto"`` /
``"required"`` / absent keep the full declared set.
+
+ A constraining ``response_format`` disables healing: schema strings resembling call markup would be promoted away.
"""
if _HEALING_DISABLED or auto_heal is False:
return None
if tool_choice == "none":
return None
+ if response_format_constrains_decoding(response_format):
+ return None
names = set()
for tool in tools or []:
if not isinstance(tool, dict):
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 9a61c6c54b4..f45dd68e249 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -753,7 +753,15 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
# backend's documented "ignores them" behavior into a TypeError.
# ``tool_protocol_active`` rides here rather than above: MLX declares no such
# parameter and takes no **kwargs, so an unconditional forward would raise.
- for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):
+ for gated in (
+ "seed",
+ "frequency_penalty",
+ "logit_bias",
+ "stop",
+ "tool_protocol_active",
+ "response_format",
+ "reasoning_is_extracted",
+ ):
if gated in cmd and _backend_declares(backend, gated):
gen_kwargs[gated] = cmd[gated]
# A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.
@@ -813,6 +821,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
"type": "gen_error",
"request_id": request_id,
"error": str(exc),
+ # Client-safe refusals would otherwise reach the caller as a generic 500.
+ "public": bool(getattr(exc, "public", False)),
+ "openai_param": getattr(exc, "openai_param", None),
"stack": traceback.format_exc(limit = 20),
},
)
@@ -1054,6 +1065,8 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
"type": "gen_error",
"request_id": request_id,
"error": str(exc),
+ "public": bool(getattr(exc, "public", False)),
+ "openai_param": getattr(exc, "openai_param", None),
"stack": traceback.format_exc(limit = 20),
},
)
diff --git a/studio/backend/core/research_runs.py b/studio/backend/core/research_runs.py
index 97645da309d..84c907edbeb 100644
--- a/studio/backend/core/research_runs.py
+++ b/studio/backend/core/research_runs.py
@@ -126,7 +126,8 @@ _MODEL_WAIT_POLL_SECONDS = 2.0
# A model that keeps disappearing would re-send forever, so cap how many times one call may wait.
_MAX_MODEL_WAITS = 3
_NO_MODEL_LOADED_DETAIL = "No model loaded"
-_NO_GRAMMAR_ENGINE_DETAIL = "needs the llama.cpp grammar engine"
+# Every "no grammar engine here" refusal says this (routes, grammar_constraint); a re-send fixes those only.
+_NO_GRAMMAR_ENGINE_DETAIL = "needs a grammar engine"
# routes.inference reports the same unloaded state this way when auto-switch finds no local match.
_MODEL_NOT_FOUND_CODE = "model_not_found"
# routes.inference 503s with this while an auto-switch to the run's model is still loading.
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7ffcb492a9f..095091e7a60 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -237,6 +237,11 @@ _install_httpcore_asyncgen_silencer()
_STATUS_PROBE_EXECUTOR = ThreadPoolExecutor(max_workers = 2, thread_name_prefix = "inference-status")
+_SCHEMA_VALIDATION_EXECUTOR = ThreadPoolExecutor(
+ max_workers = 2, thread_name_prefix = "inference-response-format"
+)
+
+
# Lease waiters must not consume default workers needed by the current holder.
_OLLAMA_LEASE_EXECUTOR = ThreadPoolExecutor(
max_workers = 2, thread_name_prefix = "inference-ollama-lease"
@@ -340,6 +345,11 @@ def _friendly_gen_stream_error(value) -> str:
return safe_error_detail(RuntimeError(text), fallback = "An internal error occurred.")
+def _refused_parameter(value) -> Optional[str]:
+ param = getattr(value, "openai_param", None)
+ return param if getattr(value, "public", False) and param else None
+
+
def _friendly_upstream_error(text: str) -> str:
"""Rewrite a raw llama-server error body into an actionable message where we can.
@@ -25524,11 +25534,7 @@ async def produce_openai_chat_completions(
or bool(payload.openai_code_exec_container_id)
or bool(payload.anthropic_code_exec_container_id)
# A JSON-schema response_format is guided-decoding structured output the
- # router forwards to the llama-server passthrough, not Unsloth's tool
- # loop, so a --enable-tools policy must not 400 it as a local-confirm
- # request under ask/auto. Read with the predicate the router itself
- # uses, so this gate cannot admit a request the router then rejects
- # after a model switch, nor reject one the router would have served.
+ # router answers with a grammar engine; read with the router's own predicate.
or _response_format_constrains_decoding(payload)
)
# permission_mode only implies the confirm gate for that local loop.
@@ -25805,15 +25811,38 @@ async def produce_openai_chat_completions(
_raise_unsupported_n("streaming chat completions")
model_info = backend.models.get(backend.active_model_name, {})
if _response_format_constrains_decoding(payload):
- if model_info.get("is_audio") and model_info.get("audio_type") != "whisper":
+ _audio_reply = model_info.get("is_audio") and model_info.get("audio_type") != "whisper"
+ _transcribes = model_info.get("audio_type") == "whisper"
+ _listens = payload.audio_base64 and model_info.get("has_audio_input")
+ if _audio_reply or _transcribes or _listens:
_raise_unsupported_openai_parameter(
"response_format",
- "response_format cannot be honored by an audio reply; send the request to a text model "
- "to use guided decoding.",
+ "response_format is not supported when the reply comes from an audio "
+ "route: a waveform admits no grammar, and transcription and audio-input "
+ "answers are decoded without one. Send the request to a text model.",
)
- _raise_unsupported_openai_parameter(
- "response_format",
- "response_format needs the llama.cpp grammar engine; load a GGUF model to use it.",
+ if not model_info.get("is_mlx"):
+ _raise_unsupported_openai_parameter(
+ "response_format",
+ "response_format needs a grammar engine, and the transformers backend "
+ "has none; load an MLX or GGUF model to use it.",
+ )
+ if payload.tool_choice not in (None, "auto", "none"):
+ _raise_unsupported_openai_parameter(
+ "tool_choice",
+ "response_format cannot be combined with a tool_choice that requires a "
+ "call: guided decoding admits only the document, so no tool would be "
+ "called. Use tool_choice 'auto' or 'none'.",
+ )
+ if _continue_final_message(payload):
+ _raise_unsupported_openai_parameter(
+ "response_format",
+ "response_format cannot resume a partial assistant message: guided "
+ "decoding starts a new document rather than continuing the text before "
+ "it. Send the turn without continue_final_message.",
+ )
+ await asyncio.get_running_loop().run_in_executor(
+ _SCHEMA_VALIDATION_EXECUTOR, _reject_unhonorable_response_format, payload
)
# ── Audio TTS path: auto-route to audio generation ────
@@ -28287,6 +28316,10 @@ async def produce_openai_chat_completions(
# request as before.
if _sf_tools_on and not _launcher_tool_default_applies(payload, _ui_events):
_sf_tools_on = False
+ if _response_format_constrains_decoding(payload) and not _explicit_studio_tool_loop_requested(
+ payload
+ ):
+ _sf_tools_on = False
# tool_choice: "none" withdraws the catalogue outright, as the GGUF loop does at its
# controller. Nothing below reads the field (not _sf_use_tools, not
# _select_request_tools, and generate_chat_completion_with_tools is never passed it), so
@@ -28424,6 +28457,8 @@ async def produce_openai_chat_completions(
_sf_continue = _continue_final_message(payload)
_sf_continued_turn = [_sf_continue]
+ _sf_single_block = _response_format_constrains_decoding(payload)
+
def _new_sf_reasoning_extractor():
prefilled = _sf_reasoning_prefilled
if _sf_continued_turn[0]:
@@ -28432,6 +28467,7 @@ async def produce_openai_chat_completions(
return _ResponsesReasoningExtractor(
parse_think_markers = _sf_parse_think,
reasoning_prefilled = prefilled,
+ single_block = _sf_single_block,
)
cancel_event = _chat_cancel_event(request)
@@ -28494,6 +28530,19 @@ async def produce_openai_chat_completions(
if _sf_use_tools and _wants_multiple_choices(payload):
_raise_unsupported_n("non-GGUF tool chat completions", monitor_id)
+ if _sf_use_tools and _response_format_constrains_decoding(payload):
+ raise _reject(
+ 400,
+ openai_error_body(
+ "response_format is not supported with Unsloth tool execution; "
+ "send the request without enable_tools or mcp_enabled to use "
+ "guided decoding.",
+ status = 400,
+ code = "unsupported_parameter",
+ param = "response_format",
+ ),
+ )
+
if _sf_use_tools:
# permission_mode ask/auto require the confirm gate for Unsloth's own tool
# loop; when a CLI policy (--enable-tools) forces the loop on without a
@@ -29042,6 +29091,7 @@ async def produce_openai_chat_completions(
frequency_penalty = payload.frequency_penalty,
logit_bias = payload.logit_bias,
stop = normalized_stop,
+ response_format = _extract_response_format(payload),
)
if _video_clip is not None:
gen_kwargs["video"] = _video_clip
@@ -29127,7 +29177,12 @@ async def produce_openai_chat_completions(
else None
)
_sf_heal = (
- heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice)
+ heal_gate(
+ payload.auto_heal_tool_calls,
+ _sf_healing_tools,
+ payload.tool_choice,
+ response_format = _extract_response_format(payload),
+ )
if _sf_client_tools
else None
)
@@ -29236,6 +29291,8 @@ async def produce_openai_chat_completions(
template = _sf_image_tpl,
prefer_tool_use = _sf_image_tpl is None,
)
+ if gen_kwargs.get("response_format") is not None:
+ gen_kwargs["reasoning_is_extracted"] = bool(_sf_parse_think)
# Request-scoped usage/timings receptacle (filled at gen_done).
stats_holder: dict = {}
@@ -29328,8 +29385,16 @@ async def produce_openai_chat_completions(
backend.reset_generation_state(cancel_event)
_msg = _friendly_gen_stream_error(cumulative)
api_monitor.fail(monitor_id, _msg)
+ _refused = _refused_parameter(cumulative)
yield _openai_stream_error_sse(
- {"error": {"message": _msg, "type": "server_error"}}
+ openai_error_body(
+ _msg,
+ status = 400,
+ code = "unsupported_parameter",
+ param = _refused,
+ )
+ if _refused
+ else {"error": {"message": _msg, "type": "server_error"}}
)
return
if await request.is_disconnected():
@@ -29468,7 +29533,14 @@ async def produce_openai_chat_completions(
backend.reset_generation_state(cancel_event)
_msg = _friendly_gen_stream_error(exc)
api_monitor.fail(monitor_id, _msg)
- yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}})
+ _refused = _refused_parameter(exc)
+ yield _openai_stream_error_sse(
+ openai_error_body(
+ _msg, status = 400, code = "unsupported_parameter", param = _refused
+ )
+ if _refused
+ else {"error": {"message": _msg, "type": "server_error"}}
+ )
except Exception as e:
backend.reset_generation_state(cancel_event)
logger.error(f"Error during OpenAI streaming: {e}", exc_info = True)
@@ -29550,6 +29622,17 @@ async def produce_openai_chat_completions(
backend.reset_generation_state(cancel_event)
_msg = _friendly_gen_stream_error(full_text)
api_monitor.fail(monitor_id, _msg)
+ _refused = _refused_parameter(full_text)
+ if _refused:
+ raise _reject(
+ 400,
+ openai_error_body(
+ _msg,
+ status = 400,
+ code = "unsupported_parameter",
+ param = _refused,
+ ),
+ )
raise HTTPException(status_code = 500, detail = _msg)
# Split prefilled <think> reasoning (GGUF parity); also covers MLX via
@@ -29559,6 +29642,7 @@ async def produce_openai_chat_completions(
full_text,
parse_think_markers = _sf_parse_think,
reasoning_prefilled = _sf_reasoning_prefilled and not _sf_continue,
+ single_block = _sf_single_block,
)
# Client-tool passthrough: promote text-form calls; opt-in single
# nudge retry on unparseable tool markup.
@@ -29575,7 +29659,10 @@ async def produce_openai_chat_completions(
if _sf_heal:
if heal_openai_message(_msg, _sf_heal, _sf_healing_tools):
_finish = "tool_calls"
- elif nudge_enabled(payload.nudge_tool_calls):
+ elif nudge_enabled(
+ payload.nudge_tool_calls,
+ response_format = _extract_response_format(payload),
+ ):
_data = {
"choices": [
{"message": {"role": "assistant", "content": _visible_text}}
@@ -29753,10 +29840,18 @@ async def produce_openai_chat_completions(
raise
except GenStreamErrorRaised as exc:
# Adapter-controlled (compare-mode) backend failure. Honor the public
- # flag so operational errors surface their real message.
+ # flag, and the refused parameter so a contract refusal stays a client error.
backend.reset_generation_state(cancel_event)
_msg = _friendly_gen_stream_error(exc)
api_monitor.fail(monitor_id, _msg)
+ _refused = _refused_parameter(exc)
+ if _refused:
+ raise _reject(
+ 400,
+ openai_error_body(
+ _msg, status = 400, code = "unsupported_parameter", param = _refused
+ ),
+ )
raise HTTPException(status_code = 500, detail = _msg)
except Exception as e:
backend.reset_generation_state(cancel_event)
@@ -32152,13 +32247,15 @@ class _ResponsesReasoningExtractor:
*,
parse_think_markers: bool = False,
reasoning_prefilled: bool = False,
+ single_block: bool = False,
) -> None:
self._buffer = ""
# reasoning_prefilled: the template inserts an unclosed <think>, so output begins inside
- # the block; start in reasoning until the first close tag. Existing callers pass False.
- self._in_reasoning = reasoning_prefilled
+ # the block; start in reasoning until the first close tag.
+ self._in_reasoning = reasoning_prefilled and not single_block
# Splitting requires marker parsing; a prefilled open implies it.
self._parse_think_markers = parse_think_markers or reasoning_prefilled
+ self._single_block = single_block
def feed(
self,
@@ -32199,6 +32296,18 @@ class _ResponsesReasoningExtractor:
self._buffer = self._buffer[-keep:] if keep else ""
break
+ if self._single_block:
+ if self._buffer.startswith(_RESPONSES_THINK_OPEN):
+ self._buffer = self._buffer[len(_RESPONSES_THINK_OPEN) :]
+ self._in_reasoning = True
+ continue
+ if _RESPONSES_THINK_OPEN.startswith(self._buffer):
+ break # still short of the opener; it may yet arrive
+ self._parse_think_markers = False
+ visible_parts.append(self._buffer)
+ self._buffer = ""
+ break
+
open_idx = self._buffer.find(_RESPONSES_THINK_OPEN)
close_idx = self._buffer.find(_RESPONSES_THINK_CLOSE)
if close_idx != -1 and (open_idx == -1 or close_idx < open_idx):
@@ -32242,10 +32351,12 @@ def _extract_responses_reasoning(
*,
parse_think_markers: bool = False,
reasoning_prefilled: bool = False,
+ single_block: bool = False,
) -> tuple[str, str]:
extractor = _ResponsesReasoningExtractor(
parse_think_markers = parse_think_markers,
reasoning_prefilled = reasoning_prefilled,
+ single_block = single_block,
)
reasoning, visible = extractor.feed(text, reasoning_content)
final_reasoning, final_visible = extractor.finish()
@@ -32822,6 +32933,7 @@ async def _responses_non_streaming(
raw_text,
msg.get("reasoning_content"),
parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend),
+ single_block = _response_format_constrains_decoding(chat_req),
)
tool_calls = msg.get("tool_calls") or []
@@ -33062,7 +33174,8 @@ async def _responses_stream(
# From the chat chunks; applied once before finish, so chunk order does not matter.
stream_finish_reason: Optional[str] = None
extractor = _ResponsesReasoningExtractor(
- parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend)
+ parse_think_markers = _responses_should_parse_think_markers(chat_req, llama_backend),
+ single_block = _response_format_constrains_decoding(chat_req),
)
reasoning_state: dict[str, Any] = {"output_index": None, "item_id": None, "opened": False}
message_state: dict[str, Any] = {
@@ -33086,6 +33199,7 @@ async def _responses_stream(
getattr(chat_req, "auto_heal_tool_calls", None),
body.get("tools"),
body.get("tool_choice"),
+ response_format = body.get("response_format"),
)
healer = StreamToolCallHealer(_allowed_tools, body.get("tools")) if _allowed_tools else None
healed_tc_index = 0
@@ -37486,7 +37600,12 @@ async def _anthropic_passthrough_stream(
# was already sent as "auto", and gating on the stale name would intersect the safe
# names with a removed one and disable healing outright. "none" survives
# reconciliation, so it still forbids promotion (#7066).
- _allowed_tools = heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))
+ _allowed_tools = heal_gate(
+ auto_heal_tool_calls,
+ body.get("tools"),
+ body.get("tool_choice"),
+ response_format = body.get("response_format"),
+ )
if _allowed_tools:
emitter.enable_healing(
_allowed_tools,
@@ -37815,13 +37934,18 @@ async def _anthropic_passthrough_non_streaming(
# was already sent as "auto", and gating on the stale name would intersect the safe
# names with a removed one and disable healing outright. "none" survives
# reconciliation, so it still forbids promotion (#7066).
- _allowed_tools = heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))
+ _allowed_tools = heal_gate(
+ auto_heal_tool_calls,
+ body.get("tools"),
+ body.get("tool_choice"),
+ response_format = body.get("response_format"),
+ )
# Opt-in single-retry nudge (mirrors the OpenAI passthrough): the tool call came out
# unusable; re-ask with the prompt prefix intact so the KV cache is reused.
if (
_allowed_tools
- and nudge_enabled(nudge_tool_calls)
+ and nudge_enabled(nudge_tool_calls, response_format = body.get("response_format"))
and nudge_should_retry(data, _allowed_tools, _healing_tools)
):
first_data = data
@@ -38401,6 +38525,21 @@ def _extract_response_format(payload):
return rf if isinstance(rf, dict) else None
+def _response_format_for_llama_server(response_format):
+ """llama-server reads a json_schema schema only from ``json_schema.schema`` (tools/server/server-common.cpp:1170)."""
+ if (
+ isinstance(response_format, dict)
+ and response_format.get("type") == "json_schema"
+ # Absent means missing or null, the reading the MLX path takes, so both agree.
+ and response_format.get("json_schema") is None
+ and isinstance(response_format.get("schema"), dict)
+ ):
+ wrapped = {key: value for key, value in response_format.items() if key != "schema"}
+ wrapped["json_schema"] = {"schema": response_format["schema"]}
+ return wrapped
+ return response_format
+
+
def _response_format_constrains_decoding(payload) -> bool:
"""Whether the request's ``response_format`` needs a grammar engine.
@@ -38411,8 +38550,19 @@ def _response_format_constrains_decoding(payload) -> bool:
members it does not know -- is a contract the caller expects to be kept, so it
goes where such a contract can be answered or rejected rather than dropped.
"""
- rf = _extract_response_format(payload)
- return isinstance(rf, dict) and rf != {"type": "text"}
+ from core.inference.passthrough_healing import response_format_constrains_decoding
+ return response_format_constrains_decoding(_extract_response_format(payload))
+
+
+def _reject_unhonorable_response_format(payload) -> None:
+ from core.inference.grammar_constraint import (
+ ResponseFormatError,
+ constraint_spec_from_response_format,
+ )
+ try:
+ constraint_spec_from_response_format(_extract_response_format(payload))
+ except ResponseFormatError as exc:
+ _raise_unsupported_openai_parameter("response_format", str(exc))
def _build_openai_passthrough_body(
@@ -38463,7 +38613,7 @@ def _build_openai_passthrough_body(
frequency_penalty = payload.frequency_penalty,
logit_bias = payload.logit_bias,
tool_choice = tool_choice,
- response_format = _extract_response_format(payload),
+ response_format = _response_format_for_llama_server(_extract_response_format(payload)),
chat_template_kwargs = tpl_kwargs,
backend_ctx = backend_ctx,
seed = payload.seed,
@@ -38762,7 +38912,10 @@ async def _openai_passthrough_stream_admitted(
# auto_heal_tool_calls=false keep the unhealed relay. tool_choice constrains
# the allowlist ("none" disables, a forced function narrows to it).
_allowed_tools = heal_gate(
- payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")
+ payload.auto_heal_tool_calls,
+ body.get("tools"),
+ body.get("tool_choice"),
+ response_format = body.get("response_format"),
)
# Keep the pre-header window short so accepted SSE clients receive
@@ -39801,7 +39954,10 @@ async def _openai_passthrough_non_streaming_upstream(
_do_fence = _guided_fence and _extract_response_format(payload) is not None
_cap_parallel = payload.parallel_tool_calls is False
_allowed_tools = heal_gate(
- payload.auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")
+ payload.auto_heal_tool_calls,
+ body.get("tools"),
+ body.get("tool_choice"),
+ response_format = body.get("response_format"),
)
try:
@@ -39823,7 +39979,7 @@ async def _openai_passthrough_non_streaming_upstream(
usage_aggregated = False
if (
_allowed_tools
- and nudge_enabled(payload.nudge_tool_calls)
+ and nudge_enabled(payload.nudge_tool_calls, response_format = body.get("response_format"))
and nudge_should_retry(data, _allowed_tools, body.get("tools"))
):
first_data = data
diff --git a/studio/backend/tests/test_control_markup_neutralize_7066.py b/studio/backend/tests/test_control_markup_neutralize_7066.py
index dcc1b6bdf83..7228454ae5e 100644
--- a/studio/backend/tests/test_control_markup_neutralize_7066.py
+++ b/studio/backend/tests/test_control_markup_neutralize_7066.py
@@ -3743,10 +3743,9 @@ def test_anthropic_healing_is_gated_on_the_sanitized_catalog():
assert "heal_gate(auto_heal_tool_calls, openai_tools, tool_choice)" not in source
# The third argument is the reconciled choice the body carries, not the caller's: see
# test_healing_is_gated_on_the_tool_choice_actually_sent.
- assert (
- source.count('heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice"))')
- == 2
- )
+ packed = " ".join(source.split()).replace("( ", "(")
+ gate = 'heal_gate(auto_heal_tool_calls, body.get("tools"), body.get("tool_choice")'
+ assert packed.count(gate + ",") + packed.count(gate + ")") == 2
assert "nudge_should_retry(data, _allowed_tools, openai_tools)" not in source
@@ -4803,9 +4802,9 @@ def test_safetensors_healing_is_gated_on_the_sanitized_catalog():
"heal_gate(payload.auto_heal_tool_calls, payload.tools, payload.tool_choice)" not in source
)
assert "_sf_renderable_tools," in source and "asyncio.to_thread(" in source
- assert (
- "heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice)" in source
- )
+ packed = " ".join(source.split()).replace("( ", "(")
+ gate = "heal_gate(payload.auto_heal_tool_calls, _sf_healing_tools, payload.tool_choice"
+ assert packed.count(gate + ",") + packed.count(gate + ")") == 1
for call in (
"StreamToolCallHealer(_sf_heal, _sf_healing_tools)",
"heal_openai_message(_msg, _sf_heal, _sf_healing_tools)",
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 21e0b91e1c2..72034ea6559 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -4903,6 +4903,320 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(
assert guard < snapshot, "the guard must be held across the snapshot, not after it"
+import numpy as np # noqa: E402
+
+from core.inference import grammar_constraint as gc
+from core.inference.grammar_constraint import (
+ ResponseFormatError,
+ build_constraint,
+ constraint_spec_from_response_format,
+)
+
+SCHEMA = {
+ "type": "object",
+ "properties": {"a": {"type": "integer"}},
+ "required": ["a"],
+ "additionalProperties": False,
+}
+
+JSON_SCHEMA_FORMAT = {"type": "json_schema", "schema": SCHEMA}
+
+
+def _char_tokenizer(
+ *extra_specials,
+ markers_are_special = False,
+ extra_chars = "",
+):
+ if not gc.LLGUIDANCE_AVAILABLE:
+ pytest.skip("llguidance.mlx did not import")
+ tk = pytest.importorskip("tokenizers")
+ transformers = pytest.importorskip("transformers")
+
+ chars = list('{}"[]:,0123456789abcdefghijklmnopqrstuvwxyz \n.-' + extra_chars)
+ backend = tk.Tokenizer(tk.models.BPE(vocab = {c: i for i, c in enumerate(chars)}, merges = []))
+ backend.pre_tokenizer = tk.pre_tokenizers.ByteLevel(add_prefix_space = False, use_regex = False)
+ backend.decoder = tk.decoders.ByteLevel()
+ backend.add_special_tokens([tk.AddedToken("<eos>", special = True)])
+ markers = [tk.AddedToken(t, special = markers_are_special) for t in ("</think>", *extra_specials)]
+ if markers_are_special:
+ backend.add_special_tokens(markers)
+ else:
+ backend.add_tokens(markers)
+ tokenizer = transformers.PreTrainedTokenizerFast(tokenizer_object = backend, eos_token = "<eos>")
+ tokenizer.chat_template = "{{ messages[0].content }}<think>"
+ return tokenizer
+
+
[email protected](scope = "module")
+def tiny_tokenizer():
+ return _char_tokenizer()
+
+
+def _metal_mx():
+ mx = pytest.importorskip("mlx.core")
+ if not mx.metal.is_available():
+ pytest.skip("llguidance.mlx masks with a Metal kernel")
+ return mx
+
+
+def _allowed(constraint, width):
+ mx = _metal_mx()
+ masked = np.asarray(constraint.mask_logits(mx.zeros((width,), dtype = mx.float32)))
+ return set(np.nonzero(~np.isneginf(masked))[0].tolist())
+
+
+def test_every_spelling_of_a_supplied_schema_agrees_and_a_non_schema_is_refused(tiny_tokenizer):
+ wrapped = constraint_spec_from_response_format(
+ {"type": "json_schema", "json_schema": {"name": "city", "schema": SCHEMA}}
+ ).grammar
+ for spelling in (
+ JSON_SCHEMA_FORMAT,
+ {"type": "json_schema", "schema": dict(reversed(list(SCHEMA.items())))},
+ {"type": "json_object", "schema": SCHEMA},
+ ):
+ assert constraint_spec_from_response_format(spelling).grammar == wrapped
+ for supplied in (False, True, "nonsense", [1, 2]):
+ with pytest.raises(ResponseFormatError):
+ constraint_spec_from_response_format({"type": "json_object", "schema": supplied})
+ plain = constraint_spec_from_response_format({"type": "json_object"}).grammar
+ for spelling in (
+ {"type": "json_object", "schema": None},
+ {"type": "json_object", "schema": {}},
+ {"type": "json_schema", "schema": {"type": "object"}},
+ ):
+ assert constraint_spec_from_response_format(spelling).grammar == plain
+ digit = tiny_tokenizer.encode("4", add_special_tokens = False)[0]
+ width = len(tiny_tokenizer)
+ any_value = build_constraint({"type": "json_schema", "schema": {}}, tiny_tokenizer, "p")
+ assert digit in _allowed(any_value, width)
+ assert digit not in _allowed(
+ build_constraint({"type": "json_object"}, tiny_tokenizer, "p"), width
+ )
+
+
+def test_the_schema_itself_constrains_not_merely_json_syntax(tiny_tokenizer):
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")
+ _allowed(constraint, len(tiny_tokenizer))
+ for token_id in tiny_tokenizer.encode('{"', add_special_tokens = False):
+ constraint.advance(int(token_id))
+ assert _allowed(constraint, len(tiny_tokenizer)) == set(
+ tiny_tokenizer.encode("a", add_special_tokens = False)
+ )
+ for token_id in tiny_tokenizer.encode('a":', add_special_tokens = False):
+ constraint.advance(int(token_id))
+ quote = tiny_tokenizer.encode('"', add_special_tokens = False)[0]
+ assert quote not in _allowed(constraint, len(tiny_tokenizer))
+
+
+def test_advance_raises_on_a_token_the_grammar_rejects(tiny_tokenizer):
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")
+ _allowed(constraint, len(tiny_tokenizer))
+ illegal = tiny_tokenizer.encode("]", add_special_tokens = False)[0]
+ with pytest.raises(gc.GrammarDesyncError, match = "desynced on token") as excinfo:
+ constraint.advance(int(illegal))
+ assert not isinstance(excinfo.value, ResponseFormatError)
+
+
+def test_binding_waits_for_the_logits_width_a_padded_lm_head_widens(tiny_tokenizer):
+ gc._TOKENIZER_CACHE.clear()
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")
+ padded = len(tiny_tokenizer) + 16
+ legal = _allowed(constraint, padded)
+ assert legal == set(tiny_tokenizer.encode("{", add_special_tokens = False))
+ assert max(legal) < len(tiny_tokenizer)
+
+
+def test_prelude_is_bounded_so_an_unclosed_think_block_cannot_eat_the_budget(
+ tiny_tokenizer, monkeypatch
+):
+ monkeypatch.setattr(gc, "_PRELUDE_MAX_TOKENS", 6)
+ constraint = build_constraint(
+ JSON_SCHEMA_FORMAT, tiny_tokenizer, "chat\n<think>\n", reasoning_is_extracted = True
+ )
+ _allowed(constraint, len(tiny_tokenizer))
+ for token_id in tiny_tokenizer.encode("abcdef", add_special_tokens = False):
+ constraint.advance(int(token_id))
+ close_id = tiny_tokenizer.encode("</think>", add_special_tokens = False)[0]
+ assert _allowed(constraint, len(tiny_tokenizer)) == {close_id}
+ constraint.advance(int(close_id))
+ assert _allowed(constraint, len(tiny_tokenizer)) == set(
+ tiny_tokenizer.encode("{", add_special_tokens = False)
+ )
+
+
+def test_reasoning_cannot_spell_the_closer_as_text_and_forge_the_document_boundary():
+ tokenizer = _char_tokenizer(extra_chars = "</>")
+
+ def _fresh():
+ constraint = build_constraint(
+ JSON_SCHEMA_FORMAT, tokenizer, "chat\n<think>", reasoning_is_extracted = True
+ )
+ constraint._bind(len(tokenizer))
+ return constraint
+
+ def _step(constraint, token_id):
+ # The numpy half of mask_logits, so the desync check reads a real mask without Metal.
+ gc._llg_mlx.fill_next_token_bitmask(constraint._matcher, constraint._bitmask)
+ constraint.advance(int(token_id))
+
+ def _chars(text):
+ return [tokenizer.convert_tokens_to_ids(c) for c in text]
+
+ constraint = _fresh()
+ for token_id in _chars("a<b</think"):
+ _step(constraint, token_id)
+ with pytest.raises(gc.GrammarDesyncError):
+ _step(constraint, tokenizer.convert_tokens_to_ids(">"))
+
+ close_id = tokenizer.encode("</think>", add_special_tokens = False)
+ assert len(close_id) == 1
+ constraint = _fresh()
+ for token_id in (
+ _chars("a<b</th") + close_id + tokenizer.encode('{"a":1}', add_special_tokens = False)
+ ):
+ _step(constraint, token_id)
+
+
[email protected]("reply_keeps_special_tokens", [False, True])
+def test_a_close_marker_is_refused_only_where_the_reply_would_lose_it(reply_keeps_special_tokens):
+ tokenizer = _char_tokenizer(markers_are_special = True)
+ ids = tokenizer.encode("</think>", add_special_tokens = False)
+ assert (
+ tokenizer.decode(ids, skip_special_tokens = True) != "</think>"
+ ), "fixture no longer models a marker a skip_special_tokens decode strips"
+
+ def _build():
+ return build_constraint(
+ JSON_SCHEMA_FORMAT,
+ tokenizer,
+ "chat\n<think>",
+ reasoning_is_extracted = True,
+ reply_keeps_special_tokens = reply_keeps_special_tokens,
+ )
+
+ if not reply_keeps_special_tokens:
+ with pytest.raises(ResponseFormatError):
+ _build()
+ return
+ constraint = _build()
+ assert constraint.allows_reasoning
+ for token_id in tokenizer.encode('x</think>{"a":1}', add_special_tokens = False):
+ assert int(token_id) in _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token_id))
+
+
+class _Recorder:
+ def __init__(self):
+ self.advanced = []
+
+ def mask_logits(self, logits):
+ return logits
+
+ def advance(self, token_id):
+ self.advanced.append(int(token_id))
+
+
+def test_the_grammar_mask_runs_first_and_its_inf_survives_every_knob():
+ mx = pytest.importorskip("mlx.core")
+ pytest.importorskip("mlx_lm")
+ from core.inference.mlx_inference import _mlx_sampling_processors
+
+ class _MaskOdd(_Recorder):
+ def mask_logits(self, logits):
+ keep = mx.array([[1.0, 0.0, 1.0, 0.0]])
+ return mx.where(keep > 0, logits, mx.array(float("-inf")))
+
+ grammar = _MaskOdd()
+ processors = _mlx_sampling_processors(
+ repetition_penalty = 1.3,
+ presence_penalty = 0.5,
+ frequency_penalty = 0.5,
+ logit_bias = {1: 100.0},
+ grammar_constraint = grammar,
+ )
+ zeros = mx.zeros((1, 4), dtype = mx.float32)
+ tokens = mx.array([0, 1])
+ leading = np.asarray(processors[0](tokens, zeros))[0]
+ assert np.isneginf(leading[1]) and np.isneginf(leading[3])
+
+ logits = zeros
+ for processor in processors:
+ logits = processor(tokens, logits)
+ out = np.asarray(logits)[0]
+ assert np.isneginf(out[1]) and np.isneginf(out[3])
+ assert np.isfinite(out[0]) and np.isfinite(out[2])
+
+ assert grammar.advanced == []
+ processors[0](mx.array([0, 1, 2, 3]), zeros)
+ processors[0](mx.array([0, 1, 2, 3, 4]), zeros)
+ assert grammar.advanced == [2, 3, 4]
+
+
+def _two_stop_tokenizer():
+ tokenizer = _char_tokenizer("<end_of_text>", markers_are_special = True)
+ tokenizer.chat_template = "{{ messages[0].content }}"
+ runtime_stop = tokenizer.convert_tokens_to_ids("<end_of_text>")
+ assert runtime_stop != tokenizer.eos_token_id, "fixture no longer models a disagreement"
+ return tokenizer, runtime_stop
+
+
+def _stop_offered_after_the_document(target, tokenizer):
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, target, "prompt")
+ for token_id in tokenizer.encode('{"a":1}', add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token_id))
+ assert constraint._matcher.is_stopped(), "the document did not complete"
+ return _allowed(constraint, len(tokenizer))
+
+
+def test_the_grammar_stops_on_the_token_the_decode_loop_stops_on():
+ mlx_lm = pytest.importorskip("mlx_lm")
+ from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])
+ assert _stop_offered_after_the_document(wrapped, tokenizer) == {runtime_stop}
+ assert _stop_offered_after_the_document(tokenizer, tokenizer) == {tokenizer.eos_token_id}
+
+
+def test_a_schema_forced_through_a_stop_token_is_refused_while_it_can_still_be_reported():
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])
+
+ def replay(schema, text):
+ constraint = build_constraint({"type": "json_schema", "schema": schema}, wrapped, "p")
+ for token in tokenizer.encode(text, add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token))
+ return _allowed(constraint, len(tokenizer))
+
+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):
+ replay({"const": tokenizer.convert_ids_to_tokens(runtime_stop)}, '"')
+ assert runtime_stop in replay({"type": "integer"}, "42")
+ assert runtime_stop in replay(SCHEMA, '{"a":1}')
+
+
+def test_a_scalar_end_token_does_not_hide_the_runtime_stop_set():
+ pytest.importorskip("mlx_vlm")
+ from mlx_vlm.utils import StoppingCriteria
+
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ assert getattr(tokenizer, "eos_token_ids", None) == tokenizer.eos_token_id, "not a scalar"
+ criteria = StoppingCriteria([runtime_stop], tokenizer = tokenizer)
+ tokenizer.stopping_criteria = criteria
+ for target in (tokenizer, SimpleNamespace(tokenizer = tokenizer)):
+ assert _stop_offered_after_the_document(target, tokenizer) == set(criteria.eos_token_ids)
+
+
+def test_advance_before_the_first_mask_raises_rather_than_dropping_the_token(tiny_tokenizer):
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tiny_tokenizer, "prompt")
+ with pytest.raises(gc.GrammarDesyncError, match = "before the grammar was bound"):
+ constraint.advance(tiny_tokenizer.encode("{", add_special_tokens = False)[0])
+
+
def _uncopyable_naive_detokenizer(detokenizers):
"""mlx-vlm's naive detokenizer as it behaves BELOW 0.6.0, which is where ``__copy__`` arrived.
@@ -6739,3 +7053,351 @@ def test_single_image_replay_notes_describe_only_retained_pixels(monkeypatch, me
else:
assert calls[-1]["prompt"].count(IMAGE_TURN_TEXT) == 1
assert len(image_marker_parts(history)) == 2
+
+
+def test_a_random_model_decoding_through_mlx_lm_emits_only_the_schema():
+ """Real mlx_lm pipelining (the next step is masked before this token is yielded) with a
+ random model: only the grammar can make the reply a valid document."""
+ mx = _metal_mx()
+ pytest.importorskip("mlx_lm")
+ from mlx_lm import stream_generate
+ from mlx_lm.models import llama
+ from mlx_lm.sample_utils import make_sampler
+ from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+ from core.inference.grammar_constraint import make_grammar_logits_processor
+
+ tokenizer = _char_tokenizer()
+ tokenizer.chat_template = "{{ messages[0].content }}"
+ mx.random.seed(0)
+ model = llama.Model(
+ llama.ModelArgs(
+ model_type = "llama",
+ hidden_size = 32,
+ num_hidden_layers = 1,
+ intermediate_size = 64,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ rms_norm_eps = 1e-5,
+ vocab_size = len(tokenizer),
+ )
+ )
+ schema = {
+ "type": "object",
+ "properties": {"a": {"enum": ["x", "y"]}, "b": {"type": "boolean"}},
+ "required": ["a", "b"],
+ "additionalProperties": False,
+ }
+ wrapped = TokenizerWrapper(tokenizer)
+ constraint = build_constraint({"type": "json_schema", "schema": schema}, wrapped, "p")
+ text = "".join(
+ chunk.text
+ for chunk in stream_generate(
+ model,
+ wrapped,
+ tokenizer.encode("go", add_special_tokens = False),
+ max_tokens = 64,
+ sampler = make_sampler(temp = 1.0),
+ logits_processors = [make_grammar_logits_processor(constraint)],
+ )
+ )
+ document = json.loads(text)
+ assert document["a"] in ("x", "y") and isinstance(document["b"], bool)
+ assert set(document) == {"a", "b"}
+
+
+def _constrained_vlm_reply(
+ monkeypatch,
+ pieces,
+ *,
+ allows_reasoning,
+ markers,
+ specials = (),
+):
+ """Snapshots ``_generate_vlm`` yields for scripted text under a stand-in grammar."""
+ from core.inference import mlx_inference
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ mlx_vlm = types.ModuleType("mlx_vlm")
+ mlx_vlm.prompt_utils = SimpleNamespace(MODEL_CONFIG = {}, apply_chat_template = None)
+ mlx_vlm.stream_generate = lambda *_a, **_k: iter(
+ [SimpleNamespace(text = piece, prompt_tokens = 1, generation_tokens = 1) for piece in pieces]
+ )
+ monkeypatch.setitem(sys.modules, "mlx_vlm", mlx_vlm)
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda *_a, **_k: "user: hi\nassistant: <think>\n",
+ )
+ monkeypatch.setattr(
+ mlx_inference, "_temporary_mlx_adapter_state", lambda *_a, **_k: contextlib.nullcontext()
+ )
+ monkeypatch.setattr(
+ mlx_inference, "detect_reasoning_channel_markers", lambda *_a, **_k: markers
+ )
+ monkeypatch.setattr(
+ mlx_inference,
+ "_build_grammar_constraint",
+ lambda *_a, **_k: SimpleNamespace(
+ allows_reasoning = allows_reasoning,
+ stops_on = lambda _ids: None,
+ decoded_dropping = lambda _ids = None: None,
+ ),
+ )
+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)
+ monkeypatch.setattr(mlx_inference, "_vlm_generation_is_diffusion", lambda _model: False)
+
+ backend = MLXInferenceBackend()
+ backend._model = SimpleNamespace(config = {"model_type": "generic_vlm"})
+ backend._processor = SimpleNamespace(chat_template = "template", all_special_tokens = specials)
+ args = ([{"role": "user", "content": "hi"}], [], 0.7, 0.9, 40, 0.01, 64, 1.0, None)
+ return list(backend._generate_vlm(*args, response_format = {"type": "json_object"}))
+
+
+def test_a_document_only_vlm_reply_is_not_rewritten_by_the_reasoning_normalizer(monkeypatch):
+ document = '{"city":"<think>Paris</think>"}'
+ snapshots = _constrained_vlm_reply(
+ monkeypatch,
+ [document[:12], document[12:]],
+ allows_reasoning = False,
+ markers = ("<think>", "</think>"),
+ )
+ assert snapshots[-1] == document
+
+
+def test_an_allowed_vlm_reasoning_block_keeps_its_opener_when_the_closer_is_special(monkeypatch):
+ snapshots = _constrained_vlm_reply(
+ monkeypatch,
+ ["x", "</think>", '{"a":1}'],
+ allows_reasoning = True,
+ markers = None,
+ specials = ("</think>",),
+ )
+ assert snapshots[-1] == '<think>\nx</think>{"a":1}'
+
+
+def test_the_text_grammar_is_told_a_closer_survives_whenever_the_prefill_is(monkeypatch):
+ """Client tools run the native decoder, which keeps a special </think>: the prefill then
+ re-emits the opener, so the grammar must allow the close rather than refuse the request."""
+ from core.inference import mlx_inference
+
+ mlx_lm = types.ModuleType("mlx_lm")
+ sample_utils = types.ModuleType("mlx_lm.sample_utils")
+ sample_utils.make_sampler = lambda **_k: object()
+ mlx_lm.sample_utils = sample_utils
+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])
+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)
+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)
+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)
+ monkeypatch.setattr(mlx_inference, "decoder_preserves_token", lambda *_a, **_k: True)
+ seen = {}
+
+ def _capture(*_a, **kwargs):
+ seen.update(kwargs)
+ return None
+
+ monkeypatch.setattr(mlx_inference, "_build_grammar_constraint", _capture)
+ backend = _budget_backend(monkeypatch)
+ monkeypatch.setattr(
+ backend,
+ "_render_text_prompt",
+ lambda *_a, **_k: SimpleNamespace(prompt = "P<think>", reasoning_channel_markers = None),
+ raising = False,
+ )
+ list(
+ backend._generate_text(
+ _TEXT_TURN,
+ 0.0,
+ 1.0,
+ 0,
+ 0.0,
+ 8,
+ 1.0,
+ None,
+ tools = [{"type": "function", "function": {"name": "f"}}],
+ response_format = {"type": "json_object"},
+ reasoning_is_extracted = True,
+ )
+ )
+ assert seen["reply_keeps_special_tokens"] is True
+
+
[email protected]("constrained", [False, True])
+def test_a_constrained_plain_reply_is_decoded_without_space_cleanup(monkeypatch, constrained):
+ from core.inference import mlx_inference
+
+ mlx_lm = types.ModuleType("mlx_lm")
+ sample_utils = types.ModuleType("mlx_lm.sample_utils")
+ sample_utils.make_sampler = lambda **_k: object()
+ mlx_lm.sample_utils = sample_utils
+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])
+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)
+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)
+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)
+ monkeypatch.setattr(
+ mlx_inference,
+ "_build_grammar_constraint",
+ lambda *_a, **_k: SimpleNamespace(
+ allows_reasoning = False, decoded_dropping = lambda _ids: None
+ )
+ if constrained
+ else None,
+ )
+ backend = _budget_backend(monkeypatch)
+ decodes = []
+
+ def _decode(_ids, **kwargs):
+ decodes.append(kwargs)
+ return "ok"
+
+ backend._tokenizer.decode = _decode
+ monkeypatch.setattr(
+ backend,
+ "_render_text_prompt",
+ lambda *_a, **_k: SimpleNamespace(prompt = "P", reasoning_channel_markers = None),
+ raising = False,
+ )
+ list(
+ backend._generate_text(
+ _TEXT_TURN,
+ 0.0,
+ 1.0,
+ 0,
+ 0.0,
+ 8,
+ 1.0,
+ None,
+ response_format = {"type": "json_object"} if constrained else None,
+ )
+ )
+ assert decodes
+ # " ." inside a grammar-approved string must reach the client unchanged.
+ for kwargs in decodes:
+ assert kwargs.get("clean_up_tokenization_spaces") is (False if constrained else None)
+
+
+def test_a_special_token_the_reply_would_drop_is_refused_not_silently_deleted():
+ """llguidance spells a forced literal with a special id when one matches, and a decode that
+ skips special tokens would delete it from a reply reported as valid."""
+ tokenizer = _char_tokenizer("ab", markers_are_special = True)
+ special = tokenizer.convert_tokens_to_ids("ab")
+ fmt = {"type": "json_schema", "schema": {"const": "ab"}}
+
+ def _after_quote(constraint):
+ for token in tokenizer.encode('"', add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token))
+ return _allowed(constraint, len(tokenizer))
+
+ assert special in _after_quote(
+ build_constraint(fmt, tokenizer, "p", reply_keeps_special_tokens = True)
+ )
+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):
+ _after_quote(build_constraint(fmt, tokenizer, "p"))
+
+
+def test_only_the_specials_the_decoder_drops_are_withheld_from_the_document():
+ """A native decoder keeps its allowlist and drops every other special id, so the grammar is
+ held to exactly that set, not to all specials or none."""
+ tokenizer = _char_tokenizer("ab", "cd", markers_are_special = True)
+ dropped, kept = (tokenizer.convert_tokens_to_ids(t) for t in ("ab", "cd"))
+
+ def _after_quote(literal):
+ constraint = build_constraint(
+ {"type": "json_schema", "schema": {"const": literal}},
+ tokenizer,
+ "p",
+ reply_keeps_special_tokens = True,
+ )
+ constraint.decoded_dropping({dropped})
+ for token in tokenizer.encode('"', add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token))
+ return _allowed(constraint, len(tokenizer))
+
+ assert kept in _after_quote("cd")
+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):
+ _after_quote("ab")
+
+
+def test_the_text_path_hands_the_constraint_its_decoders_dropped_ids(monkeypatch):
+ from core.inference import mlx_inference
+
+ mlx_lm = types.ModuleType("mlx_lm")
+ sample_utils = types.ModuleType("mlx_lm.sample_utils")
+ sample_utils.make_sampler = lambda **_k: object()
+ mlx_lm.sample_utils = sample_utils
+ mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("ok", 1)])
+ monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)
+ monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)
+ monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)
+ seen = []
+ constraint = SimpleNamespace(allows_reasoning = False, decoded_dropping = seen.append)
+ monkeypatch.setattr(mlx_inference, "_build_grammar_constraint", lambda *_a, **_k: constraint)
+
+ class _Decoder:
+ def __init__(self, *_a, **_k):
+ pass
+
+ def dropped_ids(self):
+ return frozenset({7})
+
+ monkeypatch.setattr(mlx_inference, "NativeToolTokenDecoder", _Decoder)
+ backend = _budget_backend(monkeypatch)
+ monkeypatch.setattr(
+ backend,
+ "_render_text_prompt",
+ lambda *_a, **_k: SimpleNamespace(prompt = "P", reasoning_channel_markers = None),
+ raising = False,
+ )
+ run = lambda **kw: list(
+ backend._generate_text(
+ _TEXT_TURN,
+ 0.0,
+ 1.0,
+ 0,
+ 0.0,
+ 8,
+ 1.0,
+ None,
+ response_format = {"type": "json_object"},
+ **kw,
+ )
+ )
+ try:
+ run(tools = [{"type": "function", "function": {"name": "f"}}])
+ except Exception:
+ pass # the stand-in decoder cannot decode; the handoff happened before generation
+ assert seen and seen[0] == frozenset({7})
+ seen.clear()
+ run()
+ assert seen == [None], "a plain skip_special_tokens decode drops every special id"
+
+
+def test_a_tokenizer_end_token_the_runtime_does_not_stop_on_is_withheld_too():
+ """llguidance ends only on the runtime stops, so the tokenizer's own end token is just
+ another special id the decode drops, and a literal spelled only by it is refused."""
+ pytest.importorskip("mlx_lm")
+ from mlx_lm.tokenizer_utils import TokenizerWrapper
+
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ wrapped = TokenizerWrapper(tokenizer, eos_token_ids = [runtime_stop])
+ constraint = build_constraint(
+ {"type": "json_schema", "schema": {"const": "<eos>"}}, wrapped, "p"
+ )
+ for token in tokenizer.encode('"', add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token))
+ with pytest.raises(ResponseFormatError, match = "reserves as a control token"):
+ _allowed(constraint, len(tokenizer))
+
+
+def test_the_grammar_ends_on_the_stop_the_decode_loop_names():
+ """mlx-vlm stops on config ids the tokenizer may not carry; told them, the finished
+ document is closed by that id rather than the tokenizer's end token."""
+ tokenizer, runtime_stop = _two_stop_tokenizer()
+ constraint = build_constraint(JSON_SCHEMA_FORMAT, tokenizer, "p")
+ constraint.stops_on([runtime_stop])
+ for token_id in tokenizer.encode('{"a":1}', add_special_tokens = False):
+ _allowed(constraint, len(tokenizer))
+ constraint.advance(int(token_id))
+ assert _allowed(constraint, len(tokenizer)) == {runtime_stop}
diff --git a/studio/backend/tests/test_mlx_stack_blockers.py b/studio/backend/tests/test_mlx_stack_blockers.py
index 531890da2cc..bc40d5d1bdc 100644
--- a/studio/backend/tests/test_mlx_stack_blockers.py
+++ b/studio/backend/tests/test_mlx_stack_blockers.py
@@ -382,3 +382,10 @@ def test_uv_missing_never_marks_the_environment(monkeypatch):
monkeypatch.setattr(mr, "_transformers_constraint_args", lambda: ([], None))
assert mr.attempt_mlx_repair() is False
assert mr._environment_mutated is False
+
+
+def test_a_missing_grammar_engine_names_the_package_instead_of_ignoring_the_schema(monkeypatch):
+ from core.inference import grammar_constraint as gc
+ monkeypatch.setattr(gc, "LLGUIDANCE_AVAILABLE", False)
+ with pytest.raises(gc.ResponseFormatError, match = "llguidance"):
+ gc.constraint_spec_from_response_format({"type": "json_object"})
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 1cacbb6daff..e925f213463 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -10753,6 +10753,30 @@ def test_the_two_seed_helpers_agree_on_which_seeds_are_random():
assert payload["cache_prompt"] is False, (seed, value)
+def test_a_lenient_schema_reaches_llama_server_where_it_reads_one():
+ from models.inference import ChatCompletionRequest
+ from routes.inference import _response_format_for_llama_server as for_llama
+
+ schema = {"type": "integer"}
+ wrapped = {"type": "json_schema", "json_schema": {"schema": schema}}
+ lenient = {"type": "json_schema", "schema": schema}
+ assert for_llama(lenient) == wrapped
+ assert for_llama({"type": "json_schema", "json_schema": None, "schema": schema}) == wrapped
+ for already in (
+ {"type": "json_schema", "json_schema": {"name": "x", "schema": schema}},
+ {"type": "json_schema", "json_schema": {"schema": schema}, "schema": {}},
+ {"type": "json_object", "schema": schema},
+ {"type": "json_object"},
+ {"type": "text"},
+ None,
+ ):
+ assert for_llama(already) is already
+ request = ChatCompletionRequest(
+ model = "m", messages = [{"role": "user", "content": "hi"}], response_format = lenient
+ )
+ assert _build_openai_passthrough_body(request)["response_format"] == wrapped
+
+
class TestPassthroughImageNormalization:
@staticmethod
def _data_url(fmt: str) -> str:
diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py
index 01d9ac6bebe..c38f8fe22ba 100644
--- a/studio/backend/tests/test_passthrough_healing.py
+++ b/studio/backend/tests/test_passthrough_healing.py
@@ -24,6 +24,7 @@ from core.inference.passthrough_healing import ( # noqa: E402
StreamToolCallHealer,
heal_gate,
heal_openai_message,
+ nudge_enabled,
nudge_messages,
nudge_should_retry,
response_has_promotable_calls,
@@ -1630,3 +1631,40 @@ class TestClientToolSchemaTyping:
)
assert _healed_arguments(call % ("WebFetch", "url")) == {"url": "x", "timeout": "30"}
assert _healed_arguments(call % ("Grep", "pattern")) == {"pattern": "x", "timeout": 30}
+
+
+_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}
+_CONTRACT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _SCHEMA}}
+
+
+def test_a_contract_withdraws_both_post_decode_reinterpreters():
+ tools = [{"type": "function", "function": {"name": "lookup"}}]
+ assert heal_gate(True, tools) == {"lookup"}
+ assert heal_gate(True, tools, response_format = {"type": "text"}) == {"lookup"}
+ assert nudge_enabled(True, response_format = {"type": "text"}) is True
+ for constraining in (_CONTRACT, {"type": "json_object"}, {"type": "text", "x": 1}, True, "x"):
+ assert heal_gate(True, tools, response_format = constraining) is None, constraining
+ assert nudge_enabled(True, response_format = constraining) is False, constraining
+
+
+def test_every_post_decode_reinterpreter_is_handed_the_request_s_own_contract():
+ import ast
+ from pathlib import Path
+
+ names = ("heal_gate", "nudge_enabled")
+ forwards = ('body.get("response_format")', "_extract_response_format(payload)")
+ source = (Path(__file__).resolve().parents[1] / "routes/inference.py").read_text(
+ encoding = "utf-8"
+ )
+ calls, missing = [], []
+ for node in ast.walk(ast.parse(source)):
+ if not isinstance(node, ast.Call):
+ continue
+ if (getattr(node.func, "attr", None) or getattr(node.func, "id", None)) not in names:
+ continue
+ calls.append(node)
+ kwargs = {k.arg: ast.get_source_segment(source, k.value) or "" for k in node.keywords}
+ if kwargs.get("response_format") not in forwards:
+ missing.append(node.lineno)
+ assert len(calls) >= 9, f"expected 9 {names} sites, found {len(calls)}"
+ assert not missing, f"heal_gate/nudge_enabled not handed the contract at {missing}"
diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.py
index 5c7b433f2eb..d48bb3bf078 100644
--- a/studio/backend/tests/test_presence_penalty.py
+++ b/studio/backend/tests/test_presence_penalty.py
@@ -215,6 +215,10 @@ def test_orchestrator_cmd_carries_all_sampling_params():
)
for key, val in _SAMPLING.items():
assert cmd[key] == val, f"{key} dropped/altered in orchestrator cmd"
+ rf = {"response_format": {"type": "json_object"}, "reasoning_is_extracted": True}
+ with_rf = o._build_generate_cmd("req1", None, messages = [], max_new_tokens = 8, **rf)
+ assert {key: with_rf[key] for key in rf} == rf
+ assert "reasoning_is_extracted" not in cmd
def test_worker_forwards_all_sampling_params_to_backend():
@@ -226,8 +230,14 @@ def test_worker_forwards_all_sampling_params_to_backend():
def __init__(self):
self.received = None
- def generate_chat_response(self, **kwargs):
- self.received = kwargs
+ def generate_chat_response(
+ self,
+ *,
+ response_format = None,
+ reasoning_is_extracted = None,
+ **kwargs,
+ ):
+ self.received = dict(kwargs, rf = response_format, rie = reasoning_is_extracted)
return iter(()) # empty stream -> loop exits, gen_done is sent
class _FakeQueue:
@@ -242,6 +252,8 @@ def test_worker_forwards_all_sampling_params_to_backend():
"request_id": "r",
"messages": [{"role": "user", "content": "hi"}],
"max_new_tokens": 128,
+ "response_format": {"type": "json_object"},
+ "reasoning_is_extracted": True,
**_SAMPLING,
}
backend = _RecordingBackend()
@@ -250,6 +262,8 @@ def test_worker_forwards_all_sampling_params_to_backend():
assert backend.received is not None
for key, val in _SAMPLING.items():
assert backend.received[key] == val, f"{key} dropped/altered in worker gen_kwargs"
+ assert backend.received["rf"] == {"type": "json_object"}
+ assert backend.received["rie"] is True
def test_orchestrator_cmd_carries_the_tool_protocol_flag():
@@ -323,9 +337,10 @@ def test_the_mlx_think_prefill_predicate_matches_the_decoder_it_describes():
from core.inference.mlx_inference import MLXInferenceBackend
src = inspect.getsource(MLXInferenceBackend._generate_text)
- after = src[src.index("preserves_think_close") :]
+ after = src[src.index("think_close_survives = (") :]
predicate = after[: after.index("decoder_preserves_token")]
assert "tool_protocol_active" in predicate, "the prefill predicate ignores unrestricted mode"
+ assert "preserves_think_close = think_close_survives" in src
def test_the_mlx_vlm_decoder_survives_a_reasoning_only_request():
diff --git a/studio/backend/tests/test_research_json_fallback.py b/studio/backend/tests/test_research_json_fallback.py
index d7498ae1314..0444deec371 100644
--- a/studio/backend/tests/test_research_json_fallback.py
+++ b/studio/backend/tests/test_research_json_fallback.py
@@ -23,7 +23,8 @@ from .test_sf_client_tools_passthrough import _ScriptedBackend, _fixed, _install
_NO_GRAMMAR_ENGINE = (
- "response_format needs the llama.cpp grammar engine; load a GGUF model to use it."
+ "response_format needs a grammar engine, and the transformers backend has none; load an "
+ "MLX or GGUF model to use it."
)
# Same code and param, different cause. The real-route cases keep these strings honest.
_AUDIO_REFUSAL_MESSAGE = (
@@ -107,6 +108,10 @@ def research_call(monkeypatch):
def test_local_json_research_recovers_through_the_real_route(
monkeypatch, research_call, is_mlx, phase, forced_tools
):
+ from core.inference import grammar_constraint
+
+ # MLX refuses only without its engine; pinned so a Mac with llguidance tests this too.
+ monkeypatch.setattr(grammar_constraint, "LLGUIDANCE_AVAILABLE", False)
backend = _ScriptedBackend(_fixed('{"ok": true}'))
backend.models[backend.active_model_name]["is_mlx"] = is_mlx
_install(monkeypatch, backend, supports_tools = forced_tools)
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index c97d5abd7b3..68d5cdd5e3f 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -130,6 +130,7 @@ def _install(
backend,
*,
supports_tools = True,
+ supports_reasoning = False,
):
import routes.inference as inf
from state.tool_policy import reset_tool_policy
@@ -142,7 +143,10 @@ def _install(
monkeypatch.setattr(
inf,
"_detect_safetensors_features",
- lambda *a, **k: {"supports_tools": supports_tools},
+ lambda *a, **k: {
+ "supports_tools": supports_tools,
+ "supports_reasoning": supports_reasoning,
+ },
)
return monitor
@@ -1369,6 +1373,55 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):
assert backend.calls[0]["image"] is not None
+_RF_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}
+_RF_FORMAT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _RF_SCHEMA}}
+_MARKUP_DOC = '{"city":"<think>Paris</think>"}'
+
+
[email protected]("chunked", [False, True], ids = ["whole", "streamed"])
+ "prefix, prefilled, reasoning",
+ [
+ ("", False, ""),
+ ("<think>weighing</think>", False, "weighing"),
+ ("<think>weighing</think>", True, "weighing"),
+ ("", True, ""),
+ ],
+)
+def test_markers_inside_a_constrained_document_stay_in_it(chunked, prefix, prefilled, reasoning):
+ from routes.inference import _ResponsesReasoningExtractor
+
+ extractor = _ResponsesReasoningExtractor(
+ parse_think_markers = True, reasoning_prefilled = prefilled, single_block = True
+ )
+ text = prefix + _MARKUP_DOC
+ got_reasoning = got_content = ""
+ for piece in text if chunked else [text]:
+ delta_reasoning, delta_content = extractor.feed(piece)
+ got_reasoning += delta_reasoning
+ got_content += delta_content
+ delta_reasoning, delta_content = extractor.finish()
+
+ assert got_content + delta_content == _MARKUP_DOC
+ assert got_reasoning + delta_reasoning == reasoning
+ assert json.loads(_MARKUP_DOC)["city"] == "<think>Paris</think>"
+
+
[email protected]("reasoning", [False, True], ids = ["plain", "reasoning"])
+def test_the_contract_reaches_the_backend_and_its_reply_comes_back_whole(monkeypatch, reasoning):
+ pytest.importorskip("llguidance.mlx")
+ backend = _ScriptedBackend(_fixed(_MARKUP_DOC))
+ backend.models["sf-model"]["is_mlx"] = True
+ caps = {"supports_tools": False, "supports_reasoning": reasoning}
+ reply = _call(_request(response_format = _RF_FORMAT), monkeypatch, backend, **caps)
+ body = _json_body(reply)
+ assert backend.calls[0]["response_format"] == _RF_FORMAT
+ assert backend.calls[0]["reasoning_is_extracted"] is reasoning
+ message = body["choices"][0]["message"]
+ assert message["content"] == _MARKUP_DOC
+ assert not message.get("reasoning_content")
+
+
class _VisionToolLoopBackend(_ToolLoopBackend):
def __init__(self, responder, **kwargs):
super().__init__(responder, **kwargs)
diff --git a/studio/backend/tests/test_think_prefill_reemit.py b/studio/backend/tests/test_think_prefill_reemit.py
index 3ad32472aef..d05809ee974 100644
--- a/studio/backend/tests/test_think_prefill_reemit.py
+++ b/studio/backend/tests/test_think_prefill_reemit.py
@@ -106,6 +106,14 @@ def test_guard_suppresses_when_close_tag_is_special():
assert detect_think_prefill(QWEN_PROMPT + "<think>\n", specials) == ""
+def test_guard_follows_the_path_s_rendering_not_the_token_metadata():
+ specials = ["<|im_end|>", "<think>", "</think>"]
+ assert (
+ detect_think_prefill(QWEN_PROMPT + "<think>\n", specials, preserves_think_close = True)
+ == "<think>\n"
+ )
+
+
def test_guard_emits_when_think_not_special():
specials = ["<|im_end|>", "<|endoftext|>"]
assert detect_think_prefill(QWEN_PROMPT + "<think>\n", specials) == "<think>\n"
diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py
index 10d69d618b6..581a1b1cd83 100644
--- a/studio/install_python_stack.py
+++ b/studio/install_python_stack.py
@@ -9745,6 +9745,8 @@ def _has_working_git() -> bool:
# _MLX_INSTALL_SPECS.
_MLX_PINS: tuple[str, ...] = ("mlx==0.32.2", "mlx-metal==0.32.2", "mlx-lm==0.31.3")
_MLX_VLM_SPEC = "mlx-vlm>=0.4.4,<=0.7.1"
+# Exact: llguidance.mlx / llguidance.hf are the API grammar_constraint.py binds to.
+_LLGUIDANCE_PIN = "llguidance==1.8.0"
_MLX_NAMES: tuple[str, ...] = tuple(spec.partition("==")[0] for spec in _MLX_PINS) + ("mlx-vlm",)
@@ -11383,6 +11385,8 @@ def install_python_stack() -> int:
# declared mlx-vlm range the step above honoured. Same gate, so the slot is spent on
# every Apple Silicon run with torch, including the no-wheel branch.
base_total += 1 # MLX stack re-resolve
+ if IS_MAC_ARM:
+ base_total += 1 # MLX grammar engine (step 11d), same gate as the step itself
if NO_TORCH and not skip_base:
# no-torch runtime deps, which this build announces on its own slot inside the core
# step rather than folding into it. Same gate as the step itself.
@@ -11903,6 +11907,21 @@ def install_python_stack() -> int:
# rather than no install.
_diffusers_main_step()
+ # 11d. Apple Silicon grammar engine, outside skip_base (install.sh always skips base); failure only loses MLX response_format.
+ if IS_MAC_ARM:
+ if not _full_deps_requested() and _exact_distribution_spec_is_installed(_LLGUIDANCE_PIN):
+ _progress("MLX grammar engine (satisfied, skipped)")
+ else:
+ _progress("MLX grammar engine")
+ try:
+ pip_install(
+ "Installing the MLX grammar engine (llguidance)",
+ "--no-cache-dir",
+ _LLGUIDANCE_PIN,
+ )
+ except SystemExit:
+ _note(f"{_LLGUIDANCE_PIN} failed to install; MLX response_format stays unavailable")
+
# 12. Patch metadata for single-env compatibility
_finalize_ran = _dd_deps_ran or _dd_ran or _patch_metadata_is_pending()
_progress("finalizing" if _finalize_ran else "finalizing (satisfied, skipped)")
diff --git a/tests/studio/install/test_dependency_pass_skips.py b/tests/studio/install/test_dependency_pass_skips.py
index 6888848fd71..a8eeca55f6e 100644
--- a/tests/studio/install/test_dependency_pass_skips.py
+++ b/tests/studio/install/test_dependency_pass_skips.py
@@ -2311,3 +2311,14 @@ def test_a_temp_copy_that_cannot_be_unlinked_does_not_end_the_pass(audited, monk
# The gate's own cleanup is the same shape, one step earlier: it must answer, not raise.
stack._PASS_EVIDENCE = {"pass_inputs": {}, "step_results": {}}
assert stack._requirements_satisfied(audited / "studio.txt", no_deps = False) is False
+
+
+def test_the_mlx_grammar_engine_is_pinned_skippable_and_never_fatal() -> None:
+ """An unpinned --upgrade re-ran on every update, and its SystemExit aborted the install."""
+ assert re.fullmatch(r"llguidance==\d+\.\d+\.\d+", stack._LLGUIDANCE_PIN)
+ source = STACK_PATH.read_text(encoding = "utf-8")
+ step = source[source.index("# 11d.") : source.index("# 12. Patch metadata")]
+ assert "_exact_distribution_spec_is_installed(_LLGUIDANCE_PIN)" in step
+ assert "not _full_deps_requested() and" in step
+ assert "except SystemExit:" in step
+ assert "--upgrade" not in step