Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Batched serving on the MLX path: several replies decoding at once (#10310)
Long Yixing1232012651a9
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.pyindex caadada73d7..cd11322285f 100644--- a/studio/backend/core/inference/llama_server_args.py+++ b/studio/backend/core/inference/llama_server_args.py@@ -27,6 +27,19 @@ logger = logging.getLogger(__name__)PARALLEL_MIN = 1PARALLEL_MAX = 64+PARALLEL_DEFAULT = 4+++def clamp_parallel_slots(n_parallel) -> int:+ if n_parallel is None:+ return PARALLEL_DEFAULT+ try:+ asked = int(n_parallel)+ except (TypeError, ValueError):+ return PARALLEL_DEFAULT+ return max(PARALLEL_MIN, min(PARALLEL_MAX, asked))++# --batch-size / --ubatch-size range, mirrored by N_BATCH_MIN/MAX in per-model-config.tsBATCH_MIN = 1BATCH_MAX = 65536diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 64bc72512c7..851e14b36f1 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -15,6 +15,7 @@ import timefrom collections import OrderedDictfrom contextlib import ExitStack, closing, contextmanager, nullcontextfrom typing import Optional, Generator+from core.inference.worker import RowRefusedfrom core.inference import context_refusalfrom core.inference.message_content import content_to_textfrom core.inference.native_tool_tokens import (@@ -2501,6 +2502,24 @@ def _make_mlx_presence_penalty_processor(penalty: float):return _processor+def _row_logits_processors(processors):+ """A reply's processors, each starting at the prompt's last token."""+ if not processors:+ return []++ def _shown(processor):+ state = {"offset": None}++ def _narrowed(tokens, logits):+ if state["offset"] is None:+ state["offset"] = max(int(tokens.shape[0]) - 1, 0)+ return processor(tokens[state["offset"] :], logits)++ return _narrowed++ return [_shown(processor) for processor in processors]++def _make_mlx_frequency_penalty_processor(penalty: float):"""Frequency penalty as an mlx_lm/mlx_vlm logits processor. Identical to the presence processorexcept the scatter *accumulates*, so a token repeated N times in the completion is charged N@@ -2706,6 +2725,647 @@ class _VLMMediaBlock:return entries+class _RowStream:+ def __init__(+ self,+ *,+ prefix = "",+ sequences = (),+ normalizer = None,+ decode = None,+ ):+ self._prefix = prefix+ self._sequences = sequences+ self._normalizer = normalizer+ self._decode = decode+ self.token_ids = []+ self.sampled = ""+ self.stopped = False+ self._released = 0+ self._cut = 0+ self._output = prefix++ def feed(+ self,+ token = None,+ text = "",+ ):+ if token is not None:+ self.token_ids.append(token)+ if self._decode is not None:+ self.sampled = self._decode(self.token_ids)+ else:+ self.sampled += text or ""+ if self._sequences:+ self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)+ else:+ self._cut = len(self.sampled)+ if self._normalizer is not None:+ return self._normalize(self.sampled[self._released : self._cut])+ if not self._sequences:+ return [self._prefix + self.sampled]+ if self._decode is not None:+ return []+ if self._cut > self._released:+ self._released = self._cut+ return [self._prefix + self.sampled[: self._cut]]+ return []++ def settle(self):+ if not self._sequences:+ return []+ if not self.stopped:+ self._cut = len(self.sampled)+ if self._normalizer is not None:+ return self._normalize(self.sampled[self._released : self._cut])+ if self._decode is not None:+ settled = self._prefix + self.sampled[: self._cut]+ return [settled] if settled != self._prefix else []+ if not self.stopped and self._released < len(self.sampled):+ return [self._prefix + self.sampled]+ return []++ def drain(self, *, cancelled):+ if self._normalizer is None:+ return []+ drained = cancelled and not self.stopped+ tail = self._normalizer.drain() if drained else self._normalizer.finish()+ return self._append(tail)++ def _normalize(self, raw):+ self._released = self._cut+ return self._append(self._normalizer.feed(raw))++ def _append(self, delta):+ if not delta:+ return []+ self._output += delta+ return [self._output]+++def _messages_with_system_prompt(messages, system_prompt):+ full = [{"role": "system", "content": system_prompt}] if system_prompt else []+ full.extend(messages)+ return full+++def _engine_fields(engine, name):+ return getattr(getattr(engine, name, None), "__dataclass_fields__", ())+++def _asks_for_row_processors(request):+ return (+ _mlx_sampling_processors(+ repetition_penalty = request.get("repetition_penalty", 1.0),+ presence_penalty = request.get("presence_penalty", 0.0),+ frequency_penalty = request.get("frequency_penalty", 0.0),+ logit_bias = request.get("logit_bias"),+ )+ is not None+ )+++def _row_processor_gap(engine, requests):+ """Why these replies' penalties cannot travel into a batch, or None."""+ if not any(_asks_for_row_processors(request) for request in requests):+ return None+ unavailable = getattr(engine, "row_logits_processors_unavailable_reason", None)+ return unavailable() if callable(unavailable) else None+++def _request_batch_gap(request):+ """What a reply asks for that only the one-at-a-time decode serves, or None."""+ if request.get("images") or request.get("images_base64"):+ return "a reply replays tool images"+ if request.get("video") is not None or request.get("video_base64"):+ return "a reply carries a video clip"+ if request.get("tools") or request.get("tool_protocol_active"):+ return "a reply runs the tool protocol"+ if request.get("response_format") is not None:+ return "a reply is constrained to a response format"+ return None+++def _batched_vision_engine_gap(engine):+ def fields(name):+ return _engine_fields(engine, name)++ if not hasattr(engine, "stream_batch"):+ return "the installed unsloth-zoo has no batched vision generation"+ if "seed" not in fields("SamplingParams"):+ return "the installed unsloth-zoo cannot seed a batched reply"+ if "prompt_token_count" not in fields("GenerationResult"):+ return "the installed unsloth-zoo does not report what a batched prompt cost"+ if "logits_processors" not in fields("GenerationRequest"):+ return "the installed unsloth-zoo cannot carry a penalty into a batched reply"+ return None+++class _TextRowPlan:+ __slots__ = (+ "prompt",+ "think_prefix",+ "stream",+ "sampler",+ "processors",+ "max_tokens",+ "detokenizes",+ )++ def __init__(+ self,+ *,+ prompt,+ think_prefix,+ stream,+ sampler,+ processors,+ max_tokens,+ detokenizes = False,+ ):+ self.prompt = prompt+ self.think_prefix = think_prefix+ self.stream = stream+ self.detokenizes = detokenizes+ self.sampler = sampler+ self.processors = processors+ self.max_tokens = max_tokens+++class _TextBatchRow:+ __slots__ = (+ "handle",+ "plan",+ "uid",+ "cache_key",+ "all_tokens",+ "prompt_len",+ "cached",+ "detokenizer",+ "generated",+ "reason",+ "processing_at",+ "ready_at",+ )++ def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached, detokenizer):+ self.handle = handle+ self.plan = plan+ self.uid = None+ self.cache_key = cache_key+ self.all_tokens = all_tokens+ self.prompt_len = prompt_len+ self.cached = cached+ self.detokenizer = detokenizer+ self.generated = 0+ self.reason = None+ self.processing_at = None+ self.ready_at = None+++def _row_kwargs(request):+ """The sampling and render options one batched reply is planned with."""+ return dict(+ temperature = request.get("temperature", 0.7),+ top_p = request.get("top_p", 0.9),+ top_k = request.get("top_k", 40),+ min_p = request.get("min_p", 0.0),+ max_new_tokens = request.get("max_new_tokens", 256),+ repetition_penalty = request.get("repetition_penalty", 1.0),+ tools = request.get("tools"),+ enable_thinking = request.get("enable_thinking"),+ reasoning_effort = request.get("reasoning_effort"),+ preserve_thinking = request.get("preserve_thinking"),+ continue_final_message = request.get("continue_final_message", False),+ presence_penalty = request.get("presence_penalty", 0.0),+ seed = request.get("seed"),+ frequency_penalty = request.get("frequency_penalty", 0.0),+ logit_bias = request.get("logit_bias"),+ stop = request.get("stop"),+ )+++class _TextBatchSession:+ """mlx-lm's batch with the reply set left open."""++ def __init__(+ self,+ backend,+ *,+ width,+ adapter_state = None,+ owns_model = False,+ ):+ from mlx_lm.generate import BatchGenerator++ self.backend = backend+ self._adapter_state = adapter_state+ self._rows = {}+ self._by_uid = {}+ self._settled = {}+ self._held = ExitStack()+ try:+ if owns_model:+ self._held.enter_context(backend._generation_lock)+ self._held.enter_context(+ _temporary_mlx_adapter_state(backend._model, adapter_state)+ )+ self.generator = BatchGenerator(+ backend._model,+ stop_tokens = [+ [token] for token in _mlx_stop_token_ids(backend._tokenizer, backend._model)+ ],+ completion_batch_size = width,+ prefill_batch_size = width,+ )+ except BaseException:+ self._held.close()+ raise++ @property+ def rows_in_flight(self) -> int:+ return len(self._rows)++ @property+ def handles(self):+ return list(self._rows)++ @property+ def ending(self):+ return []++ def take_stats(self, handle):+ """The usage a retired reply settled with, once."""+ return self._settled.pop(handle, None)++ def admit(self, request, handle):+ """Take one reply in, answering with the text preceding its first token."""+ backend = self.backend+ plan = backend._plan_text_row(+ _messages_with_system_prompt(+ request.get("messages") or [],+ request.get("system_prompt", ""),+ ),+ **_row_kwargs(request),+ )+ prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(+ plan.prompt,+ self._adapter_state,+ )+ row = _TextBatchRow(+ handle = handle,+ plan = plan,+ cache_key = cache_key,+ all_tokens = all_tokens,+ prompt_len = len(remainder),+ cached = len(prefix),+ detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,+ )+ uids = self.generator.insert(+ [remainder],+ max_tokens = [plan.max_tokens],+ caches = [cache],+ all_tokens = [prefix],+ samplers = [plan.sampler],+ logits_processors = [_row_logits_processors(plan.processors)],+ state_machines = None,+ )+ if len(uids) != 1:+ self.generator.remove(list(uids))+ raise RuntimeError(+ f"mlx-lm admitted {len(uids)} rows for one reply; a batch cannot "+ "tell whose tokens are whose."+ )+ row.uid = uids[0]+ self._rows[handle] = row+ self._by_uid[row.uid] = row+ logger.info(+ "Admitted reply %s to the batch: prompt=%d, cached=%d, max_tokens=%d, model=%s",+ handle,+ row.prompt_len,+ row.cached,+ plan.max_tokens,+ type(backend._model).__name__,+ )+ return plan.think_prefix++ def step(self):+ """Report what the batch produced, retiring every reply that ended."""+ began = time.perf_counter()+ prompt_events, events = self.generator.next()+ now = time.perf_counter() if (events or prompt_events) else None+ for event in prompt_events:+ row = self._by_uid.get(event.uid)+ if row is not None and row.processing_at is None:+ row.processing_at = began+ for event in events:+ row = self._by_uid.get(event.uid)+ if row is None:+ continue+ if row.processing_at is None:+ row.processing_at = began+ if row.ready_at is None:+ row.ready_at = now+ yield from self._feed(row, event)++ def _feed(self, row, event):+ stream = row.plan.stream+ text = None+ if event.finish_reason is None:+ if row.detokenizer is not None:+ row.detokenizer.add_token(event.token)+ text = row.detokenizer.last_segment+ else:+ text = ""+ if row.detokenizer is not None:+ if event.finish_reason == "length":+ row.detokenizer.add_token(event.token)+ row.detokenizer.finalize()+ text = row.detokenizer.last_segment+ row.generated += 1+ yield from (+ (row.handle, snapshot) for snapshot in stream.feed(token = event.token, text = text)+ )+ if not (stream.stopped or event.finish_reason is not None):+ return+ row.reason = "stop" if stream.stopped else event.finish_reason+ banked = self.generator.remove(+ [row.uid],+ return_prompt_caches = event.finish_reason is None,+ )+ self.backend._bank_row_cache(+ row.cache_key,+ event,+ banked.get(row.uid),+ row.all_tokens,+ stream,+ )+ yield from self._retire(row, cancelled = False)++ def withdraw(self, handles):+ rows = [self._rows[handle] for handle in handles if handle in self._rows]+ if not rows:+ return+ taken = self.generator.remove([row.uid for row in rows], return_prompt_caches = True)+ for row in rows:+ row.reason = "stop"+ self.backend._bank_row_cache(+ row.cache_key,+ None,+ taken.get(row.uid),+ row.all_tokens,+ row.plan.stream,+ )+ yield from self._retire(row, cancelled = True)++ def _retire(self, row, *, cancelled):+ self._rows.pop(row.handle, None)+ self._by_uid.pop(row.uid, None)+ stream = row.plan.stream+ yield from ((row.handle, snap) for snap in stream.settle())+ yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))+ ready_at = row.ready_at or time.perf_counter()+ decoded = time.perf_counter() - ready_at+ prefill_s = ready_at - (row.processing_at or ready_at)+ self._settled[row.handle] = _build_generation_stats(+ row.prompt_len,+ (row.prompt_len / prefill_s) if prefill_s > 0 else 0.0,+ row.generated,+ row.generated / max(decoded, 1e-9),+ row.cached,+ finish_reason = row.reason,+ )+ yield row.handle, None++ def close(self):+ try:+ try:+ if self._by_uid:+ self.generator.remove(list(self._by_uid))+ finally:+ self.generator.close()+ finally:+ self._rows.clear()+ self._by_uid.clear()+ self._settled.clear()+ self._held.close()+++class _VLMRowPlan:+ __slots__ = (+ "prompt",+ "images",+ "think_prefix",+ "stream",+ "kwargs",+ "sampling",+ "max_tokens",+ "processors",+ )++ def __init__(+ self,+ *,+ prompt,+ images,+ think_prefix,+ stream,+ kwargs,+ sampling,+ max_tokens,+ processors = None,+ ):+ self.prompt = prompt+ self.images = images+ self.think_prefix = think_prefix+ self.stream = stream+ self.kwargs = kwargs+ self.sampling = sampling+ self.max_tokens = max_tokens+ self.processors = processors+++class _VisionBatchRow:+ __slots__ = (+ "handle",+ "plan",+ "row",+ "generated",+ "reason",+ "prompt_tokens",+ "admitted_at",+ "ready_at",+ "cancelled",+ )++ def __init__(self, *, handle, plan, row):+ self.handle = handle+ self.plan = plan+ self.row = row+ self.generated = 0+ self.reason = None+ self.prompt_tokens = 0+ self.admitted_at = time.perf_counter()+ self.ready_at = None+ self.cancelled = False+++class _VisionBatchSession:+ def __init__(+ self,+ backend,+ *,+ width,+ adapter_state = None,+ owns_model = False,+ ):+ from unsloth_zoo.mlx.generate import BatchStream, GenerationDefaults++ self.backend = backend+ self._adapter_state = adapter_state+ self._rows = {}+ self._by_row = {}+ self._settled = {}+ self._held = ExitStack()+ try:+ if owns_model:+ self._held.enter_context(backend._generation_lock)+ self._held.enter_context(+ _temporary_mlx_adapter_state(backend._model, adapter_state)+ )+ self.stream = BatchStream(+ backend._model,+ backend._processor,+ defaults = GenerationDefaults(+ prefill_batch_size = 1,+ completion_batch_size = width,+ ),+ )+ except BaseException:+ self._held.close()+ raise++ @property+ def rows_in_flight(self) -> int:+ return len(self._rows)++ @property+ def handles(self):+ return list(self._rows)++ @property+ def ending(self):+ return [handle for handle, row in self._rows.items() if row.cancelled]++ def take_stats(self, handle):+ return self._settled.pop(handle, None)++ def admit(self, request, handle):+ from unsloth_zoo.mlx.generate import (+ BatchRowRefused,+ GenerationRequest,+ SamplingParams,+ )++ backend = self.backend+ plan = backend._plan_vlm_request(request)+ if plan.images:+ backend._release_vlm_snapshots()+ try:+ row_number = self.stream.add(+ GenerationRequest(+ prompt = plan.prompt,+ image = plan.images[0] if plan.images else None,+ max_tokens = plan.max_tokens,+ sampling = SamplingParams(**plan.sampling),+ logits_processors = plan.processors,+ )+ )+ except BatchRowRefused as refusal:+ raise RowRefused(str(refusal)) from refusal+ row = _VisionBatchRow(handle = handle, plan = plan, row = row_number)+ self._rows[handle] = row+ self._by_row[row_number] = row+ logger.info(+ "Admitted vision reply %s to the batch: images=%d, max_tokens=%d, model=%s",+ handle,+ len(plan.images or ()),+ plan.max_tokens,+ type(backend._model).__name__,+ )+ return plan.think_prefix++ def step(self):+ now = None+ for event in self.stream.step():+ row = self._by_row.get(event.index)+ if row is None:+ continue+ if now is None:+ now = time.perf_counter()+ if row.ready_at is None:+ row.ready_at = now+ if event.delta and not row.cancelled:+ yield from (+ (row.handle, snapshot) for snapshot in row.plan.stream.feed(text = event.delta)+ )+ if row.plan.stream.stopped:+ row.cancelled = True+ row.reason = "stop"+ if event.result is not None:+ result = event.result+ if not row.cancelled:+ row.reason = result.finish_reason+ row.prompt_tokens = result.prompt_token_count+ row.generated = len(result.token_ids) + (result.finish_reason == "stop")+ yield from self._retire(row, cancelled = row.cancelled)+ elif row.cancelled:+ yield from self._retake(row)++ def withdraw(self, handles):+ for handle in list(handles):+ row = self._rows.get(handle)+ if row is None:+ continue+ row.cancelled = True+ row.reason = "stop"+ yield from self._retake(row)++ def _retake(self, row):+ managed = self.stream.withdraw(row.row)+ if managed is not None:+ row.prompt_tokens = managed.prompt_token_count+ row.generated = len(managed.token_ids)+ yield from self._retire(row, cancelled = True)++ def _retire(self, row, *, cancelled):+ self._rows.pop(row.handle, None)+ self._by_row.pop(row.row, None)+ stream = row.plan.stream+ yield from ((row.handle, snap) for snap in stream.settle())+ yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))+ ready_at = row.ready_at or time.perf_counter()+ decoded = time.perf_counter() - ready_at+ prefill_s = ready_at - row.admitted_at+ self._settled[row.handle] = _build_generation_stats(+ row.prompt_tokens,+ (row.prompt_tokens / prefill_s) if prefill_s > 0 else 0.0,+ row.generated,+ row.generated / max(decoded, 1e-9),+ finish_reason = row.reason or "stop",+ )+ yield row.handle, None++ def close(self):+ try:+ self.stream.close()+ finally:+ self._rows.clear()+ self._by_row.clear()+ self._settled.clear()+ self._held.close()++class MLXInferenceBackend:def __init__(self):self.models = {}@@ -2716,6 +3376,7 @@ class MLXInferenceBackend:self._generation_lock = threading.Lock()# usage, timings and terminal reason of the latest generation, shipped on gen_done.self.last_generation_stats = None+ self.last_batch_generation_stats = []self._model = Noneself._model_fusion = ExitStack()@@ -3392,6 +4053,7 @@ class MLXInferenceBackend:# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell# from an enforced one."context_length_enforced": _ctx_enforced,+ "context_unbounded_when_batched": self._vision_batch_is_available(),"mlx_context_budget": self._kv_context_budget,"mlx_kv_bits": self._kv_quant["kv_bits"],"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],@@ -3589,6 +4251,24 @@ class MLXInferenceBackend:full_messages.extend(messages)return full_messages+ def _conversation_to_render(self, messages, system_prompt, image):+ """The turns this request renders, whichever way its reply is decoded."""+ if self._is_vlm and image is not None:+ from core.inference.chat_template_helpers import (+ chat_render_target as _chat_render_target,+ )+ full_messages = messages_with_attached_image(+ messages,+ system_prompt = system_prompt,+ structured_content = (+ self._processor is not None+ and _chat_render_target(self._processor) is self._processor+ ),+ )+ else:+ full_messages = self._with_system_prompt(messages, system_prompt)+ return full_messages+def count_chat_tokens(self,messages,@@ -4091,6 +4771,109 @@ class MLXInferenceBackend:if stopped:self._mark_stopped()+ def _plan_text_row(+ self,+ messages,+ *,+ temperature,+ top_p,+ top_k,+ min_p,+ max_new_tokens,+ repetition_penalty,+ tools = None,+ enable_thinking = None,+ reasoning_effort = None,+ preserve_thinking = None,+ continue_final_message = False,+ presence_penalty = 0.0,+ seed = None,+ frequency_penalty = 0.0,+ logit_bias = None,+ stop = None,+ tool_protocol_active = None,+ ) -> "_TextRowPlan":+ from mlx_lm.sample_utils import make_sampler++ from core.inference.chat_template_helpers import detect_think_prefill++ render_result = self._render_text_prompt(+ messages,+ tools = tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ )+ prompt = render_result.prompt+ reasoning_channel_markers = render_result.reasoning_channel_markers+ _resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))++ think_prefix = detect_think_prefill(+ prompt,+ getattr(self._tokenizer, "all_special_tokens", None),+ preserves_think_close = (+ bool(tools) or tool_protocol_active or reasoning_channel_markers is not None+ )+ and decoder_preserves_token(+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)+ ),+ )+ if seed is None:+ sampler = make_sampler(+ temp = temperature,+ top_p = top_p,+ top_k = int(top_k or 0),+ min_p = float(min_p or 0.0),+ min_tokens_to_keep = 1,+ )+ else:+ sampler = _make_seeded_mlx_sampler(+ seed,+ temp = temperature,+ top_p = top_p,+ top_k = int(top_k or 0),+ min_p = float(min_p or 0.0),+ )+ logits_processors = _mlx_sampling_processors(+ repetition_penalty = repetition_penalty,+ presence_penalty = presence_penalty,+ frequency_penalty = frequency_penalty,+ logit_bias = logit_bias,+ )++ normalizer = (+ make_reasoning_normalizer(+ reasoning_channel_markers,+ in_reasoning = prompt_opens_reasoning_channel(+ prompt, reasoning_channel_markers, _resumed_partial+ ),+ )+ if reasoning_channel_markers is not None+ else None+ )+ row = _RowStream(+ prefix = think_prefix,+ sequences = _mlx_stop_sequences(stop),+ normalizer = normalizer,+ decode = (+ None+ if normalizer is not None+ else lambda ids: self._tokenizer.decode(ids, skip_special_tokens = True)+ ),+ )+ return _TextRowPlan(+ prompt = prompt,+ think_prefix = think_prefix,+ stream = row,+ detokenizes = normalizer is not None,+ sampler = sampler,+ processors = logits_processors,+ max_tokens = (+ self._unset_generation_budget(prompt) if max_new_tokens is None else max_new_tokens+ ),+ )+def _render_vlm_prompt(self,messages,@@ -4223,6 +5006,118 @@ class MLXInferenceBackend:)return prompt, chat_target+ def _plan_vlm_request(self, request) -> "_VLMRowPlan":+ image = request.get("image")+ messages = request.get("messages") or []+ conversation = self._conversation_to_render(+ messages, request.get("system_prompt", ""), image+ )+ return self._plan_vlm_row(conversation, image, **_row_kwargs(request))++ def _plan_vlm_row(+ self,+ messages,+ image,+ *,+ temperature,+ top_p,+ top_k,+ min_p,+ max_new_tokens,+ repetition_penalty,+ tools = None,+ enable_thinking = None,+ reasoning_effort = None,+ preserve_thinking = None,+ continue_final_message = False,+ presence_penalty = 0.0,+ seed = None,+ frequency_penalty = 0.0,+ logit_bias = None,+ stop = None,+ ) -> "_VLMRowPlan":+ images = [image] if image is not None else None+ prompt, chat_target = self._render_vlm_prompt(+ messages,+ images,+ tools = tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ )++ from core.inference.chat_template_helpers import detect_think_prefill++ think_prefix = detect_think_prefill(+ prompt, getattr(chat_target, "all_special_tokens", None)+ )+ markers = detect_reasoning_channel_markers(chat_target, tools = tools)+ normalizer = (+ make_reasoning_normalizer(+ markers,+ in_reasoning = prompt_opens_reasoning_channel(+ prompt,+ markers,+ bool(continue_final_message and trailing_assistant_text(messages)),+ ),+ )+ if markers is not None+ else None+ )+ row = _RowStream(+ prefix = think_prefix,+ sequences = _mlx_stop_sequences(stop),+ normalizer = normalizer,+ )+ if max_new_tokens is None:+ max_new_tokens = self._unset_generation_budget(prompt)+ if images:+ max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)+ vlm_kwargs = dict(+ max_tokens = max_new_tokens,+ temperature = temperature,+ top_p = top_p,+ top_k = int(top_k or 0),+ min_p = float(min_p or 0.0),+ )+ vlm_kwargs.update(self._kv_window_generate_kwargs())+ if seed is not None:+ vlm_kwargs["sampler"] = _make_seeded_mlx_sampler(+ seed,+ temp = temperature,+ top_p = top_p,+ top_k = int(top_k or 0),+ min_p = float(min_p or 0.0),+ )+ processors = _mlx_sampling_processors(+ repetition_penalty = repetition_penalty,+ presence_penalty = presence_penalty,+ frequency_penalty = frequency_penalty,+ logit_bias = logit_bias,+ )+ if processors:+ # Never the repetition_penalty shortcut: mlx-vlm would build that processor+ # itself, outside the narrowing every other path applies.+ vlm_kwargs["logits_processors"] = _row_logits_processors(processors)++ return _VLMRowPlan(+ prompt = prompt,+ images = images,+ think_prefix = think_prefix,+ stream = row,+ kwargs = vlm_kwargs,+ sampling = {+ "temperature": vlm_kwargs["temperature"],+ "top_p": vlm_kwargs["top_p"],+ "top_k": max(vlm_kwargs["top_k"], 0),+ "min_p": vlm_kwargs["min_p"],+ "seed": seed,+ },+ max_tokens = vlm_kwargs["max_tokens"],+ processors = processors,+ )+def _generate_vlm(self,messages,@@ -4555,6 +5450,295 @@ class MLXInferenceBackend:if stopped:self._mark_stopped()+ def _kv_policy_batch_reason(self):+ if self._kv_quant_bits() is not None or getattr(self, "_kv_context_budget", None):+ return "the load quantizes or budgets its KV cache, which a batch does not carry"+ return None++ def batch_unavailable_reason(self, requests):+ if self._model is None:+ return "no model is loaded"+ for request in requests:+ reason = _request_batch_gap(request)+ if reason is not None:+ return reason+ reason = self._kv_policy_batch_reason()+ if reason is not None:+ return reason+ if len(requests) < 2:+ return "fewer than two replies were requested"+ if self._is_vlm:+ reason = self._vlm_batch_unavailable_reason(requests)+ if reason is not None:+ return reason+ if any(_mlx_stop_sequences(request.get("stop")) for request in requests):+ return "a reply asks for stop sequences"+ return None+ return self._text_batch_unavailable_reason()++ def resident_unavailable_reason(self, request):+ if self._model is None:+ return "no model is loaded"+ reason = _request_batch_gap(request)+ if reason is not None:+ return reason+ reason = self._kv_policy_batch_reason()+ if reason is not None:+ return reason+ if self._is_vlm:+ return self._vlm_resident_unavailable_reason(request)+ return self._text_batch_unavailable_reason()++ def _text_batch_unavailable_reason(self):+ """Why the installed mlx-lm cannot batch this load's replies, or None."""+ try:+ from unsloth_zoo.mlx.generate import stream_unavailable_reason+ except ImportError:+ return "the installed unsloth-zoo cannot keep a batch open"+ return stream_unavailable_reason(self._model, self._tokenizer)++ def _vlm_resident_unavailable_reason(self, request):+ reason = self._vlm_batch_unavailable_reason([request])+ if reason is not None:+ return reason+ try:+ from unsloth_zoo.mlx.generate import stream_unavailable_reason+ except ImportError:+ return "the installed unsloth-zoo cannot keep a vision batch open"+ return stream_unavailable_reason(self._model, self._processor)++ def open_resident_batch(+ self,+ *,+ width,+ adapter_state = None,+ ):+ reason = self.resident_unavailable_reason({})+ if reason is not None:+ raise RuntimeError(f"MLX batched generation is unavailable: {reason}")+ session = _VisionBatchSession if self._is_vlm else _TextBatchSession+ return session(+ self,+ width = width,+ adapter_state = adapter_state,+ owns_model = True,+ )++ def _vision_batch_is_available(self) -> bool:+ """Whether this load will decode vision replies together, asked without a request."""+ if not self._is_vlm:+ return False+ return (+ self.batch_unavailable_reason([{}, {}]) is None+ or self.resident_unavailable_reason({}) is None+ )++ def _vlm_batch_unavailable_reason(self, requests):+ try:+ from unsloth_zoo.mlx import generate as engine+ except ImportError:+ return "the installed unsloth-zoo has no MLX generation"+ gap = _batched_vision_engine_gap(engine)+ if gap is not None:+ return gap+ gap = _row_processor_gap(engine, requests)+ if gap is not None:+ return gap+ if not getattr(self._model, "_is_vlm_model", False):+ return "this model was not loaded as a vision model"+ return None++ def generate_chat_batch(+ self,+ requests,+ *,+ cancel_event = None,+ _adapter_state = None,+ ):+ reason = self.batch_unavailable_reason(requests)+ if reason is not None:+ raise RuntimeError(f"MLX batched generation is unavailable: {reason}")++ self.last_batch_generation_stats = [None] * len(requests)+ self.last_generation_stats = None++ run = self._generate_vlm_batch if self._is_vlm else self._generate_text_batch+ yield from run(requests, cancel_event = cancel_event, _adapter_state = _adapter_state)++ def _generate_text_batch(+ self,+ requests,+ *,+ cancel_event = None,+ _adapter_state = None,+ ):+ def report(events):+ for row, snapshot in events:+ if snapshot is None:+ self.last_batch_generation_stats[row] = session.take_stats(row)+ yield row, snapshot++ with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):+ session = _TextBatchSession(+ self,+ width = len(requests),+ adapter_state = _adapter_state,+ )+ try:+ prefixes = [+ (row, session.admit(request, row)) for row, request in enumerate(requests)+ ]+ for row, prefix in prefixes:+ if prefix:+ yield row, prefix+ while session.rows_in_flight:+ yield from report(session.step())+ if cancel_event is not None and cancel_event.is_set():+ yield from report(session.withdraw(list(session.handles)))+ finally:+ session.close()++ def _generate_vlm_batch(+ self,+ requests,+ *,+ cancel_event = None,+ _adapter_state = None,+ ):+ import time++ from unsloth_zoo.mlx.generate import (+ GenerationDefaults,+ GenerationRequest,+ SamplingParams,+ stream_batch,+ )++ plans = [self._plan_vlm_request(request) for request in requests]+ batch = [+ GenerationRequest(+ prompt = plan.prompt,+ image = plan.images[0] if plan.images else None,+ max_tokens = plan.max_tokens,+ sampling = SamplingParams(**plan.sampling),+ logits_processors = plan.processors,+ )+ for plan in plans+ ]+ defaults = GenerationDefaults(+ max_tokens = max(plan.max_tokens for plan in plans),+ prefill_batch_size = len(plans),+ completion_batch_size = len(plans),+ )++ with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):+ if any(plan.images for plan in plans):+ self._release_vlm_snapshots()+ logger.info(+ "Generating %d vision replies as one batch: images=%d, max_tokens=%s, model=%s",+ len(plans),+ sum(1 for plan in plans if plan.images),+ [plan.max_tokens for plan in plans],+ type(self._model).__name__,+ )+ for row, plan in enumerate(plans):+ if plan.think_prefix:+ yield row, plan.think_prefix+ started = time.perf_counter()+ prefilled_at: list[float | None] = [None] * len(plans)+ counted: dict[int, tuple[int, int]] = {}+ pending = set(range(len(plans)))+ events = stream_batch(self._model, self._processor, batch, defaults = defaults)++ def _retire(row, result, *, cancelled):+ stream = plans[row].stream+ yield from ((row, snap) for snap in stream.settle())+ yield from ((row, snap) for snap in stream.drain(cancelled = cancelled))+ ready_at = prefilled_at[row] or time.perf_counter()+ decoded = time.perf_counter() - ready_at+ prefill_s = ready_at - started+ seen_prompt, seen_generated = counted.get(row, (0, 0))+ prompt_n = result.prompt_token_count if result is not None else seen_prompt+ generated = (+ len(result.token_ids) + (result.finish_reason == "stop")+ if result is not None+ else seen_generated+ )+ self.last_batch_generation_stats[row] = _build_generation_stats(+ prompt_n,+ (prompt_n / prefill_s) if prefill_s > 0 else 0.0,+ generated,+ generated / max(decoded, 1e-9),+ finish_reason = (result.finish_reason if result is not None else "stop"),+ )+ yield row, None++ try:+ for event in events:+ row = event.index+ if row not in pending:+ continue+ if prefilled_at[row] is None:+ prefilled_at[row] = time.perf_counter()+ counted[row] = (+ getattr(event, "prompt_tokens", 0),+ getattr(event, "generated_tokens", 0),+ )+ if event.delta:+ yield from (+ (row, snapshot) for snapshot in plans[row].stream.feed(text = event.delta)+ )+ if event.result is not None:+ pending.discard(row)+ yield from _retire(row, event.result, cancelled = False)+ if cancel_event is not None and cancel_event.is_set():+ break+ if pending and not (cancel_event is not None and cancel_event.is_set()):+ raise RuntimeError(+ f"Batched vision generation ended with {len(pending)} of "+ f"{len(plans)} replies unfinished."+ )+ for row in sorted(pending):+ yield from _retire(row, None, cancelled = True)+ finally:+ close = getattr(events, "close", None)+ if callable(close):+ close()++ def _batch_prompt_row(self, prompt, adapter_state):+ rest, cache, key, tokens, cached_n = self._prepare_prompt_cache(prompt, adapter_state)+ if cache is None or tokens is None:+ tokens = self._encode_prompt(prompt)+ rest, cached_n, key = tokens, 0, None+ cache = self._make_row_cache()+ return list(tokens[:cached_n]), list(rest), cache, key, list(tokens)++ def _make_row_cache(self):+ """A row's own cache: mlx-lm applies ``max_kv_size`` only to caches it builds."""+ from mlx_lm.models.cache import make_prompt_cache++ window = getattr(self, "_kv_cache_window", None)+ if window is None:+ return make_prompt_cache(self._model)+ return make_prompt_cache(self._model, max_kv_size = window)++ def _bank_row_cache(self, cache_key, event, withdrawn, prompt_tokens, stream):+ history = self._prompt_cache_history+ if history is None or cache_key is None:+ return+ cache = getattr(event, "prompt_cache", None)+ tokens = getattr(event, "all_tokens", None)+ if cache is None and withdrawn is not None:+ cache, tokens = withdrawn+ if cache is None:+ return+ if not tokens:+ tokens = prompt_tokens + stream.token_ids+ try:+ history.insert(cache_key, list(tokens), cache)+ except Exception as exc:+ logger.debug("MLX prompt cache insert failed: %s", exc)+def generate_audio_input_response(self,messages,@@ -4687,7 +5871,6 @@ class MLXInferenceBackend:max_new_tokens,),)- # As in _generate_text: what was withheld is ordinary text now.if sequences and not stopped:delta = sampled[released:]if normalizer is not None:diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 2ec80c61d12..71fcf282611 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -9,6 +9,7 @@ Pattern follows core/training/training.py."""import atexitimport base64+import contextlibimport osimport signalfrom loggers import get_logger@@ -30,6 +31,7 @@ from core.inference.audio_errors import (AudioBackendUnsupportedError,AudioGenerationCancelledError,)+from core.inference.worker import PendingTeardowns, StopLedgerfrom utils.hardware import get_device, prepare_gpu_selectionfrom utils.utils import hf_env_offline, is_metal_queue_dead@@ -191,10 +193,13 @@ _CTX = mp.get_context("spawn")_DISPATCH_READ_TIMEOUT = 30.0++_STOP_NOTICE_INTERVAL = 0.1_DISPATCH_POLL_INTERVAL = 0.5_DISPATCH_STOP_TIMEOUT = 5.0_DISPATCH_IDLE_TIMEOUT = 30.0_DISPATCH_DRAIN_TIMEOUT = 5.0+_CANCELLED_ROWS_GRACE = 5.0# Only bounds the Transformers subprocess path; llama.cpp TTS never reaches here. 120s was tuned against GGUF speeds# and killed real work: a safetensors LoRA on a mid-range GPU needs minutes for the same clip a GGUF returns in@@ -370,6 +375,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:"max_context_length": model_info.get("max_context_length"),"requested_context_length": model_info.get("requested_context_length"),"context_length_enforced": model_info.get("context_length_enforced"),+ "context_unbounded_when_batched": model_info.get("context_unbounded_when_batched"),"mlx_context_budget": model_info.get("mlx_context_budget"),}@@ -390,6 +396,8 @@ class InferenceOrchestrator:# Set for the whole unload; the worker never clears it (unlike _cancel_event), so a generate queued behind the# cancelled one is skipped, not run.self._drain_event: Any = None+ self._stop_ledger: Any = None+ self._pending_teardowns: Any = Noneself._gen_lock = threading.Lock() # Serializes generation# Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the running generation or is# queued behind it (the worker's event is shared).@@ -397,16 +405,11 @@ class InferenceOrchestrator:self._executing_cancel_events: list = []self._active_cancel_lock = threading.Lock()# Held across claim + _send_cmd so claim order matches the subprocess dequeue order, which _owns_worker relies- # on.- self._send_order_lock = threading.Lock()+ self._send_order_lock = threading.RLock()# Set during a switch so a generation winning the _gen_lock handoff bails instead of starting on the outgoing# modelself._unload_pending = False- # Blocking TTS has no token response that can safely establish worker ownership while it is queued behind- # compare-mode work. Reserve the dispatcher admission lane for the whole TTS request so its shared cancel- # event cannot hit a sibling. Mutated under _dispatcher_lifecycle_lock while holding _gen_lock.- self._exclusive_tts_pending = False- self._exclusive_vram_probe_pending = False+ self._worker_reserved_for: Optional[str] = None# Dispatcher state for compare mode (adapter-controlled requests): bypass _gen_lock, send commands directly,# read from per-request mailboxes routed by a dispatcher thread on request_id.@@ -426,6 +429,7 @@ class InferenceOrchestrator:# and each spawn one, orphaning the extra thread (self._dispatcher_thread tracks only the last). The orphan# later steals the "unloaded" reply off resp_queue and hangs unload_model.self._dispatcher_lifecycle_lock = threading.Lock()+ self._worker_released = threading.Condition(self._dispatcher_lifecycle_lock)# Local state mirrors (updated from subprocess responses)self.active_model_name: Optional[str] = None@@ -495,6 +499,23 @@ class InferenceOrchestrator:self._top_models_started = Truethreading.Thread(target = self._fetch_top_models, daemon = True, name = "top-models").start()+ def set_parallel_slots(self, n_parallel) -> int:+ from core.inference.llama_server_args import clamp_parallel_slots++ slots = clamp_parallel_slots(n_parallel)+ entry = self.models.get(self.active_model_name or "")+ if entry is not None:+ entry["parallel_slots"] = slots+ return slots++ @property+ def effective_parallel_slots(self) -> int:+ from core.inference.llama_server_args import PARALLEL_DEFAULT++ entry = self.models.get(self.active_model_name or "") or {}+ slots = entry.get("parallel_slots")+ return slots if isinstance(slots, int) and slots > 0 else PARALLEL_DEFAULT+@propertydef default_models(self) -> list[str]:self._refresh_static_models_if_stale()@@ -614,6 +635,8 @@ class InferenceOrchestrator:self._resp_queue = _CTX.Queue()self._cancel_event = _CTX.Event()self._drain_event = _CTX.Event()+ self._stop_ledger = StopLedger(_CTX)+ self._pending_teardowns = PendingTeardowns(_CTX)# Built into a local FIRST, and started through that local. A shutdown can# observe a not-yet-alive child, clear self._proc and finish its sweep while@@ -626,6 +649,8 @@ class InferenceOrchestrator:"resp_queue": self._resp_queue,"cancel_event": self._cancel_event,"drain_event": self._drain_event,+ "stop_ledger": self._stop_ledger,+ "pending_teardowns": self._pending_teardowns,"config": config,}if self._stderr_capture is not None:@@ -684,10 +709,18 @@ class InferenceOrchestrator:logger.info("Inference subprocess started (pid=%s)", _spawned_proc.pid)def _cancel_generation(self) -> None:- """Cancel any ongoing generation in the subprocess (instant)."""if self._cancel_event is not None:self._cancel_event.set()+ def _cancel_every_generation(self) -> None:+ self._teardown_going_out()+ self._cancel_generation()+ try:+ self._send_cmd({"type": "cancel"})+ except Exception:+ self._teardown_not_sent()+ logger.debug("Could not tell the worker to let go of its batch", exc_info = True)+def is_worker_alive(self) -> bool:"""True while the inference subprocess is running, even with no model active (a failed loadcan leave a live worker holding sidecar modules)."""@@ -708,12 +741,12 @@ class InferenceOrchestrator:return Nonewith self._dispatcher_lifecycle_lock:- if self._exclusive_tts_pending or getattr(self, "_exclusive_vram_probe_pending", False):+ if self._worker_reserved_for is not None:return None- self._exclusive_vram_probe_pending = True+ self._worker_reserved_for = "model switch is checking GPU memory"acquired = Falsetry:- if not self._wait_dispatcher_idle():+ if not self._wait_worker_idle():return Noneacquired = self._gen_lock.acquire(timeout = 5.0)if not acquired:@@ -767,7 +800,7 @@ class InferenceOrchestrator:if acquired:self._gen_lock.release()with self._dispatcher_lifecycle_lock:- self._exclusive_vram_probe_pending = False+ self._worker_reserved_for = Nonedef _shutdown_subprocess(self, timeout: float = 10.0) -> bool:with self._subprocess_shutdown_lock:@@ -787,6 +820,7 @@ class InferenceOrchestrator:self._proc = Nonereturn True+ self._teardown_going_out()self._cancel_generation()time.sleep(0.5)@@ -795,7 +829,7 @@ class InferenceOrchestrator:try:self._cmd_queue.put({"type": "shutdown"})except (OSError, ValueError):- pass+ self._teardown_not_sent()try:self._proc.join(timeout = timeout)@@ -841,6 +875,8 @@ class InferenceOrchestrator:self._resp_queue = Noneself._cancel_event = Noneself._drain_event = None+ self._stop_ledger = None+ self._pending_teardowns = Noneself._reset_worker_scoped_state()logger.info("Inference subprocess shut down")return True@@ -925,11 +961,7 @@ class InferenceOrchestrator:return not expected_free_gbdef _reset_worker_scoped_state(self) -> None:- """Drop bookkeeping that only means anything for the worker that just died. Ownership is- scoped by cancel-event identity alone, so a consumer still blocked on its mailbox when- the process was replaced stayed recorded as the executor, and a generation on the fresh- worker then failed _owns_worker and could not be stopped. Mailboxes go too: nothing will- ever route to them, and a stale one reads as compare activity to the unload path."""+ """Drop bookkeeping that only means anything for the worker that just died."""with self._active_cancel_lock:self._active_cancel_events.clear()self._executing_cancel_events.clear()@@ -1014,6 +1046,14 @@ class InferenceOrchestrator:except Exception:return ""+ def _teardown_going_out(self) -> None:+ if self._pending_teardowns is not None:+ self._pending_teardowns.sending()++ def _teardown_not_sent(self) -> None:+ if self._pending_teardowns is not None:+ self._pending_teardowns.unsent()+def _ensure_subprocess_alive(self) -> bool:return self._proc is not None and self._proc.is_alive()@@ -1207,7 +1247,11 @@ class InferenceOrchestrator:except (EOFError, OSError, ValueError):return events- def _direct_reader(self, request_id: str):+ def _direct_reader(+ self,+ request_id: str,+ cancel_event = None,+ ):"""Response reader for a _gen_lock generation, safe once compare exists.The dispatcher and this reader would otherwise both consume _resp_queue. A dispatcher@@ -1221,6 +1265,8 @@ class InferenceOrchestrator:mailbox = _WorkerMailbox(self._proc)with self._mailbox_lock:self._direct_mailboxes[request_id] = mailbox+ if cancel_event is not None:+ self._request_cancel_events[request_id] = cancel_eventdef read_one(timeout: float = 1.0):try:@@ -1276,6 +1322,8 @@ class InferenceOrchestrator:def release() -> None:with self._mailbox_lock:self._direct_mailboxes.pop(request_id, None)+ if cancel_event is not None:+ self._request_cancel_events.pop(request_id, None)return read_one, drain, release@@ -1321,6 +1369,7 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ rows: Optional[list] = None,video_b64: Optional[str] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,@@ -1343,6 +1392,7 @@ class InferenceOrchestrator:"presence_penalty": presence_penalty,"frequency_penalty": frequency_penalty,"logit_bias": logit_bias,+ "parallel_slots": self.effective_parallel_slots,}if seed is not None:cmd["seed"] = seed@@ -1365,6 +1415,8 @@ class InferenceOrchestrator:cmd["preserve_thinking"] = preserve_thinkingif continue_final_message:cmd["continue_final_message"] = True+ if rows:+ cmd["rows"] = rowsif tool_protocol_active is not None:cmd["tool_protocol_active"] = tool_protocol_activereturn cmd@@ -1375,29 +1427,53 @@ class InferenceOrchestrator:drain_on_cancel,*,crash_context: str,+ request_id: str = "",cancel_event = None,stats_holder: Optional[dict] = None,read_timeout: float = 30.0,mark_started: bool = True,- ) -> Generator[str, None, None]:- """Yield tokens from a response stream until gen_done/gen_error. ``read_one(timeout)``- returns the next response (or None on timeout) and owns the queue choice -- the shared- resp_queue under _gen_lock, or a per-request mailbox on the dispatcher path -- so this- loop stays agnostic of which queue is read. On cancel, ``drain_on_cancel()`` consumes the- cancel ack from that same source so stale events don't leak into the next request."""+ rows: Optional[int] = None,+ ) -> Generator[Any, None, None]:+ """Yield tokens from a response stream until gen_done/gen_error."""# Latch this stream's subprocess/queue: if a wedged worker is torn down and a later load spawns a fresh one,# bail rather than re-block on the new queue under _gen_lock (deadlock).initial_proc = self._procinitial_resp_queue = self._resp_queue+ reading_on_until = None+ stop_recorded = False+ stop_sent = Falsewhile True:if self._proc is not initial_proc or self._resp_queue is not initial_resp_queue:+ if stop_sent:+ return# No tail here whatever the lists say: the capture is the REPLACEMENT's.detail = self._subprocess_crash_message(crash_context)yield GenStreamError(f"Error: {detail}", public = True)return- resp = read_one(read_timeout)+ if not stop_sent and cancel_event is not None:+ if cancel_event.is_set():+ stop_recorded, stop_sent = self._stop_and_signal(+ request_id,+ cancel_event,+ stop_recorded,+ may_signal = False,+ )+ if stop_sent and rows is not None and reading_on_until is None:+ reading_on_until = time.monotonic() + _CANCELLED_ROWS_GRACE+ timeout = read_timeout+ if not stop_recorded and cancel_event is not None:+ timeout = min(timeout, _STOP_NOTICE_INTERVAL)+ if reading_on_until is not None:+ remaining = reading_on_until - time.monotonic()+ if remaining <= 0:+ drain_on_cancel()+ return+ timeout = min(timeout, remaining)+ resp = read_one(timeout)if resp is None:if not self._ensure_subprocess_alive():+ if stop_sent:+ return# Only the request the worker was RUNNING gets its last words; the rest were queued behind it.detail = self._subprocess_crash_message(crash_context, with_worker_output = self._owns_worker(cancel_event)@@ -1409,32 +1485,52 @@ class InferenceOrchestrator:rtype = resp.get("type", "")if rtype == "status":continue- # The worker is answering THIS request, so it is the one executing: only now may its cancel event speak- # for the shared worker one. The dispatched path opts out: its dispatcher already did this in worker- # order, which a mailbox read can lag behind.- if mark_started:+ if mark_started and not stop_sent:self._mark_worker_started(cancel_event)# Subprocess-level error (no request_id); request-scoped failures arrive as gen_error belowif rtype == "error" and not resp.get("request_id"):+ if stop_sent:+ returnyield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")return- if rtype == "token":- if cancel_event is not None and cancel_event.is_set():- # Same rule as reset_generation_state: the shared worker event may only be set by the generation- # the worker is running. A dispatched request can still be draining stale mailbox tokens after the- # dispatcher retired it, and signalling from here would end the next one instead. Tearing this- # stream down is always safe, so the local drain happens either way.- if self._owns_worker(cancel_event):- self._cancel_generation()+ if rtype == "token" and cancel_event is not None and cancel_event.is_set():+ if not stop_sent:+ stop_recorded, stop_sent = self._stop_and_signal(+ request_id,+ cancel_event,+ stop_recorded,+ )+ if stop_sent and rows is not None and reading_on_until is None:+ reading_on_until = time.monotonic() + _CANCELLED_ROWS_GRACE+ if rows is None:drain_on_cancel()return- yield resp.get("text", "")- elif rtype == "gen_done":+ # Several replies read on for their completions, but a token drawn+ # after the Stop is one the same reply alone would never have shown.+ continue++ if rtype == "row_done" and rows is not None:if stats_holder is not None:+ reported = stats_holder.setdefault("stats", [None] * rows)+ row = int(resp.get("row", 0))+ if 0 <= row < len(reported):+ reported[row] = resp.get("stats")+ yield int(resp.get("row", 0)), None+ elif rtype == "token":+ if rows is not None:+ yield int(resp.get("row", 0)), resp.get("text", "")+ else:+ yield resp.get("text", "")+ elif rtype == "gen_done":+ if stats_holder is not None and rows is None:stats_holder["stats"] = resp.get("stats")+ self._forget_request(request_id, cancel_event)returnelif rtype == "gen_error":+ self._forget_request(request_id, cancel_event)+ if stop_sent:+ return_budget = resp.get("context_budget")if _budget:# Rebuilt rather than yielded as text: the route arms match on the type.@@ -1448,28 +1544,13 @@ class InferenceOrchestrator:)return- def _start_dispatcher(self) -> bool:- """Start the dispatcher thread if not already running. The dispatcher reads the shared- resp_queue and routes responses to per-request mailbox queues, letting multiple- adapter-controlled (compare) requests be in-flight without holding _gen_lock. The whole- check-then-spawn runs under _dispatcher_lifecycle_lock so concurrent compare requests- (which bypass _gen_lock) can't both observe no live dispatcher and each spawn one.- Returns True only for the caller that actually started a new thread; False if one was- already alive."""+ def _start_dispatcher(self) -> Optional[threading.Thread]:+ """Start the dispatcher thread if not already running."""with self._dispatcher_lifecycle_lock:- # Refuse to start while an unload is in progress. unload_model sets _unload_pending under this same lock- # before it stops the idle dispatcher, so a start queued behind that stop observes the unload here and- # bails. Without this a fresh dispatcher would be spawned after the stop, become the resp_queue reader,- # and consume the worker's "unloaded" reply (unroutable, so dropped) before unload_model's _wait_response- # sees it -- hanging the unload 300s.- if (- self._unload_pending- or self._exclusive_tts_pending- or getattr(self, "_exclusive_vram_probe_pending", False)- ):- return False+ if self._unload_pending or self._worker_reserved_for:+ return Noneif self._dispatcher_thread is not None and self._dispatcher_thread.is_alive():- return False+ return Noneself._dispatcher_stop.clear()self._dispatcher_thread = threading.Thread(@@ -1479,16 +1560,15 @@ class InferenceOrchestrator:)self._dispatcher_thread.start()logger.debug("Dispatcher thread started")- return True+ return self._dispatcher_thread- def _stop_dispatcher(self) -> None:- """Signal the dispatcher to stop and wait for it. Runs under _dispatcher_lifecycle_lock- (paired with _start_dispatcher) so a stop can't interleave with a concurrent start.- Callers must NOT hold _mailbox_lock here: this joins the dispatcher, and the dispatcher- loop takes _mailbox_lock, so holding it would deadlock the join."""+ def _stop_dispatcher(self, thread: Optional[threading.Thread] = None) -> None:+ """Signal the dispatcher to stop and wait for it."""with self._dispatcher_lifecycle_lock:if self._dispatcher_thread is None:return+ if thread is not None and self._dispatcher_thread is not thread:+ returnself._dispatcher_stop.set()self._dispatcher_thread.join(timeout = _DISPATCH_STOP_TIMEOUT)self._dispatcher_thread = None@@ -1575,10 +1655,12 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ rows: Optional[list] = None,+ expected_model: Optional[str] = None,video: Optional[str] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,- ) -> Generator[str, None, None]:+ ) -> Generator[Any, None, None]:"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-requestmailbox for tokens so two compare-mode requests can be queued at once. The subprocessstill runs commands sequentially, so GPU work stays serialized; this only avoids@@ -1592,25 +1674,14 @@ class InferenceOrchestrator:return# Latch the target model so the recheck below can detect a switch that completed between _start_dispatcher and# mailbox registration (mirrors the locked path's expected_model check).- expected_model = self.active_model_name+ if expected_model is None:+ expected_model = self.active_model_name# Switch in flight (unload waiting on _gen_lock). This path bypasses the lock, so without this early-out a# compare request would enqueue a generate on the outgoing model and delay the switch.if self._unload_pending:yield GenStreamError("Error: model is being unloaded", public = True)return- if self._exclusive_tts_pending:- yield GenStreamError("Error: audio generation is in progress", public = True)- return-- # Ensure the dispatcher runs. _start_dispatcher serializes concurrent starters under- # _dispatcher_lifecycle_lock and returns True only for the caller that spawned the thread, so at most one- # dispatcher ever exists even when two compare requests race here. Derive dispatcher_preexisting from that- # atomic result (not a separate unlocked is_alive() read): if THIS call started the dispatcher and then bails- # on a racing unload, it must stop it again (see the unloading bail below).- started = self._start_dispatcher()- dispatcher_preexisting = not started-request_id = str(uuid.uuid4())image_b64 = self._pil_to_base64(image) if image is not None else None@@ -1643,90 +1714,89 @@ class InferenceOrchestrator:continue_final_message = continue_final_message,tool_protocol_active = tool_protocol_active,seed = seed,+ rows = rows,video_b64 = video,)mailbox = _WorkerMailbox(self._proc)- with self._mailbox_lock:- # _unload_pending alone is not enough: an unload that ran fully since _start_dispatcher clears it in its- # finally and stops the dispatcher, so it reads False here though the dispatcher is gone and the model- # swapped. Also bail when the active model changed or the dispatcher died: a mailbox with no dispatcher to- # route gen_done/gen_error hangs the compare stream.- dispatcher_alive = (- self._dispatcher_thread is not None and self._dispatcher_thread.is_alive()- )- unloading = (- self._unload_pending- or self.active_model_name != expected_model- or not dispatcher_alive- )- tts_reserved = self._exclusive_tts_pending- probe_reserved = getattr(self, "_exclusive_vram_probe_pending", False)- blocked = unloading or tts_reserved or probe_reserved+ admission_deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT+ while True:+ self._wait_while_worker_held(cancel_event, deadline = admission_deadline)+ if cancel_event is not None and cancel_event.is_set():+ return++ started = self._start_dispatcher()++ with self._mailbox_lock:+ dispatcher_alive = (+ self._dispatcher_thread is not None+ and self._dispatcher_thread.is_alive()+ and not self._dispatcher_stop.is_set()+ )+ unloading = self._unload_pending or self.active_model_name != expected_model+ reserved_for = self._worker_reserved_for+ blocked = unloading or bool(reserved_for) or not dispatcher_alive+ if not blocked:+ self._mailboxes[request_id] = mailbox+ if cancel_event is not None:+ self._request_cancel_events[request_id] = cancel_event+ orphaned_dispatcher = unloading and started is not None and not self._mailboxes+ if orphaned_dispatcher:+ self._dispatcher_stop.set()if not blocked:- self._mailboxes[request_id] = mailbox- if cancel_event is not None:- self._request_cancel_events[request_id] = cancel_event- # When bailing without a mailbox, note whether any OTHER compare request still routes through the- # dispatcher; if none and this call started it, stop it below.- orphaned_dispatcher = blocked and not dispatcher_preexisting and not self._mailboxes- if blocked:- # A racing unload can pass its _wait_dispatcher_idle() while the dispatcher was stopped, then set- # _unload_pending. The one we just started would otherwise linger with no mailboxes, race unload_model's- # _wait_response for the "unloaded" reply off resp_queue, and drop it as unroutable -- hanging the unload- # 300s. Stop it here so the unload stays the sole resp_queue reader. Outside _mailbox_lock:+ break# _stop_dispatcher joins the dispatcher, which itself takes that lock.if orphaned_dispatcher:- self._stop_dispatcher()- detail = (- "Error: audio generation is in progress"- if tts_reserved- else "Error: model switch is checking GPU memory"- if probe_reserved- else "Error: model is being unloaded"- )+ self._stop_dispatcher(started)+ if not unloading and time.monotonic() < admission_deadline:+ time.sleep(_STOP_NOTICE_INTERVAL)+ continue+ if reserved_for:+ detail = f"Error: {reserved_for}"+ elif unloading:+ detail = "Error: model is being unloaded"+ else:+ detail = "Error: nothing is routing replies right now; try again"yield GenStreamError(detail, public = True)return- # Claim before sending, like the locked path: dispatched runs are concurrent by design, so without this a Stop- # on one saw no owner and reset the worker, ending its siblings. Claim and enqueue under one lock, or two- # dispatcher threads interleave and claim order stops matching the subprocess's command order, which- # _owns_worker reads.- try:- with self._send_order_lock:- self._claim_worker(cancel_event)- self._send_cmd(cmd)- except RuntimeError as exc:- self._release_worker(cancel_event)- with self._mailbox_lock:- self._mailboxes.pop(request_id, None)- self._request_cancel_events.pop(request_id, None)- yield GenStreamError(f"Error: {exc}")- return-def read_mailbox(timeout):try:return self._read_mailbox(mailbox, timeout)except queue.Empty:return None+ send_error = Nonetry:- yield from self._consume_token_stream(- read_mailbox,- lambda: self._drain_mailbox(mailbox, timeout = 5.0),- crash_context = "generation",- cancel_event = cancel_event,- stats_holder = stats_holder,- read_timeout = _DISPATCH_READ_TIMEOUT,- mark_started = False,- )+ try:+ with self._send_order_lock:+ if cancel_event is not None and cancel_event.is_set():+ return+ self._claim_worker(cancel_event)+ self._send_cmd(cmd)+ except RuntimeError as exc:+ send_error = exc++ if send_error is None:+ yield from self._consume_token_stream(+ read_mailbox,+ lambda: self._drain_mailbox(mailbox, timeout = 5.0),+ crash_context = "generation",+ request_id = request_id,+ cancel_event = cancel_event,+ stats_holder = stats_holder,+ read_timeout = _DISPATCH_READ_TIMEOUT,+ mark_started = False,+ rows = len(rows) if rows else None,+ )finally:- # Normally already retired by the dispatcher at gen_done; this covers streams that end without one- # (cancel, disconnect, a dead subprocess).+ # Covers streams that end without a gen_done (never sent, cancel, disconnect, dead worker).self._release_worker(cancel_event)with self._mailbox_lock:self._mailboxes.pop(request_id, None)self._request_cancel_events.pop(request_id, None)+ if send_error is not None:+ yield GenStreamError(f"Error: {send_error}")def _drain_mailbox(self,@@ -1747,36 +1817,62 @@ class InferenceOrchestrator:returnlogger.warning("Timed out draining mailbox after cancel")- def _wait_dispatcher_idle(self, cancel_event = None) -> bool:- """Wait for all dispatched requests to complete, then stop dispatcher. Returns True if the- dispatcher was stopped (all mailboxes drained, or no dispatcher was running), and False- if it was left running because compare requests were still active after- _DISPATCH_IDLE_TIMEOUT. Callers must reserve admission before using this as an exclusive- handoff; TTS does so under _gen_lock, while older direct paths call it before the lock."""- if self._dispatcher_thread is None or not self._dispatcher_thread.is_alive():- return True+ @contextlib.contextmanager+ def _reserve_worker(self, reason: str):+ """Hold the worker for one caller, answering everything else with ``reason``."""+ with self._worker_released:+ if self._worker_reserved_for is not None:+ raise RuntimeError(f"The worker is already held: {self._worker_reserved_for}")+ self._worker_reserved_for = reason+ try:+ yield+ finally:+ with self._worker_released:+ self._worker_reserved_for = None+ self._worker_released.notify_all()++ def _wait_while_worker_held(+ self,+ cancel_event = None,+ deadline = None,+ ) -> None:+ if deadline is None:+ deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT+ with self._worker_released:+ while self._worker_reserved_for is not None:+ if cancel_event is not None and cancel_event.is_set():+ return+ remaining = deadline - time.monotonic()+ if remaining <= 0:+ return+ self._worker_released.wait(min(remaining, _STOP_NOTICE_INTERVAL))++ def _replies_in_flight(self) -> int:+ with self._mailbox_lock:+ return len(self._mailboxes) + len(self._direct_mailboxes)- # Wait for all mailboxes to be emptied (dispatched requests complete)- deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT+ def _wait_worker_idle(+ self,+ cancel_event = None,+ timeout: float = _DISPATCH_IDLE_TIMEOUT,+ ) -> bool:+ deadline = time.monotonic() + timeoutwhile time.monotonic() < deadline:if cancel_event is not None and cancel_event.is_set():return False- with self._mailbox_lock:- if not self._mailboxes:- break+ if not self._replies_in_flight():+ breaktime.sleep(0.1)if cancel_event is not None and cancel_event.is_set():return False- # Only stop the dispatcher if all mailboxes drained- with self._mailbox_lock:- still_active = bool(self._mailboxes)- if still_active:+ remaining = self._replies_in_flight()+ if remaining:logger.warning(- "Dispatcher still has %d active mailbox(es); "- "leaving dispatcher running for compare requests",- len(self._mailboxes),+ "%d repl(ies) still in flight after the idle wait; leaving the dispatcher "+ "running for them",+ remaining,)return Falseself._stop_dispatcher()@@ -1791,54 +1887,53 @@ class InferenceOrchestrator:if not self._ensure_subprocess_alive():raise RuntimeError("Inference subprocess is not running")- self._wait_dispatcher_idle()- with self._mailbox_lock:- if self._mailboxes:- raise RuntimeError(- "Cannot share distributed objects while compare requests are active"- )- request_id = str(uuid.uuid4())- cmd = {- "type": "share_object",- "request_id": request_id,- "object": obj,- }-with self._gen_lock:- self._send_cmd(cmd)- deadline = None if timeout is None else time.monotonic() + timeout- while deadline is None or time.monotonic() < deadline:- remaining = 1.0 if deadline is None else max(0.1, deadline - time.monotonic())- resp = self._read_resp(timeout = min(remaining, 1.0))- if resp is None:- if not self._ensure_subprocess_alive():- raise RuntimeError(- self._subprocess_crash_message(- "sharing chat turn",- with_worker_output = self._owns_worker(None),+ with self._reserve_worker("a distributed share is in progress"):+ if not self._wait_worker_idle():+ raise RuntimeError(+ "Cannot share distributed objects while a reply is still generating"+ )+ request_id = str(uuid.uuid4())+ cmd = {+ "type": "share_object",+ "request_id": request_id,+ "object": obj,+ }++ self._send_cmd(cmd)+ deadline = None if timeout is None else time.monotonic() + timeout+ while deadline is None or time.monotonic() < deadline:+ remaining = 1.0 if deadline is None else max(0.1, deadline - time.monotonic())+ resp = self._read_resp(timeout = min(remaining, 1.0))+ if resp is None:+ if not self._ensure_subprocess_alive():+ raise RuntimeError(+ self._subprocess_crash_message(+ "sharing chat turn",+ with_worker_output = self._owns_worker(None),+ ))- )- continue+ continue- rtype = resp.get("type", "")- rid = resp.get("request_id")- if rid and rid != request_id:- logger.debug(- "Skipping response for request_id=%s while sharing request_id=%s",- rid,- request_id,- )- continue- if rtype == "shared":- return resp.get("object")- if rtype == "share_error":- raise RuntimeError(resp.get("error", "Failed to share object"))- if rtype == "error":- raise RuntimeError(resp.get("error", "Subprocess error"))- if rtype == "status":- continue+ rtype = resp.get("type", "")+ rid = resp.get("request_id")+ if rid and rid != request_id:+ logger.debug(+ "Skipping response for request_id=%s while sharing request_id=%s",+ rid,+ request_id,+ )+ continue+ if rtype == "shared":+ return resp.get("object")+ if rtype == "share_error":+ raise RuntimeError(resp.get("error", "Failed to share object"))+ if rtype == "error":+ raise RuntimeError(resp.get("error", "Subprocess error"))+ if rtype == "status":+ continue- raise RuntimeError("Timeout waiting for distributed object share")+ raise RuntimeError("Timeout waiting for distributed object share")# Monotonic count of PUBLISHED loads; lets the install route detect a load (including a same-model reload) that# completed while it waited on the gate. Bumped when the load result is published, not at load start: a start-time@@ -1869,9 +1964,12 @@ class InferenceOrchestrator:cache_environment: Optional[Mapping[str, str]] = None,anonymous_hf_access: bool = False,audio_codec_path: Optional[str] = None,+ n_parallel: Optional[int] = None,) -> bool:- """Load a model for inference. Always spawns a fresh subprocess per load for a clean- interpreter (no stale unsloth patches, torch.compile caches, or getsource failures)."""+ """Load a model for inference."""+ from core.inference.llama_server_args import clamp_parallel_slots++ parallel_slots = clamp_parallel_slots(n_parallel)from utils.transformers_version import needs_transformers_5from utils.hf_xet_fallback import DownloadStallError@@ -2115,6 +2213,10 @@ class InferenceOrchestrator:self.models[self.active_model_name] = _mirrored_model_entry(model_info, model_name)+ self.models[self.active_model_name]["parallel_slots"] = parallel_slots+ self.models[self.active_model_name]["can_batch"] = model_info.get(+ "can_batch"+ )# Lets the already-loaded shortcut tell a CPU request from the GPU# model it would otherwise report as satisfied. Native audio only:# marking anything else tells training a GPU model holds no VRAM.@@ -2255,13 +2357,6 @@ class InferenceOrchestrator:self.models.pop(model_name, None)return True- # The subprocess runs commands sequentially, so a bare unload queues behind a running generate (a 2-3 min- # hang). Cancel first (via the mp.Event the worker polls each token), then take _gen_lock as sole resp_queue- # reader (like GGUF). Set _unload_pending under _dispatcher_lifecycle_lock so it is ordered ahead of the- # dispatcher stop that _wait_dispatcher_idle runs under the same lock: a compare request's _start_dispatcher- # queued behind that stop then observes the unload and refuses to spawn a fresh dispatcher that would eat the- # "unloaded" reply off resp_queue. This is a standalone acquisition (no _gen_lock held yet), so it keeps the- # _gen_lock -> _dispatcher_lifecycle_lock order and can't deadlock.with self._dispatcher_lifecycle_lock:self._unload_pending = True# Cancelling only the running generation isn't enough: the worker clears cancel_event at each generate start,@@ -2270,7 +2365,7 @@ class InferenceOrchestrator:if self._drain_event is not None:self._drain_event.set()try:- self._cancel_generation()+ self._cancel_every_generation()acquired = self._gen_lock.acquire(timeout = _UNLOAD_GEN_LOCK_TIMEOUT)if not acquired:logger.warning(@@ -2285,11 +2380,7 @@ class InferenceOrchestrator:return Truetry:- # Stop the compare-mode dispatcher so it can't consume the "unloaded" reply off resp_queue before we- # do. A dispatched generation bypasses _gen_lock, so a wedged one slips past the acquire above; if the- # dispatcher is still active it owns resp_queue and the queued unload hangs _wait_response behind the- # stuck generate.- if not self._wait_dispatcher_idle():+ if not self._wait_worker_idle(timeout = _DISPATCH_IDLE_TIMEOUT):logger.warning("Unload: compare-mode dispatcher still active after idle ""wait; shutting the inference subprocess down to free the model"@@ -2300,12 +2391,17 @@ class InferenceOrchestrator:self.active_model_name = Nonereturn Trueself._drain_queue()- self._send_cmd(- {- "type": "unload",- "model_name": model_name,- }- )+ self._teardown_going_out()+ try:+ self._send_cmd(+ {+ "type": "unload",+ "model_name": model_name,+ }+ )+ except Exception:+ self._teardown_not_sent()+ raiseself._wait_response("unloaded")self.models.pop(model_name, None)@@ -2354,6 +2450,11 @@ class InferenceOrchestrator:nowhere."""if not self._gen_lock.acquire(blocking = False):raise RuntimeError("Cannot count tokens while a generation is in progress")+ with self._mailbox_lock:+ dispatched = bool(self._mailboxes)+ if dispatched:+ self._gen_lock.release()+ raise RuntimeError("Cannot count tokens while a generation is in progress")request_id = str(uuid.uuid4())read_one, _drain, release_mailbox = self._direct_reader(request_id)try:@@ -2466,6 +2567,70 @@ class InferenceOrchestrator:video = video,)+ def generate_chat_batch(+ self,+ rows: list,+ messages: list,+ system_prompt: str = "",+ image = None,+ images: Optional[list] = None,+ image_ordinal: Optional[int] = None,+ temperature: float = 0.7,+ top_p: float = 0.9,+ top_k: int = 40,+ min_p: float = 0.0,+ max_new_tokens: int = 256,+ repetition_penalty: float = 1.0,+ cancel_event = None,+ tools: Optional[list] = None,+ enable_thinking: Optional[bool] = None,+ reasoning_effort: Optional[str] = None,+ preserve_thinking: Optional[bool] = None,+ continue_final_message: bool = False,+ tool_protocol_active: Optional[bool] = None,+ stats_holder: Optional[dict] = None,+ presence_penalty: float = 0.0,+ seed: Optional[int] = None,+ frequency_penalty: float = 0.0,+ logit_bias: Optional[dict] = None,+ stop: Optional[list] = None,+ video: Optional[str] = None,+ response_format: Optional[dict] = None,+ reasoning_is_extracted: bool = False,+ ) -> Generator[Any, None, None]:+ """Ask for several replies to one prompt in a single command."""+ yield from self._generate_inner(+ messages = messages,+ system_prompt = system_prompt,+ image = image,+ images = images,+ image_ordinal = image_ordinal,+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ min_p = min_p,+ max_new_tokens = max_new_tokens,+ repetition_penalty = repetition_penalty,+ cancel_event = cancel_event,+ use_adapter = None,+ tools = tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ tool_protocol_active = tool_protocol_active,+ stats_holder = stats_holder,+ presence_penalty = presence_penalty,+ seed = seed,+ frequency_penalty = frequency_penalty,+ logit_bias = logit_bias,+ stop = stop,+ rows = rows,+ video = video,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,+ )+def generate_chat_completion_with_tools(self,messages: list,@@ -2700,10 +2865,11 @@ class InferenceOrchestrator:frequency_penalty: float = 0.0,logit_bias: Optional[dict] = None,stop: Optional[list] = None,+ rows: Optional[list] = None,video: Optional[str] = None,response_format: Optional[dict] = None,reasoning_is_extracted: bool = False,- ) -> Generator[str, None, None]:+ ) -> Generator[Any, None, None]:"""Inner generation logic: sends the command to the subprocess and yields tokens. Serializedby _gen_lock (one generation at a time) so concurrent readers don't consume each other'stokens off the shared resp_queue."""@@ -2716,10 +2882,41 @@ class InferenceOrchestrator:returnexpected_model = self.active_model_name- self._wait_dispatcher_idle()+ if self._stop_ledger is not None and self._stop_ledger.read_by_worker():+ yield from self._generate_dispatched(+ messages = messages,+ system_prompt = system_prompt,+ image = image,+ images = images,+ image_ordinal = image_ordinal,+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ min_p = min_p,+ max_new_tokens = max_new_tokens,+ repetition_penalty = repetition_penalty,+ cancel_event = cancel_event,+ use_adapter = use_adapter,+ tools = tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = continue_final_message,+ tool_protocol_active = tool_protocol_active,+ stats_holder = stats_holder,+ presence_penalty = presence_penalty,+ seed = seed,+ frequency_penalty = frequency_penalty,+ logit_bias = logit_bias,+ stop = stop,+ rows = rows,+ expected_model = expected_model,+ video = video,+ response_format = response_format,+ reasoning_is_extracted = reasoning_is_extracted,+ )+ return- # Serialize generation: two concurrent readers on resp_queue would consume and drop each other's token events.- # Hold _gen_lock across the cmd build + send + whole stream so we stay the sole resp_queue reader.with self._gen_lock:if self._unload_pending or self.active_model_name != expected_model:yield GenStreamError("Error: model is being unloaded", public = True)@@ -2756,14 +2953,11 @@ class InferenceOrchestrator:continue_final_message = continue_final_message,tool_protocol_active = tool_protocol_active,seed = seed,+ rows = rows,video_b64 = video,)- # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the lock above, having- # generated nothing -- cannot reset the generation this is starting. Claiming after the send left the- # command running unclaimed. Released in the finally. Own mailbox: a compare request can start the- # dispatcher while this is streaming, and it would otherwise consume our responses and drop them.- read_one, drain, release_mailbox = self._direct_reader(request_id)+ read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)try:try:with self._send_order_lock:@@ -2777,29 +2971,26 @@ class InferenceOrchestrator:read_one,lambda: drain(timeout = 5.0),crash_context = "generation",+ request_id = request_id,cancel_event = cancel_event,stats_holder = stats_holder,+ rows = len(rows) if rows else None,)finally:self._release_worker(cancel_event)release_mailbox()def _claim_worker(self, cancel_event) -> None:- """Record this request as one the worker will run. Admission only: the subprocess executes- generations one at a time, so a dispatched request sitting behind another in the command- queue is claimed but not executing, and must not be able to signal the shared cancel- event (that would end whichever request IS executing). _mark_worker_started promotes it- once the worker answers it."""+ """Record this request as one the worker will run."""with self._active_cancel_lock:self._active_cancel_events.append(cancel_event)def _mark_worker_started(self, cancel_event) -> None:- """Promote a claimed request to executing, on its first worker response. Sole executor: the- subprocess runs one generation at a time, so answering this one means it has left the- previous one behind."""if cancel_event is None:returnwith self._active_cancel_lock:+ if not any(ev is cancel_event for ev in self._active_cancel_events):+ returnif self._executing_cancel_events[:1] != [cancel_event]:self._executing_cancel_events[:] = [cancel_event]@@ -2826,23 +3017,75 @@ class InferenceOrchestrator:# oldest claim is the executor; anyone else here is queued behind it.return self._active_cancel_events[0] is cancel_event+ def _stop_and_signal(+ self,+ request_id: str,+ cancel_event,+ recorded: bool = False,+ *,+ may_signal: bool = True,+ ) -> tuple[bool, bool]:+ """End one request: by name where the worker reads stops, else via the shared event.+ Reports (recorded, sent)."""+ ledger = self._stop_ledger+ sent = False+ if ledger is not None:+ if not recorded:+ recorded = bool(+ request_id and self._ensure_subprocess_alive() and ledger.stop(request_id)+ )+ sent = bool(recorded and ledger.read_by_worker())+ if not sent and may_signal:+ with self._active_cancel_lock:+ answered = any(ev is cancel_event for ev in self._executing_cancel_events)+ if answered:+ self._cancel_generation()+ sent = True+ if sent:+ self._release_worker(cancel_event)+ return recorded, sent++ def _forget_request(self, request_id: str, cancel_event) -> None:+ """Retire a reply as it ends: a cancel event outliving its request stops a later one."""+ if request_id:+ with self._mailbox_lock:+ self._request_cancel_events.pop(request_id, None)+ self._release_worker(cancel_event)++ def _request_of(self, cancel_event) -> Optional[str]:+ if cancel_event is None:+ return None+ with self._mailbox_lock:+ for request_id, event in self._request_cancel_events.items():+ if event is cancel_event:+ return request_id+ return None+def reset_generation_state(self, caller_cancel_event = None):- """Cancel any ongoing generation and reset state. ``caller_cancel_event`` scopes the reset- to one request. The worker has a single cancel event and generation is serialized on- _gen_lock, so a chat that is still queued has no generation of its own to reset: calling- this from its Stop handler would kill whichever chat currently holds the lock. Pass the- request's own event and the reset is dropped unless that request is the one running. Omit- it for genuinely global resets (unload, switch)."""- if caller_cancel_event is not None and not self._owns_worker(caller_cancel_event):+ """Cancel any ongoing generation and reset state."""+ if caller_cancel_event is not None:+ request_id = self._request_of(caller_cancel_event)+ if request_id is not None:+ self._stop_and_signal(request_id, caller_cancel_event)+ return+ with self._send_order_lock:+ if not self._owns_worker(caller_cancel_event):+ return+ self._reset_worker()return+ self._reset_worker()++ def _reset_worker(self) -> None:+ self._teardown_going_out()self._cancel_generation()if not self._ensure_subprocess_alive():+ self._teardown_not_sent()returntry:with self._send_order_lock:self._send_cmd({"type": "reset"})except RuntimeError:- pass+ self._teardown_not_sent()def generate_audio_response(self,@@ -2871,15 +3114,13 @@ class InferenceOrchestrator:# idle wait is racy: a compare request can register between the wait and this command, leaving TTS queued# without safe ownership of the worker's single shared cancel event.with self._gen_lock:- with self._dispatcher_lifecycle_lock:- self._exclusive_tts_pending = True- try:- dispatcher_idle = self._wait_dispatcher_idle(cancel_event = cancel_event)+ with self._reserve_worker("audio generation is in progress"):+ idle = self._wait_worker_idle(cancel_event = cancel_event)if cancel_event is not None and cancel_event.is_set():raise AudioGenerationCancelledError("Audio generation cancelled")- if not dispatcher_idle:+ if not idle:raise RuntimeError(- "Cannot start audio generation while compare requests are active"+ "Cannot start audio generation while a reply is still generating")# Recheck after the dispatcher wait: unload can set its flag without _gen_lock, and a switch may have@@ -2931,7 +3172,7 @@ class InferenceOrchestrator:cmd["seed"] = int(seed)# Same shared-queue hazard as _generate_inner: see _direct_reader.- read_one, _drain, release_mailbox = self._direct_reader(request_id)+ read_one, _drain, release_mailbox = self._direct_reader(request_id, cancel_event)try:# Claim before enqueueing so request-scoped reset ownership follows the same discipline as text# and audio-input generation@@ -3036,9 +3277,6 @@ class InferenceOrchestrator:finally:self._release_worker(cancel_event)release_mailbox()- finally:- with self._dispatcher_lifecycle_lock:- self._exclusive_tts_pending = Falsedef generate_whisper_response(self,@@ -3152,7 +3390,7 @@ class InferenceOrchestrator:if stop:cmd["stop"] = stop- read_one, drain, release_mailbox = self._direct_reader(request_id)+ read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)try:try:# Claim under the send lock, like _generate_inner: unclaimed, a compare request queued behind this@@ -3168,6 +3406,7 @@ class InferenceOrchestrator:read_one,lambda: drain(timeout = 5.0),crash_context = "audio input generation",+ request_id = request_id,cancel_event = cancel_event,stats_holder = stats_holder,)diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 502ffc914f4..705c54e631c 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -23,7 +23,7 @@ import timeimport tracebackfrom io import BytesIOfrom pathlib import Path-from typing import Any+from typing import Any, Optionallogger = get_logger(__name__)from core.inference.audio_errors import AUDIO_UNSUPPORTED_CODE@@ -33,6 +33,110 @@ from utils.hardware import apply_gpu_ids, is_apple_silicon# Fresh spawned interpreter: re-apply the OS-trust-store injection.from utils.native_tls import activate_native_tls++_ID_BYTES = 36++_SLOTS = 256+++class StopLedger:+ """The request ids the parent has stopped, and whether the worker reads them."""++ def __init__(self, ctx: Any):+ self._lock = ctx.Lock()+ self._slots = ctx.Array("c", _SLOTS * _ID_BYTES, lock = False)+ self._read_by_worker = ctx.Value("b", 0, lock = False)+ self._written = ctx.Value("l", 0, lock = False)++ def worker_reads_this(self) -> None:+ """Said once by the worker, as it enters the loop that reads names from here."""+ self._read_by_worker.value = 1++ def read_by_worker(self) -> bool:+ return bool(self._read_by_worker.value)++ def stop(self, request_id: str) -> bool:+ """Record a stop. False for an id no slot could hold."""+ entry = _entry(request_id)+ if entry is None:+ return False+ with self._lock:+ if not self._holds(entry):+ start = (self._written.value % _SLOTS) * _ID_BYTES+ self._slots[start : start + _ID_BYTES] = entry+ self._written.value += 1+ return True++ def snapshot(self, since: int = -1) -> tuple[int, Optional[set]]:+ written = self._written.value+ if written == since:+ return written, None+ with self._lock:+ written = self._written.value+ raw = bytes(self._slots)+ return written, {+ raw[start : start + _ID_BYTES].rstrip(b"\0").decode("utf-8", "replace")+ for start in range(0, min(written, _SLOTS) * _ID_BYTES, _ID_BYTES)+ }++ def _holds(self, entry: bytes) -> bool:+ for slot in range(min(self._written.value, _SLOTS)):+ start = slot * _ID_BYTES+ if bytes(self._slots[start : start + _ID_BYTES]) == entry:+ return True+ return False+++def _entry(request_id: Optional[str]) -> Optional[bytes]:+ """One slot's worth of bytes, or None for an id no slot could hold."""+ if not request_id:+ return None+ entry = str(request_id).encode("utf-8", "replace")+ if len(entry) > _ID_BYTES or b"\0" in entry:+ return None+ return entry.ljust(_ID_BYTES, b"\0")+++class PendingTeardowns:+ """How many commands that end everything are on their way to the worker."""++ def __init__(self, ctx: Any):+ self._count = ctx.Value("l", 0)++ def sending(self) -> None:+ with self._count.get_lock():+ self._count.value += 1++ def unsent(self) -> None:+ self._counted_off()++ def taken(self) -> None:+ self._counted_off()++ def _counted_off(self) -> None:+ """One fewer on its way, clamped: the two ends are different processes."""+ with self._count.get_lock():+ self._count.value = max(0, self._count.value - 1)++ def any_in_flight(self) -> bool:+ return self._count.value > 0+++class RowRefused(Exception):+ """This batch will not take this reply, and is exactly as it was."""+++def narrow_load_reason(cmd: dict) -> Optional[str]:+ """Why no batch of either kind may take this command, or None."""+ width = int(cmd.get("parallel_slots") or 1)+ if width <= 1:+ return "this load decodes one reply at a time"+ rows = len(cmd.get("rows") or [])+ if rows > width:+ return f"the load decodes {width} replies at once, and this asks for {rows}"+ return None++activate_native_tls()_SHARE_OBJECT_MAX_BYTES = 1 << 20@@ -530,6 +634,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:"is_audio": getattr(mc, "is_audio", False),"audio_type": getattr(mc, "audio_type", None),"has_audio_input": getattr(mc, "has_audio_input", False),+ "can_batch": _load_can_batch(backend),}_bm = getattr(backend, "models", {}) or {}_entry = (@@ -567,6 +672,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:{k: _entry[k]for k in (+ "context_unbounded_when_batched","mlx_kv_bits","mlx_kv_bits_requested","mlx_kv_quant_eligibility",@@ -659,6 +765,86 @@ def _drain_skip_generate(return True+def _abandon_held_commands(held: list, resp_queue: Any) -> None:+ while held:+ _abandon_one(held.pop(0), resp_queue)+++class _Stops:+ def __init__(self, stop_ledger, resp_queue, batch, held: list):+ self._ledger = stop_ledger+ self._resp_queue = resp_queue+ self._batch = batch+ self._held = held+ self._written = 0+ self._stopped: set = set()++ def answer(self) -> None:+ self._refresh()+ self._batch.drop_stopped(self)+ for cmd in [held for held in self._held if held.get("request_id", "") in self]:+ self._held.remove(cmd)+ _abandon_one(cmd, self._resp_queue)++ def _refresh(self) -> None:+ if self._ledger is None:+ return+ self._written, stopped = self._ledger.snapshot(self._written)+ if stopped is not None:+ self._stopped = stopped++ def __contains__(self, request_id) -> bool:+ return request_id in self._stopped+++class _StopWhileItRuns:+ def __init__(self, cancel_event, stops, request_id: str):+ self._cancel_event = cancel_event+ self._stops = stops+ self._request_id = request_id++ def is_set(self) -> bool:+ if self._cancel_event is not None and self._cancel_event.is_set():+ return True+ self._stops.answer()+ return self._request_id in self._stops+++_TEARDOWN_COMMANDS = frozenset({"cancel", "reset", "unload", "shutdown"})+++def _teardown_skip(cmd: dict, resp_queue: Any, pending_teardowns) -> bool:+ if pending_teardowns is None or not pending_teardowns.any_in_flight():+ return False+ _abandon_one(cmd, resp_queue)+ return True+++def _stopped_before_it_ran(cmd: dict, resp_queue: Any, stopped) -> bool:+ stopped.answer()+ if cmd.get("request_id", "") not in stopped:+ return False+ _abandon_one(cmd, resp_queue)+ return True+++def _abandon_one(cmd: dict, resp_queue: Any) -> None:+ logger.info(+ "Abandoning held %s for request %s",+ cmd.get("type", ""),+ cmd.get("request_id", ""),+ )+ _send_response(+ resp_queue,+ {+ "type": "gen_done",+ "request_id": cmd.get("request_id", ""),+ "cancelled": True,+ "stats": None,+ },+ )++def _prepare_generate_audio(cmd, resp_queue: Any, cancel_event, drain_event) -> bool:"""Clear stale cancellation and acknowledge when this TTS command owns the worker.@@ -702,75 +888,82 @@ def _backend_declares(return False+def _dispatch_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:+ if cmd.get("rows"):+ _handle_generate_rows(backend, cmd, resp_queue, cancel_event)+ else:+ _handle_generate(backend, cmd, resp_queue, cancel_event)+++def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:+ image = None+ image_b64 = cmd.get("image_base64")+ if image_b64:+ image = _resize_image(_decode_image(image_b64))+ images = [+ _resize_image(_decode_image(encoded))+ for encoded in cmd.get("images_base64") or ()+ if encoded+ ]++ gen_kwargs = {+ "messages": cmd["messages"],+ "system_prompt": cmd.get("system_prompt", ""),+ "image": image,+ "images": images,+ "temperature": cmd.get("temperature", 0.7),+ "top_p": cmd.get("top_p", 0.9),+ "top_k": cmd.get("top_k", 40),+ "min_p": cmd.get("min_p", 0.0),+ "max_new_tokens": cmd.get("max_new_tokens", 256),+ "repetition_penalty": cmd.get("repetition_penalty", 1.0),+ "presence_penalty": cmd.get("presence_penalty", 0.0),+ "cancel_event": cancel_event,+ }++ for opt_key in (+ "tools",+ "enable_thinking",+ "reasoning_effort",+ "preserve_thinking",+ "continue_final_message",+ ):+ if opt_key in cmd:+ gen_kwargs[opt_key] = cmd[opt_key]++ if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):+ gen_kwargs["image_ordinal"] = cmd["image_ordinal"]++ for gated in (+ "seed",+ "frequency_penalty",+ "logit_bias",+ "stop",+ "tool_protocol_active",+ "response_format",+ "reasoning_is_extracted",+ ):+ if gated in cmd and _backend_declares(backend, gated):+ gen_kwargs[gated] = cmd[gated]+ if cmd.get("video_base64"):+ if not _backend_declares(backend, "video"):+ raise RuntimeError("The loaded backend does not read video.")+ gen_kwargs["video"] = cmd["video_base64"]++ return gen_kwargs++def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:"""Handle a generate command: stream tokens back via resp_queue.- cancel_event is an mp.Event the parent can set anytime (user stop, or new- model load mid-generate); generation stops within 1-2 tokens.+ cancel_event is asked between tokens; what it answers for depends on the caller --+ the batching loop passes something reading both the record and the shared event, the+ one-at-a-time loop the shared event alone. Nothing is asked during the prefill."""request_id = cmd.get("request_id", "")try:- image = None- image_b64 = cmd.get("image_base64")- if image_b64:- image = _resize_image(_decode_image(image_b64))- images = [- _resize_image(_decode_image(encoded))- for encoded in cmd.get("images_base64") or ()- if encoded- ]-- gen_kwargs = {- "messages": cmd["messages"],- "system_prompt": cmd.get("system_prompt", ""),- "image": image,- "images": images,- "temperature": cmd.get("temperature", 0.7),- "top_p": cmd.get("top_p", 0.9),- "top_k": cmd.get("top_k", 40),- "min_p": cmd.get("min_p", 0.0),- "max_new_tokens": cmd.get("max_new_tokens", 256),- "repetition_penalty": cmd.get("repetition_penalty", 1.0),- "presence_penalty": cmd.get("presence_penalty", 0.0),- "cancel_event": cancel_event,- }-- # Forward only present optional keys so the backend signature can evolve.- for opt_key in (- "tools",- "enable_thinking",- "reasoning_effort",- "preserve_thinking",- "continue_final_message",- ):- if opt_key in cmd:- gen_kwargs[opt_key] = cmd[opt_key]-- if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):- gen_kwargs["image_ordinal"] = cmd["image_ordinal"]-- # Not every backend declares these (transformers declares only ``stop``)- # and none takes **kwargs, so forwarding unconditionally would turn a- # backend's documented "ignores them" behavior into a TypeError.- # ``tool_protocol_active`` rides here rather than above: MLX declares no such- # parameter and takes no **kwargs, so an unconditional forward would raise.- for gated in (- "seed",- "frequency_penalty",- "logit_bias",- "stop",- "tool_protocol_active",- "response_format",- "reasoning_is_extracted",- ):- if gated in cmd and _backend_declares(backend, gated):- gen_kwargs[gated] = cmd[gated]- # A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.- if cmd.get("video_base64"):- if not _backend_declares(backend, "video"):- raise RuntimeError("The loaded backend does not read video.")- gen_kwargs["video"] = cmd["video_base64"]+ gen_kwargs = _generation_kwargs(backend, cmd, cancel_event)use_adapter = cmd.get("use_adapter")if use_adapter is not None:@@ -785,7 +978,6 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:try:for cumulative_text in generator:- # cancel_event is an mp.Event - checked instantly, no queue polling.if cancel_event.is_set():logger.info("Generation cancelled for request %s", request_id)break@@ -854,18 +1046,360 @@ def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:)-def _decline_count_tokens(cmd: dict, resp_queue: Any) -> None:+def _decline_count_tokens(+ cmd: dict,+ resp_queue: Any,+ reason: str = "Counting is not supported on the transformers backend.",+) -> None:"""Answer a count this backend cannot serve; dropping it costs the caller its timeout."""_send_response(resp_queue,{"type": "count_tokens_response","request_id": cmd.get("request_id"),- "error": "Counting is not supported on the transformers backend.",+ "error": reason,},)+def _load_can_batch(backend) -> bool:+ """Whether this load serves several replies at once at all."""+ fixed = getattr(backend, "batch_unavailable_reason", None)+ resident = getattr(backend, "resident_unavailable_reason", None)+ return (callable(fixed) and fixed([{}, {}]) is None) or (+ callable(resident) and resident({}) is None+ )+++def _rows_apart(backend, requests, cancel_event, stats: list):+ """A declined batch served reply by reply, as the batch's own (row, snapshot) events."""+ for row, request in enumerate(requests):+ if not cancel_event.is_set():+ generator = backend.generate_chat_response(+ **{+ name: value+ for name, value in request.items()+ if _backend_declares(backend, name)+ },+ cancel_event = cancel_event,+ )+ try:+ for cumulative_text in generator:+ if cancel_event.is_set():+ break+ yield row, cumulative_text+ finally:+ close = getattr(generator, "close", None)+ if callable(close):+ close()+ stats[row] = getattr(backend, "last_generation_stats", None)+ yield row, None+++def _handle_generate_rows(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:+ request_id = cmd.get("request_id", "")+ rows = cmd.get("rows") or []++ try:+ shared = _generation_kwargs(backend, cmd, cancel_event)+ shared.pop("cancel_event", None)+ requests = [{**shared, **row} for row in rows]++ reason = narrow_load_reason(cmd)+ if reason is None:+ unavailable = getattr(backend, "batch_unavailable_reason", None)+ reason = unavailable(requests) if callable(unavailable) else "backend cannot batch"+ if reason is not None:+ logger.info(+ "Declining %d replies in one command for request_id=%s: %s",+ len(requests),+ request_id,+ reason,+ )+ stats = [None] * len(requests)+ events = _rows_apart(backend, requests, cancel_event, stats)+ else:+ logger.info(+ "Starting batched generation for request_id=%s rows=%d",+ request_id,+ len(requests),+ )+ stats = None+ events = backend.generate_chat_batch(requests, cancel_event = cancel_event)+ try:+ for row, snapshot in events:+ if snapshot is None:+ reported = backend.last_batch_generation_stats if stats is None else stats+ message = {"type": "row_done", "stats": reported[row]}+ else:+ message = {"type": "token", "text": snapshot}+ _send_response(resp_queue, {**message, "request_id": request_id, "row": row})+ finally:+ close = getattr(events, "close", None)+ if callable(close):+ close()++ _send_response(+ resp_queue,+ {"type": "gen_done", "request_id": request_id, "stats": None},+ )++ except Exception as exc:+ logger.error("Multi-reply generation error: %s", exc, exc_info = True)+ _send_response(resp_queue, _generation_error_payload(request_id, exc))+++def _admitted_width(cmd: dict) -> int:+ return max(1, int(cmd.get("parallel_slots") or 1))+++def _held_head_leaves_the_hold(batch: "_ResidentBatch", held: list) -> bool:+ """Whether the head comes off the hold on this pass; the batch can still turn it away."""+ if not held:+ return False+ if not batch.rows_in_flight:+ return True+ head = held[0]+ return head.get("type") == "generate" and batch.unavailable_reason(head) is None+++class _ResidentBatch:+ """The replies an MLX worker is decoding at once."""++ def __init__(self, backend, resp_queue: Any):+ self.backend = backend+ self.resp_queue = resp_queue+ self.session = None+ self.width = None+ self._owed: dict = {}+ self._refused: set = set()++ @property+ def rows_in_flight(self) -> int:+ return self.session.rows_in_flight if self.session is not None else 0++ def _live(self, request_id: str) -> list:+ if self.session is None:+ return []+ return [handle for handle in self.session.handles if handle[0] == request_id]++ def unavailable_reason(self, cmd: dict) -> Optional[str]:+ if cmd.get("request_id", "") in self._refused:+ return "this batch has already refused these replies"+ if cmd.get("use_adapter") is not None:+ return "the reply asks for a particular adapter state"+ reason = narrow_load_reason(cmd)+ if reason is not None:+ return reason+ if self.width is not None and self.width != _admitted_width(cmd):+ # A command at another width waits for the batch to drain: joining at the old+ # one would refill the batch, so a narrowed load never reaches its new width.+ return "the open batch is decoding at a different width"+ if (+ self.width is not None+ and self.rows_in_flight + len(cmd.get("rows") or [None]) > self.width+ ):+ return "the open batch is full"+ probe = getattr(self.backend, "resident_unavailable_reason", None)+ if not callable(probe):+ return "this backend has no batch a reply can join"+ rows = cmd.get("rows") or []+ for request in [{**cmd, **row} for row in rows] if rows else [cmd]:+ reason = probe(request)+ if reason is not None:+ return reason+ return None++ def admit(self, cmd: dict, cancel_event) -> bool:+ request_id = cmd.get("request_id", "")+ rows = cmd.get("rows") or []+ shared = _generation_kwargs(self.backend, cmd, cancel_event)+ shared.pop("cancel_event", None)+ requests = [{**shared, **row} for row in rows] if rows else [shared]++ if self.session is None:+ self.width = _admitted_width(cmd)+ try:+ self.session = self.backend.open_resident_batch(width = self.width)+ except Exception as unopened:+ logger.warning(+ "No batch could be opened for request_id=%s, decoding it alone: %s",+ request_id,+ unopened,+ )+ self.width = None+ return False+ handles = [(request_id, row if rows else None) for row in range(len(requests))]+ self._owed[request_id] = None+ prefixes = []+ try:+ for request, handle in zip(requests, handles):+ prefixes.append((handle, self.session.admit(request, handle)))+ except RowRefused as refusal:+ if not self._forget(request_id, withdraw = True):+ self._fail_all(refusal)+ raise+ logger.info("Request_id=%s cannot join this batch: %s", request_id, refusal)+ self._refused.add(request_id)+ self._close_if_empty()+ return False+ except BaseException as exc:+ if self._forget(request_id, withdraw = True):+ self._close_if_empty()+ else:+ self._fail_all(exc)+ raise+ logger.info(+ "Admitted request_id=%s (%d replies) to a batch of %d",+ request_id,+ len(requests),+ self.rows_in_flight,+ )+ for handle, prefix in prefixes:+ if prefix:+ self._send_token(handle, prefix)+ return True++ def cancel(self, request_id: str) -> bool:+ if request_id not in self._owed:+ return False+ live = self._live(request_id)+ if not live:+ return False+ logger.info("Cancelling request_id=%s in a batch of %d", request_id, self.rows_in_flight)+ try:+ for handle, snapshot in self.session.withdraw(sorted(live, key = _handle_order)):+ self._report(handle, snapshot)+ except Exception as exc:+ logger.error("Batched cancellation error: %s", exc, exc_info = True)+ self._fail_all(exc)+ return True+ self._close_if_empty()+ return True++ def cancel_all(self) -> None:+ for request_id in list(self._owed):+ self.cancel(request_id)++ def drop_stopped(self, stopped) -> None:+ for request_id in list(self._owed):+ if request_id in stopped:+ self.cancel(request_id)++ def step(self) -> None:+ if self.session is None or not self.session.rows_in_flight:+ return+ try:+ for handle, snapshot in self.session.step():+ self._report(handle, snapshot)+ except Exception as exc:+ logger.error("Batched generation error: %s", exc, exc_info = True)+ self._fail_all(exc)+ return+ self._close_if_empty()++ def _fail_all(self, exc: BaseException) -> None:+ """Give up on the batch: every request in it is owed an error, not silence."""+ stack = traceback.format_exc(limit = 20)+ for request_id in list(self._owed):+ self._fail(request_id, exc, stack)+ self.close()++ def _close_if_empty(self) -> None:+ if self.session is not None and not self.session.rows_in_flight:+ self.close()++ def close(self) -> None:+ ending = (+ {handle[0] for handle in self.session.ending} if self.session is not None else set()+ )+ for request_id in [r for r in self._owed if r in ending]:+ self._end(request_id, cancelled = True)+ session, self.session = self.session, None+ self.width = None+ self._owed.clear()+ self._refused.clear()+ if session is None:+ return+ try:+ session.close()+ except Exception:+ logger.error("Could not close the resident batch", exc_info = True)++ def _report(self, handle, snapshot) -> None:+ request_id, row = handle+ if request_id not in self._owed:+ self._take_stats(handle)+ return+ if snapshot is not None:+ self._send_token(handle, snapshot)+ return+ if row is not None:+ _send_response(+ self.resp_queue,+ {+ "type": "row_done",+ "request_id": request_id,+ "row": row,+ "stats": self._take_stats(handle),+ },+ )+ if not self._live(request_id):+ self._end(request_id)++ def _send_token(self, handle, text) -> None:+ request_id, row = handle+ event = {"type": "token", "request_id": request_id, "text": text}+ if row is not None:+ event["row"] = row+ _send_response(self.resp_queue, event)++ def _end(+ self,+ request_id: str,+ *,+ cancelled: bool = False,+ ) -> None:+ """Answer a request's gen_done; cancelled when it ends with its batch, not a reply."""+ stats = self._take_stats((request_id, None))+ self._forget(request_id, withdraw = False)+ event = {"type": "gen_done", "request_id": request_id, "stats": stats}+ if cancelled:+ event["cancelled"] = True+ _send_response(self.resp_queue, event)+ logger.info("Ended request_id=%s (cancelled=%s)", request_id, cancelled)++ def _fail(self, request_id: str, exc: BaseException, stack: str) -> None:+ self._forget(request_id, withdraw = False)+ _send_response(+ self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}+ )++ def _forget(self, request_id: str, *, withdraw: bool) -> bool:+ """Stop answering for a request. False where taking its rows back failed."""+ live = self._live(request_id)+ self._owed.pop(request_id, None)+ taken_back = True+ if withdraw and live:+ try:+ for _handle, _snapshot in self.session.withdraw(sorted(live, key = _handle_order)):+ pass+ except Exception:+ logger.error("Could not withdraw rows for request_id=%s", request_id, exc_info = True)+ taken_back = False+ for handle in live:+ self._take_stats(handle)+ return taken_back++ def _take_stats(self, handle):+ return self.session.take_stats(handle) if self.session is not None else None+++def _handle_order(handle):+ _request_id, row = handle+ return -1 if row is None else row++def _handle_share_object(backend, cmd: dict, resp_queue: Any) -> None:"""Share a small Python object across MLX distributed ranks."""request_id = cmd.get("request_id", "")@@ -1106,6 +1640,8 @@ def run_inference_process(cancel_event,config: dict,drain_event = None,+ stop_ledger = None,+ pending_teardowns = None,) -> None:"""Subprocess entrypoint. Persistent — runs the command loop until shutdown.@@ -1118,6 +1654,11 @@ def run_inference_process(cancel_event (cleared at the start of every generate), it is never clearedhere, so a generate still queued behind a cancelled one is skipped ratherthan run — the cancel survives the queue handoff.+ stop_ledger: StopLedger in shared memory naming the requests the parent has+ stopped. Read rather than received: a reply decoding beside others is stopped+ by name, and a queue put is not readable the moment it returns.+ pending_teardowns: counts the commands that end everything on their way here, so a+ held command is answered rather than run in front of one. Read, for the same reason."""# Apply request credentials before a Hugging Face import snapshots the environment._apply_worker_hf_token_environment(config)@@ -1263,22 +1804,66 @@ def run_inference_process()return- # Enter the same command loop as the GPU path.logger.info("MLX inference subprocess ready, entering command loop")+ batch = _ResidentBatch(backend, resp_queue)+ deferred: list[dict] = []+ stops = _Stops(stop_ledger, resp_queue, batch, deferred)+ if stop_ledger is not None:+ stop_ledger.worker_reads_this()while True:- try:- cmd = cmd_queue.get(timeout = 1.0)- except _queue.Empty:- continue- except (EOFError, OSError):- return+ stops.answer()+ tearing_down = pending_teardowns is not None and pending_teardowns.any_in_flight()+ if not tearing_down:+ batch.step()+ from_deferred = False+ if _held_head_leaves_the_hold(batch, deferred):+ cmd = deferred.pop(0)+ from_deferred = True+ else:+ try:+ cmd = cmd_queue.get(+ timeout = 0.0+ if (batch.rows_in_flight or deferred) and not tearing_down+ else 1.0+ )+ except _queue.Empty:+ continue+ except (EOFError, OSError):+ batch.close()+ returnif cmd is None:continuecmd_type = cmd.get("type", "")+ if pending_teardowns is not None and cmd_type in _TEARDOWN_COMMANDS:+ pending_teardowns.taken()try:if cmd_type == "generate":if _drain_skip_generate(cmd, resp_queue, drain_event):continue+ if deferred and not from_deferred:+ deferred.append(cmd)+ continue+ reason = batch.unavailable_reason(cmd)+ if reason is None:+ if _teardown_skip(cmd, resp_queue, pending_teardowns):+ continue+ if _stopped_before_it_ran(cmd, resp_queue, stops):+ continue+ if batch.admit(cmd, None):+ continue+ reason = "it does not prepare like the replies in the batch"+ if batch.rows_in_flight:+ logger.info(+ "Holding request_id=%s until the batch takes it or drains: %s",+ cmd.get("request_id", ""),+ reason,+ )+ if from_deferred:+ deferred.insert(0, cmd)+ else:+ deferred.append(cmd)+ continue+ batch.close()cancel_event.clear()# Re-check the drain after clearing: the parent sets drain_event# then cancel_event for an unload, so if that pair landed between@@ -1287,15 +1872,36 @@ def run_inference_process(# which would stall the switch until the dispatcher idle-timeout.if _drain_skip_generate(cmd, resp_queue, drain_event):continue- _handle_generate(backend, cmd, resp_queue, cancel_event)+ if _teardown_skip(cmd, resp_queue, pending_teardowns):+ continue+ if _stopped_before_it_ran(cmd, resp_queue, stops):+ continue+ _dispatch_generate(+ backend,+ cmd,+ resp_queue,+ _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),+ )elif cmd_type == "generate_audio_input":- # Drain discipline as in "generate" (see that branch).if _drain_skip_generate(cmd, resp_queue, drain_event):continue+ if batch.rows_in_flight or (deferred and not from_deferred):+ deferred.append(cmd)+ continue+ batch.close()cancel_event.clear()if _drain_skip_generate(cmd, resp_queue, drain_event):continue- _handle_generate_audio_input(backend, cmd, resp_queue, cancel_event)+ if _teardown_skip(cmd, resp_queue, pending_teardowns):+ continue+ if _stopped_before_it_ran(cmd, resp_queue, stops):+ continue+ _handle_generate_audio_input(+ backend,+ cmd,+ resp_queue,+ _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),+ )elif cmd_type == "generate_audio":# No TTS here, but codec checkpoints still reach this loop# (dispatch is by device). Answer, or the parent waits 120s.@@ -1317,19 +1923,33 @@ def run_inference_process(},)elif cmd_type == "count_tokens":- _handle_count_tokens(backend, cmd, resp_queue)+ if batch.rows_in_flight:+ _decline_count_tokens(cmd, resp_queue, "A generation is in progress.")+ else:+ _handle_count_tokens(backend, cmd, resp_queue)elif cmd_type == "share_object":_handle_share_object(backend, cmd, resp_queue)elif cmd_type == "load":+ batch.cancel_all()+ batch.close()+ _abandon_held_commands(deferred, resp_queue)if backend.active_model_name:backend.unload_model(backend.active_model_name)_handle_load(backend, cmd, resp_queue)elif cmd_type == "unload":+ batch.cancel_all()+ batch.close()+ _abandon_held_commands(deferred, resp_queue)_handle_unload(backend, cmd, resp_queue)elif cmd_type == "cancel":+ batch.cancel_all()+ _abandon_held_commands(deferred, resp_queue)cancel_event.set()elif cmd_type == "reset":+ batch.cancel_all()+ batch.close()cancel_event.set()+ _abandon_held_commands(deferred, resp_queue)backend.reset_generation_state()_send_response(resp_queue, {"type": "reset_ack"})elif cmd_type == "gpu_memory":@@ -1355,6 +1975,7 @@ def run_inference_process(},)elif cmd_type == "shutdown":+ batch.close()returnelse:# As in the GPU loop: dropping a command silently costs the@@ -1560,7 +2181,7 @@ def run_inference_process(# the switch until the dispatcher idle-timeout tears the subprocess down.if _drain_skip_generate(cmd, resp_queue, drain_event):continue- _handle_generate(backend, cmd, resp_queue, cancel_event)+ _dispatch_generate(backend, cmd, resp_queue, cancel_event)elif cmd_type == "count_tokens":_decline_count_tokens(cmd, resp_queue)@@ -1586,7 +2207,6 @@ def run_inference_process(_handle_unload(backend, cmd, resp_queue)elif cmd_type == "cancel":- # Redundant with mp.Event but handle gracefully.cancel_event.set()logger.info("Cancel command received")diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 364b3b9a5f2..2f37b0369e0 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -186,11 +186,14 @@ class LoadRequest(BaseModel):ge = PARALLEL_MIN,le = PARALLEL_MAX,description = (- "Parallel decode slots for llama-server (--parallel) for this "- f"load ({PARALLEL_MIN}..{PARALLEL_MAX}). Omit for the server-wide "- "default set at launch (the --parallel CLI flag). The VRAM fitter "- "may launch fewer slots to keep the model fully on GPU. Ignored "- "for non-GGUF models."+ f"Replies this load may decode at once ({PARALLEL_MIN}..{PARALLEL_MAX}). "+ "For GGUF these are llama-server's --parallel slots, and the VRAM "+ "fitter may launch fewer to keep the model fully on GPU; omit for the "+ "server-wide default set at launch. Elsewhere it is a ceiling, not a promise: "+ "a runtime that decodes replies together commits its memory to this width and "+ "spends it on both the replies to one turn and the concurrent chats joining a "+ "batch already decoding, while one that does not answers every reply on its "+ "own whatever this says. The load reply's parallel_slots is what it came to."),)n_batch: Optional[int] = Field(@@ -1221,9 +1224,26 @@ class _InferenceRuntimeFields(BaseModel):context_length_enforced: Optional[bool] = Field(None,description = (- "Whether context_length actually bounds the runtime's KV cache. True confirmed, "- "false confirmed unbounded, null the backend does not answer. MLX builds a cache "- "to check, since a model with its own make_cache ignores the requested size."+ "Whether context_length bounds the runtime's KV cache for a reply decoded on "+ "its own. True confirmed, false confirmed unbounded, null the backend does not "+ "answer. MLX builds a cache to check, since a model with its own make_cache "+ "ignores the requested size. Replies decoded together are answered for by "+ "context_unbounded_when_batched, not by this."+ ),+ )+ context_unbounded_when_batched: bool = Field(+ False,+ description = (+ "Whether replies this load decodes together escape the window this reports. "+ "mlx-vlm's batch generator takes no window control, so a vision load able to "+ "run either batch reports true; everything else, llama.cpp included, reports "+ "false. Which batch a given reply gets is a per-request question this cannot "+ "settle, so it errs towards not promising a window. Both fields describe the "+ "load as it was loaded and neither changes afterwards, so a client combines "+ "them with parallel_slots to decide what to tell a user: a load reporting true "+ "and more than one slot cannot promise a window. That is the load's answer, not "+ "the reply's -- a load whose release keeps no batch open decodes a plain chat "+ "bounded while still reporting true -- so read it as the weaker claim it is."),)mlx_context_budget: Optional[int] = Field(@@ -1430,18 +1450,23 @@ class _InferenceRuntimeFields(BaseModel):requested_parallel_slots: Optional[int] = Field(None,description = (- "Parallel decode slots the load was invoked with (per-load "- "n_parallel, else the server-wide --parallel default). None for "- "non-GGUF loads and for the diffusion runner, which ignores "- "--parallel."+ "Replies the load was invoked with (per-load n_parallel, else the "+ "default; for GGUF, the server-wide --parallel set at launch). None "+ "for the diffusion runner, which ignores --parallel."),)parallel_slots: Optional[int] = Field(None,description = (- "Serving slots the active llama-server actually runs (--parallel "- "after any fit-time slot reduction). None for non-GGUF loads and "- "for the diffusion runner, which ignores --parallel."+ "Replies the loaded model is configured to decode at once. Not what any "+ "one reply is running under: a batch already decoding keeps the width it "+ "opened at until it drains, since resizing it would drop the replies "+ "inside it, and a request submitted before a change carries the width it "+ "was submitted with. For "+ "GGUF these are llama-server's --parallel slots after any fit-time "+ "reduction; elsewhere it is 1 when the runtime has no batched "+ "generation, or its settings rule it out. None for the diffusion "+ "runner, which ignores --parallel, and when the runtime did not say."),)requested_n_batch: Optional[int] = Field(diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex ee0c82474ac..a9de136bca0 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -7595,6 +7595,19 @@ def _is_explicit_tensor_drop(request: LoadRequest) -> bool:return override is not None and override.strip().lower() != "tensor"+def _unsloth_serving_fields(model_info: dict) -> dict:+ """What a non-GGUF load says about serving replies, for the replies that report it."""+ slots = model_info.get("parallel_slots")+ batches = model_info.get("can_batch")+ effective = None if slots is None or batches is None else (slots if batches else 1)+ return {+ "requested_parallel_slots": slots,+ "parallel_slots": effective,+ "context_length_enforced": model_info.get("context_length_enforced"),+ "context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),+ }++def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:"""Runtime state shared by load, dedupe, and status; duplicates echo active settings."""fields = {@@ -7613,8 +7626,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:mlx_kv_quant_note = None,mlx_context_budget = None,chat_template_override_reason = None,- # llama.cpp allocates the window it reports: bounded by construction.context_length_enforced = True,+ context_unbounded_when_batched = False,# Older/custom backend doubles predate this additive runtime field.preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),speculative_type = llama_backend.requested_spec_mode,@@ -16600,6 +16613,7 @@ async def _load_model_impl():api_monitor.discard(_load_event) # nothing loaded, no monitor rowlogger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")+ backend.set_parallel_slots(_n_parallel)# A no-op Unsloth load of a preview-owned checkpoint still claims it._set_preview_resident(None)inference_config = load_inference_config(backend.active_model_name)@@ -16636,6 +16650,7 @@ async def _load_model_impl(has_audio_input = _model_info.get("has_audio_input", False),has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),+ **_unsloth_serving_fields(_model_info),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),@@ -16661,7 +16676,6 @@ async def _load_model_impl(_model_info.get("native_context_length")),max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),- context_length_enforced = _model_info.get("context_length_enforced"),mlx_context_budget = _model_info.get("mlx_context_budget"),chat_template = _chat_template,)@@ -17323,6 +17337,7 @@ async def _load_model_impl(on_prior_worker_released = _release_chat_after_teardown,post_handoff_expected_free_gb = post_chat_handoff_expected_free_gb,audio_device = request.audio_device,+ n_parallel = _n_parallel,cache_environment = cache_environment,**anonymous_hf_kw,**speech_codec_kw,@@ -17449,6 +17464,7 @@ async def _load_model_impl(has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),has_video_input = _model_info.get("has_video_input", False),is_mlx = bool(_model_info.get("is_mlx", False)),+ **_unsloth_serving_fields(_model_info),mlx_kv_bits = _model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),@@ -17470,7 +17486,6 @@ async def _load_model_impl(context_length = _positive_int_or_none(_model_info.get("context_length")),native_context_length = _positive_int_or_none(_model_info.get("native_context_length")),max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),- context_length_enforced = _model_info.get("context_length_enforced"),mlx_context_budget = _model_info.get("mlx_context_budget"),chat_template = _chat_template,)@@ -19486,6 +19501,9 @@ async def generate_stream(breakchunk = await asyncio.to_thread(next, gen, _DONE)if chunk is _DONE:+ if cancel_event.is_set():+ mark_response_failed(_gs_scope)+ breakcompleted = Truebreakif isinstance(chunk, GenStreamError):@@ -19890,6 +19908,7 @@ async def get_status(current_subject: str):has_audio_input = has_audio_input,has_video_input = has_video_input,is_mlx = bool(model_info.get("is_mlx", False)),+ **_unsloth_serving_fields(model_info),mlx_kv_bits = model_info.get("mlx_kv_bits"),mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),mlx_kv_quant_eligibility = model_info.get("mlx_kv_quant_eligibility"),@@ -19913,7 +19932,6 @@ async def get_status(current_subject: str):context_length = _positive_int_or_none(model_info.get("context_length")),native_context_length = _positive_int_or_none(model_info.get("native_context_length")),max_context_length = _positive_int_or_none(model_info.get("max_context_length")),- context_length_enforced = model_info.get("context_length_enforced"),mlx_context_budget = model_info.get("mlx_context_budget"),# 0 is an answer (size it yourself); None means no request is recorded. Either# spelling: the route stamps max_seq_length_requested on every non-GGUF load,@@ -26193,6 +26211,9 @@ async def produce_openai_chat_completions(returnchunk_text = await asyncio.to_thread(next, gen, _DONE)if chunk_text is _DONE:+ if cancel_event.is_set():+ cancelled = True+ mark_current_response_failed()breakif isinstance(chunk_text, GenStreamError):_msg = _friendly_gen_stream_error(chunk_text)@@ -29846,20 +29867,77 @@ async def produce_openai_chat_completions(final = tokenreturn final+ _produced: dict = {}++ async def _produce_choices(_indices):+ """These choices' first replies, asked for in one command."""+ rows = [+ {} if idx == 0 else {"seed": _choice_seed(payload.seed, idx)}+ for idx in _indices+ ]+ batch_stats: dict = {}+ texts = [""] * len(rows)+ _finished = set()++ def _drain_batch():+ for event in backend.generate_chat_batch(+ rows = rows,+ cancel_event = cancel_event,+ stats_holder = batch_stats,+ **gen_kwargs,+ ):+ if isinstance(event, GenStreamError):+ return event+ _row, _text = event+ if _text is None:+ _finished.add(_row)+ else:+ texts[_row] = _text+ return None++ failed = await _run_blocking_generation(+ _drain_batch,+ cancel_event,+ name = "openai-chat-nonstream-batch",+ )+ _reported = batch_stats.get("stats") or []++ def _keep(_row, idx):+ _stats = _reported[_row] if _row < len(_reported) else None+ if texts[_row] or _stats is not None:+ _produced[idx] = (texts[_row], _stats)++ if isinstance(failed, GenStreamError):+ backend.reset_generation_state(cancel_event)+ logger.warning(+ "Falling back to one command per choice: %s",+ _friendly_gen_stream_error(failed),+ )+ for _row, idx in enumerate(_indices):+ if _row in _finished:+ _keep(_row, idx)+ return False+ for _row, idx in enumerate(_indices):+ _keep(_row, idx)+ return any(idx in _produced for idx in _indices)+async def _one_choice(choice_index):"""One sampled answer, healing and nudge retry included.- A full generation of its own -- its own seed, its own stats --- exactly as the llama-server path drains its own ``n``.+ Its own seed and its own stats, exactly as the llama-server path drains+ its own ``n``. Everything after the reply runs per choice either way."""# Cleared per choice: one cancelled before it publishes must report# nothing rather than inherit the previous choice's token count.stats_holder.pop("stats", None)- full_text = await _run_blocking_generation(- lambda: _drain_generate(choice_index = choice_index),- cancel_event,- name = "openai-chat-nonstream",- )+ if choice_index in _produced:+ full_text, stats_holder["stats"] = _produced[choice_index]+ else:+ full_text = await _run_blocking_generation(+ lambda: _drain_generate(choice_index = choice_index),+ cancel_event,+ name = "openai-chat-nonstream",+ )if isinstance(full_text, GenStreamError):backend.reset_generation_state(cancel_event)_msg = _friendly_gen_stream_error(full_text)@@ -29892,10 +29970,12 @@ async def produce_openai_chat_completions(if _reasoning_text:_msg["reasoning_content"] = _reasoning_text# Budget exhaustion unless a heal below promotes a tool call; a cancelled turn- # stopped on request, not at the cap, so it stays "stop".+ # stopped on request, not at the cap, so it stays "stop". A reply the+ # batch produced reports its own end instead: a Stop aimed at some other+ # choice must not rewrite the reason this one already finished for._finish = ("stop"- if cancel_event.is_set()+ if cancel_event.is_set() and choice_index not in _producedelse _stats_finish_reason(stats_holder.get("stats")))if _sf_heal:@@ -29985,12 +30065,19 @@ async def produce_openai_chat_completions(_msg["tool_calls"] = _tcs[:1]return _msg, _finish, stats_holder.get("stats")+ from core.inference.llama_server_args import PARALLEL_DEFAULT++ _width = getattr(backend, "effective_parallel_slots", PARALLEL_DEFAULT)+ if _n > 1 and _width > 1 and payload.use_adapter is None and not cancel_event.is_set():+ for _start in range(0, _n, _width):+ if cancel_event.is_set():+ break+ if not await _produce_choices(range(_start, min(_start + _width, _n))):+ break+for _idx in range(_n):- # Stop spawning the remaining choices once cancelled. Index 0 runs- # regardless, so a cancelled turn still answers with one choice- # rather than the empty `choices` a client indexes into.- if _idx and cancel_event.is_set():- break+ if _idx and cancel_event.is_set() and _idx not in _produced:+ continue_msg, _finish, _choice_stats = await _one_choice(_idx)_choices.append(CompletionChoice(@@ -30050,9 +30137,11 @@ async def produce_openai_chat_completions(if len(_choices) > 1:# Every choice, labelled, as the llama-server drain reports them:# showing the last one alone would hide the rest of the turn.+ # Numbered from the choice's own index, not its place in the list: a+ # cancelled turn skips the choices the batch never produced._monitor_reply = "\n\n".join(- f"Choice {_i + 1}:\n{_reply_text(_reply, _choice.finish_reason)}"- for _i, (_reply, _choice) in enumerate(zip(_monitor_replies, _choices))+ f"Choice {_choice.index + 1}:\n{_reply_text(_reply, _choice.finish_reason)}"+ for _reply, _choice in zip(_monitor_replies, _choices))else:_monitor_reply = _reply_text(_msg, _finish)@@ -30558,8 +30647,13 @@ def _openai_model_objects() -> list[dict]:_value = _positive_int_or_none(model_info.get(_field))if _value is not None:entry[_field] = _value- if model_info.get("context_length_enforced") is not None:- entry["context_length_enforced"] = bool(model_info["context_length_enforced"])+ _serving = _unsloth_serving_fields(model_info)+ if _serving["context_length_enforced"] is not None:+ entry["context_length_enforced"] = bool(_serving["context_length_enforced"])+ if _serving["context_unbounded_when_batched"]:+ entry["context_unbounded_when_batched"] = True+ if _serving["parallel_slots"] is not None:+ entry["parallel_slots"] = _serving["parallel_slots"]models.append(entry)return modelsdiff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.pyindex ba124b0581e..9e1d0d574bc 100644--- a/studio/backend/tests/test_active_generations.py+++ b/studio/backend/tests/test_active_generations.py@@ -7,15 +7,18 @@ A load/unload has to know which streaming chats it would interrupt. Everythingunder test is a dict + threading.Lock, so this passes on every platform."""+import multiprocessing as _mpimport osimport sysimport threading+import uuidimport pytest_backend = os.path.join(os.path.dirname(__file__), "..")sys.path.insert(0, _backend)+from core.inference.worker import _SLOTS, StopLedgerfrom state import active_generations@@ -276,16 +279,25 @@ def test_tracked_cancel_shares_its_event_with_the_registry():tracker.__exit__(None, None, None)-def _stub_load_route(monkeypatch, *, active_model_name):- """Point POST /load at an in-memory safetensors backend.-- active_model_name == the requested path makes the request idempotent, so- _load_model_impl takes its already_loaded fast return.- """+def _stub_load_route(+ monkeypatch,+ *,+ active_model_name,+ backend = None,+):from types import SimpleNamespaceimport routes.inference as inf_mod+ from core.inference.orchestrator import InferenceOrchestrator++ if backend is None:+ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)+ backend._stop_ledger = None+ backend._pending_teardowns = None+ backend.active_model_name = active_model_name+ backend.models = {}+monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", lambda: None)monkeypatch.setattr(inf_mod, "validate_extra_args", lambda args: [])monkeypatch.setattr(@@ -307,11 +319,7 @@ def _stub_load_route(monkeypatch, *, active_model_name):},)monkeypatch.setattr(inf_mod, "_resolve_loaded_trust_remote_code", lambda *a, **k: False)- monkeypatch.setattr(- inf_mod,- "get_inference_backend",- lambda: SimpleNamespace(active_model_name = active_model_name, models = {}),- )+ monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)monkeypatch.setattr(inf_mod,"get_llama_cpp_backend",@@ -343,6 +351,27 @@ def test_idempotent_load_neither_refuses_nor_cancels_running_chats(monkeypatch):assert not ev.is_set()+def test_a_width_change_reaches_the_resident_model_without_replacing_it(monkeypatch):+ _route_gate()+ import asyncio++ from models.inference import LoadRequest++ inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A")+ applied = []+ inf_mod.get_inference_backend().set_parallel_slots = applied.append++ ev = threading.Event()+ with active_generations.ActiveGeneration(ev, thread_id = "t1"):+ response = asyncio.run(+ inf_mod.load_model(LoadRequest(model_path = "org/A", n_parallel = 2), object(), "tester")+ )++ assert response.status == "already_loaded"+ assert applied == [2]+ assert not ev.is_set(), "a width change ended a reply that was decoding"++def test_a_real_reload_still_refuses_while_chats_stream(monkeypatch):# A load that would really replace the model still 409s and names the chats._route_gate()@@ -2550,6 +2579,70 @@ def test_drain_returns_as_soon_as_the_cancelled_requests_unwind(monkeypatch):assert polls == 3+def _ids(n):+ return [str(uuid.uuid4()) for _ in range(n)]+++def _stopped(ledger):+ """What the worker would read: the only way the record is read."""+ return ledger.snapshot()[1]+++def test_a_request_reads_as_stopped_once_it_is():+ ledger = StopLedger(_mp.get_context("spawn"))+ mine, theirs = _ids(2)++ assert _stopped(ledger) == set()+ assert ledger.stop(mine)+ assert _stopped(ledger) == {mine}, "a stop names one request and no other"+ assert theirs not in _stopped(ledger)+++def test_the_oldest_stop_is_the_one_that_ages_out():+ ledger = StopLedger(_mp.get_context("spawn"))+ recorded = _ids(_SLOTS + 1)+ for request_id in recorded:+ assert ledger.stop(request_id)++ assert _stopped(ledger) == set(recorded[1:]), "the oldest made way for the newest"+++def test_stopping_the_same_request_twice_spends_one_slot():+ ledger = StopLedger(_mp.get_context("spawn"))+ theirs, mine = _ids(2)+ assert ledger.stop(theirs)+ for _ in range(_SLOTS):+ assert ledger.stop(mine)++ assert _stopped(ledger) == {mine, theirs}, "the one stopped first is still stopped"+++def test_a_snapshot_answers_for_every_reply_at_once():+ ledger = StopLedger(_mp.get_context("spawn"))+ mine, absent = _ids(2)+ theirs = "short"+ ledger.stop(mine)+ ledger.stop(theirs)++ written, stopped = ledger.snapshot()+ assert written == 2+ assert stopped == {mine, theirs}, "the ids as recorded, with nothing padded onto them"+ assert absent not in stopped+++class _Ledger:+ def __init__(self, read_by_worker = False):+ self.stopped: set = set()+ self._read_by_worker = read_by_worker++ def stop(self, request_id):+ self.stopped.add(request_id)+ return True++ def read_by_worker(self) -> bool:+ return self._read_by_worker++# ── queued chats must not cancel the running one ──────────────────────@@ -2560,10 +2653,15 @@ def _orchestrator_for_ownership():"core.inference.orchestrator", reason = "inference stack not installed")orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)+ orch._stop_ledger = _Ledger()+ orch._pending_teardowns = Noneorch._gen_lock = threading.Lock()orch._active_cancel_events = []orch._executing_cancel_events = []orch._active_cancel_lock = threading.Lock()+ orch._send_order_lock = threading.Lock()+ orch._mailbox_lock = threading.Lock()+ orch._request_cancel_events = {}orch._cancel_event = threading.Event()orch._ensure_subprocess_alive = lambda: False # stop before _send_cmdreturn orch@@ -2738,6 +2836,8 @@ def test_claim_order_matches_send_order_under_concurrent_dispatch():"core.inference.orchestrator", reason = "inference stack not installed")orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)+ orch._stop_ledger = None+ orch._pending_teardowns = Noneorch._active_cancel_events = []orch._executing_cancel_events = []orch._active_cancel_lock = threading.Lock()diff --git a/studio/backend/tests/test_audio_tts_cancellation.py b/studio/backend/tests/test_audio_tts_cancellation.pyindex 25aa74f3205..177b6715133 100644--- a/studio/backend/tests/test_audio_tts_cancellation.py+++ b/studio/backend/tests/test_audio_tts_cancellation.py@@ -73,6 +73,8 @@ from models.inference import ChatCompletionRequest # noqa: E402def _bare_orchestrator():orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orchestrator._stop_ledger = None+ orchestrator._pending_teardowns = Noneorchestrator._gen_lock = threading.Lock()orchestrator._send_order_lock = threading.Lock()orchestrator._active_cancel_lock = threading.Lock()@@ -86,13 +88,14 @@ def _bare_orchestrator():orchestrator._dispatcher_thread = Noneorchestrator._dispatcher_stop = threading.Event()orchestrator._dispatcher_lifecycle_lock = threading.Lock()+ orchestrator._worker_released = threading.Condition(orchestrator._dispatcher_lifecycle_lock)orchestrator._subprocess_shutdown_lock = threading.Lock()orchestrator._mailbox_lock = threading.Lock()orchestrator._mailboxes = {}orchestrator._direct_mailboxes = {}orchestrator._request_cancel_events = {}orchestrator._unload_pending = False- orchestrator._exclusive_tts_pending = False+ orchestrator._worker_reserved_for = Noneorchestrator.active_model_name = "model"orchestrator.models = {"model": {}}orchestrator.loading_models = set()@@ -222,7 +225,11 @@ def test_audio_response_cancellation_signals_worker_and_drains_terminal_responsemonkeypatch.setattr(orchestrator,"_direct_reader",- lambda _request_id: (read_one, lambda **_kwargs: None, lambda: released.append(True)),+ lambda _request_id, _cancel_event = None: (+ read_one,+ lambda **_kwargs: None,+ lambda: released.append(True),+ ),)with pytest.raises(RuntimeError, match = "cancel"):@@ -330,7 +337,7 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):monkeypatch.setattr(orchestrator,"_direct_reader",- lambda _request_id: (+ lambda _request_id, _cancel_event = None: (read_one,lambda **_kwargs: pytest.fail("the cancellation drain window was already spent"),lambda: None,@@ -339,7 +346,7 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):shutdown_state = []def shutdown(*, timeout):- shutdown_state.append((orchestrator._exclusive_tts_pending, timeout))+ shutdown_state.append((orchestrator._worker_reserved_for, timeout))return Truemonkeypatch.setattr(orchestrator, "_shutdown_subprocess", shutdown)@@ -350,10 +357,10 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):assert time.monotonic() - started < 0.5assert cancel_signals == [True]- assert shutdown_state == [(True, 0.03)]+ assert shutdown_state == [("audio generation is in progress", 0.03)]assert orchestrator.active_model_name is Noneassert orchestrator.models == {}- assert orchestrator._exclusive_tts_pending is False+ assert orchestrator._worker_reserved_for is Nonedef test_audio_response_cancellation_before_worker_start_is_still_bounded(monkeypatch):@@ -382,7 +389,7 @@ def test_audio_response_cancellation_before_worker_start_is_still_bounded(monkeymonkeypatch.setattr(orchestrator,"_direct_reader",- lambda _request_id: (+ lambda _request_id, _cancel_event = None: (read_one,lambda **_kwargs: pytest.fail("the cancellation drain window was already spent"),lambda: None,@@ -401,7 +408,7 @@ def test_audio_response_cancellation_before_worker_start_is_still_bounded(monkeyassert time.monotonic() - started < 0.5assert shutdown_state == [0.03]- assert orchestrator._exclusive_tts_pending is False+ assert orchestrator._worker_reserved_for is Nonedef test_audio_generation_timeout_scales_with_requested_tokens(monkeypatch):@@ -444,7 +451,7 @@ def test_audio_worker_command_uses_the_model_token_budget(sent = []monkeypatch.setattr(orchestrator, "_send_cmd", lambda cmd: sent.append(cmd))- def direct_reader(request_id):+ def direct_reader(request_id, cancel_event = None):responses = queue.Queue()responses.put({@@ -491,7 +498,7 @@ def test_audio_response_timeout_cancels_and_drains_before_releasing(monkeypatch)cancel_state = []def cancel_generation():- cancel_state.append(orchestrator._exclusive_tts_pending)+ cancel_state.append(orchestrator._worker_reserved_for)orchestrator._cancel_event.set()orchestrator._resp_queue.put({@@ -506,8 +513,10 @@ def test_audio_response_timeout_cancels_and_drains_before_releasing(monkeypatch)with pytest.raises(RuntimeError, match = "Timeout waiting for audio generation"):orchestrator.generate_audio_response("hello")- assert cancel_state == [True], "timeout cancellation must occur under TTS exclusivity"- assert orchestrator._exclusive_tts_pending is False+ assert cancel_state == [+ "audio generation is in progress"+ ], "timeout cancellation must occur under TTS exclusivity"+ assert orchestrator._worker_reserved_for is Noneassert orchestrator._active_cancel_events == []assert orchestrator._executing_cancel_events == []@@ -530,7 +539,7 @@ def test_audio_response_timeout_tears_down_unresponsive_worker_before_release(moshutdown_state = []def shutdown(*, timeout):- shutdown_state.append((orchestrator._exclusive_tts_pending, timeout))+ shutdown_state.append((orchestrator._worker_reserved_for, timeout))return Truemonkeypatch.setattr(orchestrator, "_shutdown_subprocess", shutdown)@@ -538,8 +547,8 @@ def test_audio_response_timeout_tears_down_unresponsive_worker_before_release(mowith pytest.raises(RuntimeError, match = "Timeout waiting for audio generation"):orchestrator.generate_audio_response("hello")- assert shutdown_state == [(True, 0.03)]- assert orchestrator._exclusive_tts_pending is False+ assert shutdown_state == [("audio generation is in progress", 0.03)]+ assert orchestrator._worker_reserved_for is Noneassert orchestrator.active_model_name is Noneassert orchestrator.models == {}@@ -598,14 +607,14 @@ class _AliveDispatcher:def test_dispatcher_refuses_during_exclusive_tts_and_resumes_after():orchestrator = _bare_orchestrator()orchestrator._resp_queue = queue.Queue()- orchestrator._exclusive_tts_pending = True+ orchestrator._worker_reserved_for = "audio generation is in progress"- assert orchestrator._start_dispatcher() is False+ assert orchestrator._start_dispatcher() is Noneassert orchestrator._dispatcher_thread is None- orchestrator._exclusive_tts_pending = False+ orchestrator._worker_reserved_for = Nonetry:- assert orchestrator._start_dispatcher() is True+ assert orchestrator._start_dispatcher() is orchestrator._dispatcher_threadassert orchestrator._dispatcher_thread is not Noneassert orchestrator._dispatcher_thread.is_alive()finally:@@ -628,7 +637,7 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):responses = queue.Queue()- def direct_reader(request_id):+ def direct_reader(request_id, cancel_event = None):responses.put({"type": "audio_started", "request_id": request_id})responses.put({@@ -652,9 +661,9 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):thread.start()deadline = time.monotonic() + 2- while not orchestrator._exclusive_tts_pending and time.monotonic() < deadline:+ while not orchestrator._worker_reserved_for and time.monotonic() < deadline:time.sleep(0.01)- assert orchestrator._exclusive_tts_pending is True+ assert orchestrator._worker_reserved_for == "audio generation is in progress"assert sent == [], "TTS must not enqueue behind the active compare request"with orchestrator._mailbox_lock:@@ -664,7 +673,7 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):assert thread.is_alive() is Falseassert result["value"] == (b"RIFFfake", 24000)assert sent and sent[0]["type"] == "generate_audio"- assert orchestrator._exclusive_tts_pending is False+ assert orchestrator._worker_reserved_for is Nonedef test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):@@ -691,9 +700,9 @@ def test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):thread = threading.Thread(target = run)thread.start()deadline = time.monotonic() + 2- while not orchestrator._exclusive_tts_pending and time.monotonic() < deadline:+ while not orchestrator._worker_reserved_for and time.monotonic() < deadline:time.sleep(0.01)- assert orchestrator._exclusive_tts_pending is True+ assert orchestrator._worker_reserved_for == "audio generation is in progress"caller_cancel.set()thread.join(timeout = 2)@@ -702,30 +711,7 @@ def test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):assert "cancel" in str(error["value"]).lower()assert cancel_calls == []assert set(orchestrator._mailboxes) == {"compare"}- assert orchestrator._exclusive_tts_pending is False---def test_dispatched_generation_rechecks_tts_reservation_before_registration(monkeypatch):- orchestrator = _bare_orchestrator()- orchestrator._dispatcher_thread = _AliveDispatcher()- monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: True)- monkeypatch.setattr(orchestrator, "_start_dispatcher", lambda: False)-- def reserve_tts(*_args, **_kwargs):- orchestrator._exclusive_tts_pending = True- return {"type": "generate", "request_id": "compare-1"}-- monkeypatch.setattr(orchestrator, "_build_generate_cmd", reserve_tts)- monkeypatch.setattr(- orchestrator,- "_send_cmd",- lambda _cmd: pytest.fail("compare must not enqueue after TTS reservation"),- )-- output = list(orchestrator._generate_dispatched(messages = [{"role": "user", "content": "x"}]))-- assert any("audio generation" in str(chunk).lower() for chunk in output)- assert orchestrator._mailboxes == {}+ assert orchestrator._worker_reserved_for is Nonedef test_worker_audio_forwards_shared_cancel_event():diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.pyindex 5dfff397478..a1f9494acbc 100644--- a/studio/backend/tests/test_gguf_load_cache_reuse.py+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py@@ -1062,6 +1062,8 @@ class TestLoadHubDownloadExclusion:"chat_template_override_reason",# Constant True: llama.cpp allocates the window it reports."context_length_enforced",+ # Constant False: each slot decodes against its own window.+ "context_unbounded_when_batched",# Read from requested_extra_args, which is what the load was invoked# with rather than the rewritten launch list."requested_llama_extra_args",diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.pyindex 00172e6a0de..003f7e72d59 100644--- a/studio/backend/tests/test_inference_dispatcher_resilience.py+++ b/studio/backend/tests/test_inference_dispatcher_resilience.py@@ -14,7 +14,6 @@ import astimport queueimport sysimport threading-import timefrom pathlib import Pathimport pytest@@ -38,6 +37,8 @@ class _ScriptedQueue:def _dispatcher():o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._stop_ledger = None+ o._pending_teardowns = Noneo._dispatcher_stop = threading.Event()o._mailbox_lock = threading.Lock()o._mailboxes = {}@@ -128,6 +129,8 @@ def test_route_llama_streaming_async_clients_disable_proxy_env():def _direct_reader_host():"""Orchestrator with only what _direct_reader and the ownership helpers touch."""o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._stop_ledger = None+ o._pending_teardowns = Noneo._mailbox_lock = threading.Lock()o._mailboxes = {}o._direct_mailboxes = {}@@ -185,10 +188,14 @@ def test_rerouting_a_foreign_gen_done_retires_that_request():release()-def _direct_reader_calls(o, request_id):+def _direct_reader_calls(+ o,+ request_id,+ cancel_event = None,+):"""_direct_reader wired to a scripted _read_resp (o._scripted, popped in order)."""o._read_resp = lambda timeout = 1.0, observe = True: (o._scripted.pop(0) if o._scripted else None)- return o._direct_reader(request_id)+ return o._direct_reader(request_id, cancel_event)@pytest.mark.parametrize(diff --git a/studio/backend/tests/test_inference_orchestrator_crash_message.py b/studio/backend/tests/test_inference_orchestrator_crash_message.pyindex be1a673e620..ac32c30c0e3 100644--- a/studio/backend/tests/test_inference_orchestrator_crash_message.py+++ b/studio/backend/tests/test_inference_orchestrator_crash_message.py@@ -21,6 +21,7 @@ def test_subprocess_crash_message_includes_signal_and_oom_hint():spec.loader.exec_module(module)orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)+ orchestrator._stop_ledger = Noneorchestrator._proc = SimpleNamespace(pid = 1234, exitcode = -9)msg = orchestrator._subprocess_crash_message("wait")diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.pyindex 2c2e6901290..61ea1dba778 100644--- a/studio/backend/tests/test_inference_worker_stderr_tail.py+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py@@ -1135,7 +1135,7 @@ def test_a_stream_whose_worker_was_swapped_gets_no_tail_at_all():body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)swap = body.index("initial_proc or self._resp_queue is not initial_resp_queue")- following = body[swap : body.index("resp = read_one(read_timeout)", swap)]+ following = body[swap : body.index("resp = read_one(", swap)]assert "_subprocess_crash_message(crash_context)" in following, followingassert "with_worker_output" not in following, followingdiff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex d94fc4090ff..dfd327f88f9 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -394,12 +394,17 @@ def _install_fake_mlx(monkeypatch):mlx_core.set_wired_limit = _DummyMX.set_wired_limitmlx_core.device_info = _DummyMX.device_infomlx_core.synchronize = _DummyMX.synchronize+ mlx_core.clear_cache = lambda: Nonemlx_utils.tree_unflatten = dict+ mlx_nn = types.ModuleType("mlx.nn")+ mlx_nn.Module = type("Module", (), {"__init__": lambda self: None})mlx_pkg.core = mlx_coremlx_pkg.utils = mlx_utils+ mlx_pkg.nn = mlx_nnmonkeypatch.setitem(sys.modules, "mlx", mlx_pkg)monkeypatch.setitem(sys.modules, "mlx.core", mlx_core)monkeypatch.setitem(sys.modules, "mlx.utils", mlx_utils)+ monkeypatch.setitem(sys.modules, "mlx.nn", mlx_nn)def _install_fake_fast_mlx(monkeypatch, calls):@@ -3323,6 +3328,60 @@ def test_template_probe_renders_through_the_path_generation_uses(monkeypatch):backend._render_template_probe(False)+def _vlm_backend(+ monkeypatch,+ *,+ markers,+ prompt_tail = "",+):+ from core.inference import mlx_inference++ monkeypatch.setattr(+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",+ lambda *a, **k: "<start_of_turn>user\nhi<end_of_turn>\n<start_of_turn>model\n"+ + prompt_tail,+ )+ monkeypatch.setattr(+ mlx_inference, "detect_reasoning_channel_markers", lambda target, tools = None: markers+ )+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)+ backend._tokenizer = SimpleNamespace(chat_template = "t", all_special_tokens = [])+ backend._processor = SimpleNamespace(+ chat_template = "t", tokenizer = backend._tokenizer, apply_chat_template = lambda *a, **k: ""+ )+ backend._model = object()+ return backend+++def _vlm_row(+ backend,+ *,+ stop = None,+ messages = None,+ continue_final_message = False,+):+ """One vision reply resolved by that backend."""+ return backend._plan_vlm_row(+ messages or [{"role": "user", "content": "hi"}],+ None,+ temperature = 0.7,+ top_p = 0.9,+ top_k = 0,+ min_p = 0.0,+ max_new_tokens = 16,+ repetition_penalty = 1.0,+ continue_final_message = continue_final_message,+ stop = stop,+ ).stream+++def test_a_vision_reply_cuts_at_its_own_stop_sequence(monkeypatch):+ """The row carries the request's stop sequences, matched on the sampled text."""+ row = _vlm_row(_vlm_backend(monkeypatch, markers = None), stop = ["HALT"])+ assert row.feed(text = "some text HALT and more") == ["some text "]+ assert row.stopped++def test_the_audio_refusal_puts_the_native_template_back(monkeypatch):"""The refusal itself, not just the marker check.@@ -5232,6 +5291,309 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(assert guard < snapshot, "the guard must be held across the snapshot, not after it"+class _TwoRowStream:+ """A vision batch reporting two rows, recording whatever is taken back."""++ def __init__(+ self,+ events,+ retired = (),+ ):+ self._events, self._retired, self.withdrawn = events, set(retired), []++ def step(self):+ return iter(self._events)++ def withdraw(self, row):+ self.withdrawn.append(row)+ if row in self._retired:+ return None+ return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2])+++def _open_vision_session(rows):+ """A session holding the given rows, with nothing else started."""+ from core.inference import mlx_inference++ session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)+ session._rows = {row.handle: row for row in rows}+ session._by_row, session._settled = {row.row: row for row in rows}, {}+ return session++[email protected]("stopping", (0, 1))[email protected]("terminal", (False, True))+def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, stopping, terminal):+ """Whichever row stops is closed out where it stopped and its neighbour."""+ from core.inference import mlx_inference++ backend = _vlm_backend(monkeypatch, markers = None)+ handles = ("a", "b")+ session = _open_vision_session(+ [+ mlx_inference._VisionBatchRow(+ handle = handle,+ plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"])),+ row = number,+ )+ for number, handle in enumerate(handles)+ ]+ )+ other = 1 - stopping+ result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1, 2])+ session.stream = _TwoRowStream(+ [+ SimpleNamespace(+ index = stopping, delta = "one HALT two", result = result if terminal else None+ ),+ SimpleNamespace(index = other, delta = "still going", result = None),+ ],+ retired = (stopping,) if terminal else (),+ )++ reported = list(session.step())++ assert (handles[stopping], None) in reported, "the stopped row is reported finished"+ assert session._settled[handles[stopping]]["finish_reason"] == "stop"+ assert handles[stopping] not in session._rows, "and it has left the batch"+ assert handles[other] in session._rows, "its neighbour is still in the batch"+ assert (handles[other], "still going") in reported+ assert session.stream.withdrawn == (+ [] if terminal else [stopping]+ ), "a row the engine has already retired is not handed back again"+++def _batch_engine(monkeypatch):+ """The zoo batch engine, or a stand-in where the backend CI installs no unsloth-zoo."""+ try:+ from unsloth_zoo.mlx import generate as engine+ except ImportError:+ engine = types.ModuleType("unsloth_zoo.mlx.generate")+ engine.stream_unavailable_reason = lambda *a, **k: None+ for name in ("unsloth_zoo", "unsloth_zoo.mlx"):+ if not isinstance(sys.modules.get(name), types.ModuleType):+ monkeypatch.setitem(sys.modules, name, types.ModuleType(name))+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", engine)+ monkeypatch.setattr(sys.modules["unsloth_zoo.mlx"], "generate", engine, raising = False)+ return engine+++def test_a_text_load_whose_engine_cannot_batch_says_so_before_it_commits(monkeypatch):+ """The reply is owed the decode it would have had, not the error building raises."""+ from core.inference.mlx_inference import MLXInferenceBackend++ engine = _batch_engine(monkeypatch)++ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False+ for gap in ("BatchGenerator.insert missing", None):+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: gap)+ assert backend.resident_unavailable_reason({}) == gap+ assert backend.batch_unavailable_reason([{}, {}]) == gap+++ "extra",+ (+ {"images": [object()]},+ {"images_base64": ["AAAA"]},+ {"video": "AAAAGGZ0eXBtcDQy"},+ {"video_base64": "AAAAGGZ0eXBtcDQy"},+ {"tools": [{"type": "function", "function": {"name": "lookup"}}]},+ {"tool_protocol_active": True},+ {"response_format": {"type": "json_object"}},+ ),+)+def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):+ """Replayed tool images, a clip and the native tool protocol live only on the"""+ from core.inference.mlx_inference import MLXInferenceBackend++ engine = _batch_engine(monkeypatch)++ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)+ plain = {"messages": [], "images": [], "image_ordinal": 0}++ assert backend.resident_unavailable_reason(plain) is None+ assert backend.batch_unavailable_reason([plain, plain]) is None+ assert backend.resident_unavailable_reason({**plain, **extra}) is not None+ assert backend.batch_unavailable_reason([plain, {**plain, **extra}]) is not None+++def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):+ from core.inference.mlx_inference import MLXInferenceBackend++ engine = _batch_engine(monkeypatch)++ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, None+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)+ assert backend.resident_unavailable_reason({}) is None++ for quant, budget in (({"kv_bits": 8}, None), ({"kv_bits": None}, 4096)):+ backend._kv_quant, backend._kv_context_budget = quant, budget+ assert backend.resident_unavailable_reason({}) is not None+ assert backend.batch_unavailable_reason([{}, {}]) is not None+++try:+ import mlx.core as _mx+ _METAL = _mx.metal.is_available()+except Exception:+ _METAL = False++metal_only = pytest.mark.skipif(not _METAL, reason = "requires Apple Silicon Metal")++BATCH_MODEL = "mlx-community/SmolLM2-135M-Instruct"+BATCH_PROMPT = [{"role": "user", "content": "Name a colour and explain why."}]+BATCH_VLM_MODEL = "mlx-community/Qwen3.5-0.8B-4bit"++[email protected](scope = "module")+def batch_backend(allow_outbound_network):+ from core.inference.mlx_inference import MLXInferenceBackend++ loaded = MLXInferenceBackend()+ with allow_outbound_network():+ loaded.load_model(BATCH_MODEL)+ return loaded++[email protected](scope = "module")+def batch_vlm_backend(allow_outbound_network):+ from core.inference.mlx_inference import MLXInferenceBackend++ loaded = MLXInferenceBackend()+ with allow_outbound_network():+ loaded.load_model(SimpleNamespace(identifier = BATCH_VLM_MODEL, is_vision = True))+ return loaded+++def _batch_request(**overrides):+ base = dict(messages = BATCH_PROMPT, max_new_tokens = 24, temperature = 0.9)+ base.update(overrides)+ return base+++def _drain_batch(backend, requests, **kwargs):+ """Cumulative snapshots per row, plus the stats latched at the end."""+ replies: dict[int, list[str]] = {row: [] for row in range(len(requests))}+ finished = []+ for row, snapshot in backend.generate_chat_batch(requests, **kwargs):+ if snapshot is None:+ finished.append(row)+ else:+ replies[row].append(snapshot)+ assert sorted(finished) == sorted(replies), "every row must report completion once"+ return replies, list(backend.last_batch_generation_stats)+++@metal_only+def test_a_batch_costs_one_forward_per_token_not_one_per_reply(batch_backend):+ model = batch_backend._model+ original = type(model).__call__+ calls = {"n": 0}++ def counted(self, *args, **kwargs):+ calls["n"] += 1+ return original(self, *args, **kwargs)++ type(model).__call__ = counted+ try:+ _replies, stats = _drain_batch(+ batch_backend, [_batch_request(seed = 200 + i) for i in range(4)]+ )+ finally:+ type(model).__call__ = original++ generated = sum(entry["usage"]["completion_tokens"] for entry in stats)+ assert generated >= 4 * 20+ assert (+ calls["n"] < generated / 2+ ), f"{calls['n']} forwards for {generated} tokens across 4 replies"+++@metal_only+def test_an_out_of_range_logit_bias_id_cannot_corrupt_a_batch(batch_backend):+ """MLX does no bounds checking on the gather these processors do."""+ plain = _batch_request(seed = 21)+ stray = _batch_request(seed = 21, logit_bias = {10**9: 5.0, -4: 3.0})+ _first, _ = _drain_batch(batch_backend, [plain, plain])+ opener = batch_backend._tokenizer.encode(_first[0][-1])[0]+ real = _batch_request(seed = 21, logit_bias = {int(opener): -100.0})++ replies, stats = _drain_batch(batch_backend, [plain, stray, stray, real])+ assert all(entry["usage"]["completion_tokens"] > 0 for entry in stats)+ assert replies[1] == replies[0]+ assert replies[2] == replies[0]+ assert replies[3] != replies[0]+++def test_a_repeated_token_is_charged_the_penalty_it_earned():+ """The frequency processor counts occurrences in float32 and scales once."""+ mx = pytest.importorskip("mlx.core")++ from core.inference.mlx_inference import _make_mlx_frequency_penalty_processor++ processor = _make_mlx_frequency_penalty_processor(0.3)+ logits = mx.zeros((1, 8), dtype = mx.float16)+ processor(mx.array([0, 1]), logits) # latches the prompt length+ tokens = mx.array([0, 1] + [5] * 1000)+ charged = processor(tokens, logits)+ assert abs(float(charged[0, 5]) + 300.0) < 0.5, float(charged[0, 5])+ assert float(charged[0, 4]) == 0.0+++@metal_only+def test_a_loaded_vision_model_says_a_batch_would_leave_its_window_behind(batch_vlm_backend):+ stored = batch_vlm_backend.models[batch_vlm_backend.active_model_name]++ assert stored["context_unbounded_when_batched"] is True+++@metal_only+def test_a_vision_reply_stopped_partway_reports_the_tokens_it_actually_used(batch_vlm_backend):+ """A stopped row has no result, so its usage has to come from what it produced."""+ import threading++ from PIL import Image++ def rows():+ return [+ dict(+ messages = [{"role": "user", "content": "Describe this in detail."}],+ image = Image.new("RGB", (64, 64), (12, 200, 60)),+ max_new_tokens = 64,+ temperature = 0.0,+ )+ for _ in range(2)+ ]++ _replies, whole = _drain_batch(batch_vlm_backend, rows())++ cancel = threading.Event()+ delivered: dict[int, list[str]] = {0: [], 1: []}+ for row, snapshot in batch_vlm_backend.generate_chat_batch(rows(), cancel_event = cancel):+ if snapshot is not None:+ delivered[row].append(snapshot)+ if all(len(seen) >= 3 for seen in delivered.values()):+ cancel.set()++ cut_short = batch_vlm_backend.last_batch_generation_stats+ assert all(delivered.values()), "nothing was delivered before the batch was cancelled"+ for row, seen in delivered.items():+ usage, entire = cut_short[row]["usage"], whole[row]["usage"]+ assert (+ usage["prompt_tokens"] == entire["prompt_tokens"]+ ), "a stopped row read a different prompt than the same row read whole"+ assert len(seen) <= usage["completion_tokens"] < entire["completion_tokens"], (+ f"row {row} delivered {len(seen)} snapshots and reported "+ f"{usage['completion_tokens']} of {entire['completion_tokens']} tokens"+ )++import numpy as np # noqa: E402from core.inference import grammar_constraint as gcdiff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.pynew file mode 100644index 00000000000..2dfa88027a1--- /dev/null+++ b/studio/backend/tests/test_mlx_resident_batch.py@@ -0,0 +1,438 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Replies joining and leaving a batch the MLX worker keeps open."""++import functools+import importlib+import inspect+import multiprocessing as _mp+import queue as _queue+import sys+import threading+import types+from types import SimpleNamespace+from unittest.mock import MagicMock++import pytest+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - stub unusable imports+ pass+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))++from core.inference import worker # noqa: E402+from core.inference.worker import RowRefused # noqa: E402+from core.inference.worker import StopLedger # noqa: E402+++class _RespQueue:+ def __init__(self):+ self.sent = []++ def put(self, item, *a, **k):+ self.sent.append(item)+++class _Session:+ """A batch that hands back whatever the test scripted for each reply."""++ def __init__(self, *, width):+ self.opened_at = width+ self.rows, self.ending, self.refusals = [], [], []+ self.settled, self.script, self.stats_at_retire = {}, {}, {}+ self.will_not_take, self.closed, self.admitted = set(), False, []++ @property+ def rows_in_flight(self):+ return len(self.rows)++ @property+ def handles(self):+ return list(self.rows)++ def take_stats(self, handle):+ return self.settled.pop(handle, None)++ def admit(self, request, handle):+ if handle[0] in self.will_not_take or handle in self.will_not_take:+ self.refusals.append(handle)+ raise RowRefused("it does not prepare like the replies in the batch")+ self.admitted.append((handle, request))+ self.rows.append(handle)+ return ""++ def _retire(self, handle):+ self.rows.remove(handle)+ self.settled[handle] = self.stats_at_retire.get(handle, {})++ def step(self):+ for handle in list(self.rows):+ events = self.script.get(handle) or []+ if not events:+ continue+ event = events.pop(0)+ if event is None:+ self._retire(handle)+ yield handle, event++ def withdraw(self, handles):+ for handle in [h for h in handles if h in self.rows]:+ self._retire(handle)+ yield handle, None++ def close(self):+ self.closed = True+++class _Backend:+ def __init__(+ self,+ reason = None,+ script = None,+ ):+ self.reason = reason+ self.script = script or {}+ self.sessions = []+ self.on_open = None+ self.on_reason = None+ self.apart = []++ def generate_chat_response(self, **kwargs):+ self.apart.append(kwargs)+ yield "apart"++ def resident_unavailable_reason(self, request):+ if self.on_reason is not None:+ self.on_reason()+ return self.reason++ def open_resident_batch(+ self,+ *,+ width,+ adapter_state = None,+ ):+ session = _Session(width = width)+ session.script.update(self.script)+ if self.on_open is not None:+ self.on_open(session)+ self.sessions.append(session)+ return session+++def _cmd(request_id = "r1", **extra):+ return {"type": "generate", "request_id": request_id, "messages": [], **extra}+++def _batch(backend = None):+ resp = _RespQueue()+ return worker._ResidentBatch(backend or _Backend(), resp), resp+++def test_replies_a_command_asked_for_together_are_addressed_and_counted_by_row():+ batch, resp = _batch()+ batch.admit(_cmd(rows = [{"seed": 1}, {"seed": 2}]), None)+ for row, stats in ((0, {"n": 1}), (1, {"n": 2})):+ batch.session.script[("r1", row)] = ["ab"[row], None]+ batch.session.stats_at_retire[("r1", row)] = stats+ for _ in range(2):+ batch.step()++ rows = [(m["type"], m.get("row")) for m in resp.sent]+ assert ("token", 0) in rows and ("token", 1) in rows+ assert rows.count(("row_done", 0)) == 1 and rows.count(("row_done", 1)) == 1+ done = [m for m in resp.sent if m["type"] == "row_done"]+ assert sorted(m["stats"]["n"] for m in done) == [1, 2]+ assert [m["type"] for m in resp.sent][-1] == "gen_done"+ assert resp.sent[-1]["stats"] is None+++IDLE = object()+++def _run_loop(monkeypatch, backend, cmds):+ """Drive the real worker loop over a scripted queue."""+ import queue as _stdqueue++ from utils.hardware import hardware as _hw+ from loggers.config import LogConfig+ from utils import worker_stderr+ import core.inference.mlx_inference as mlx_mod++ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")+ inert = lambda *a, **k: None # noqa: E731+ for name, value in (+ ("is_apple_silicon", lambda: True),+ ("apply_gpu_ids", inert),+ ("_recorded_local_base", lambda m: (None, False)),+ ("_hub_targets_are_local", lambda *a, **k: True),+ ("_activate_transformers_version", inert),+ ("_handle_load", inert),+ ):+ monkeypatch.setattr(worker, name, value)+ monkeypatch.setattr(_hw, "detect_hardware", inert)+ monkeypatch.setattr(_hw, "DEVICE", _hw.DeviceType.MLX)+ monkeypatch.setattr(LogConfig, "setup_logging", staticmethod(lambda *a, **k: None))+ monkeypatch.setattr(worker_stderr, "mark_log_record_continuations", inert)+ monkeypatch.setattr(mlx_mod, "MLXInferenceBackend", lambda *a, **k: backend)++ class _Script:+ def __init__(self, items):+ self._items = list(items)++ def get(self, timeout = None):+ if self._items and (item := self._items.pop(0)) is not IDLE:+ return item+ raise _stdqueue.Empty++ resp = _RespQueue()+ never = SimpleNamespace(is_set = lambda: False, clear = lambda: None, set = lambda: None)+ worker.run_inference_process(+ cmd_queue = _Script([*cmds, {"type": "shutdown"}]),+ resp_queue = resp,+ cancel_event = never,+ config = {"model_name": "unsloth/orpheus-3b-0.1-ft"},+ )+ return resp+++def test_a_one_slot_load_runs_its_reply_without_ever_opening_a_batch(monkeypatch):+ """A batch one reply wide is one nothing can join, so it is never opened."""+ backend = _Backend()++ resp = _run_loop(monkeypatch, backend, [_cmd("r1", parallel_slots = 1), IDLE])++ assert backend.sessions == [], "a batch was opened for a single reply"+ assert [m["type"] for m in resp.sent if m.get("request_id") == "r1"] == ["token", "gen_done"]+++_HOLD_CASES = [+ pytest.param(+ dict(+ cmds = [("r1", 4), ("r2", 2), ("r3", 4), ("r4", 4)],+ script = {"r1": ["a", None], "r2": ["b", None]},+ widths = [4, 2, 4],+ order = ["r1", "r2", "r3", "r4"],+ ),+ id = "the head runs on a queue that never falls quiet",+ ),+ pytest.param(+ dict(+ cmds = [("r1", 4), ("r2", 2), ("r3", 2), ("r4", 2)],+ script = {+ "r1": ["a"] * 4 + [None],+ "r2": ["b", "b", "b", None],+ "r3": ["c", "c", None],+ "r4": ["d", None],+ },+ idle = 12,+ widths = [4, 2, 2],+ order = ["r1", "r2", "r3", "r4"],+ ),+ id = "everything held behind one incompatible command still batches, within the width",+ ),+ pytest.param(+ dict(+ cmds = [("r1", 2), ("r2", 2), ("r3", 2)],+ script = {"r1": ["a"] * 4 + [None], "r2": ["b", "b", None], "r3": ["c", None]},+ refuse = ("r2", 0),+ idle = 12,+ refusals = (0, "r2"),+ order = ["r1", "r2", "r3"],+ ),+ id = "a command the batch turned away is not offered to it again",+ ),+ pytest.param(+ dict(+ cmds = [("r1", 4), ("r0", 2), ("r2", 2), ("r3", 2)],+ script = {+ "r1": ["a"] * 5 + [None],+ "r0": ["x", "x", "x", None],+ "r2": ["b", None],+ "r3": ["c", None],+ },+ refuse = ("r2", 1),+ idle = 16,+ refusals = (1, "r2"),+ order = ["r1", "r0", "r2", "r3"],+ ),+ id = "a head the batch turns away keeps its place over what waited behind it",+ ),+]++[email protected]("case", _HOLD_CASES)+def test_the_hold_releases_its_head_and_keeps_its_order(monkeypatch, case):+ """The hold's whole contract, over the traffic that breaks each part of it."""+ backend = _Backend()+ backend.script = {(name, None): events for name, events in case["script"].items()}+ if "refuse" in case:+ who, when = case["refuse"]+ backend.on_open = lambda session: (+ session.will_not_take.add(who) if len(backend.sessions) == when else None+ )++ resp = _run_loop(+ monkeypatch,+ backend,+ [_cmd(name, parallel_slots = width) for name, width in case["cmds"]]+ + [IDLE] * case.get("idle", 0),+ )++ admitted = [handle[0] for session in backend.sessions for handle, _r in session.admitted]+ assert admitted == case["order"], f"the hold lost or reordered a command: {admitted}"+ if "widths" in case:+ assert [session.opened_at for session in backend.sessions] == case["widths"]+ if "refusals" in case:+ index, name = case["refusals"]+ assert backend.sessions[index].refusals == [(name, None)]+ if case.get("idle"):+ terminal = [+ m["request_id"] for m in resp.sent if m.get("type") in ("gen_done", "gen_error")+ ]+ assert sorted(terminal) == sorted(case["order"]), f"a held reply was lost: {resp.sent}"+++class _DecliningBackend:+ def __init__(self, reason):+ self._reason, self.batched = reason, 0+ self.last_generation_stats = {"usage": {"completion_tokens": 2}}++ def batch_unavailable_reason(self, requests):+ return None if len(requests) < 2 else self._reason++ def generate_chat_response(+ self,+ seed = None,+ **kwargs,+ ):+ yield f"seed {seed}"+ yield f"seed {seed} done"++ def generate_chat_batch(self, requests, **kwargs):+ self.batched += 1+ raise AssertionError("a declined request set must not reach the batch")+++def _run(+ reason,+ cancelled = False,+ backend = None,+):+ from core.inference import worker++ sent, cancel = [], threading.Event()+ backend = backend or _DecliningBackend(reason)+ if cancelled:+ cancel.set()+ worker._handle_generate_rows(+ backend,+ {+ "request_id": "r1",+ "messages": [{"role": "user", "content": "hi"}],+ "rows": [{"seed": 1}, {"seed": 2}],+ "parallel_slots": 2,+ },+ SimpleNamespace(put = sent.append),+ cancel,+ )+ return backend, sent+++def test_a_declined_request_set_is_decoded_one_reply_at_a_time():+ """Reporting the command done without decoding would answer a caller's two."""+ backend, sent = _run("a reply asks for stop sequences")+ kinds = [event["type"] for event in sent if event["type"] != "batch_state"]+ tokens = [event for event in sent if event["type"] == "token"]++ assert backend.batched == 0+ assert kinds == ["token", "token", "row_done", "token", "token", "row_done", "gen_done"]+ assert [(event["row"], event["text"]) for event in tokens] == [+ (0, "seed 1"),+ (0, "seed 1 done"),+ (1, "seed 2"),+ (1, "seed 2 done"),+ ], "each row decodes with the request it was given, not with row zero's"+ assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]+ assert all(event["request_id"] == "r1" for event in sent)+ assert all(+ event["stats"] == backend.last_generation_stats+ for event in sent+ if event["type"] == "row_done"+ )+++def test_a_cancelled_fallback_still_reports_every_row_done():+ """A row nobody decoded is still a row somebody is waiting on."""+ _backend, sent = _run("a reply asks for stop sequences", cancelled = True)++ assert [event["type"] for event in sent if event["type"] != "batch_state"] == [+ "row_done",+ "row_done",+ "gen_done",+ ]+ assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]+ assert all(event["stats"] is None for event in sent if event["type"] == "row_done")+++def test_a_batch_that_will_not_open_still_answers_the_reply(monkeypatch):+ """Building the batch is what fails, and the reply predates any batch existing."""++ def refuse(**_kwargs):+ raise RuntimeError("BatchGenerator.insert missing")++ backend = _Backend()+ backend.open_resident_batch = refuse+ resp = _run_loop(monkeypatch, backend, [_cmd(parallel_slots = 2), IDLE])++ assert [m["type"] for m in resp.sent if m["type"] != "status"] == ["token", "gen_done"]+ assert len(backend.apart) == 1, "it was decoded one reply at a time"+++def test_a_row_override_the_backend_cannot_take_is_dropped_not_raised():+ """The route varies the seed per row, and the fallback decodes on backends without one."""+ InferenceBackend = pytest.importorskip("core.inference.inference").InferenceBackend++ class _NoSeedBackend(_DecliningBackend):+ """Transformers-shaped: wraps() carries the real signature, which declares no seed."""++ @functools.wraps(InferenceBackend.generate_chat_response)+ def generate_chat_response(self, *args, **kwargs):+ inspect.signature(InferenceBackend.generate_chat_response).bind(self, *args, **kwargs)+ yield "reply"++ _backend, sent = _run(None, backend = _NoSeedBackend("a reply asks for stop sequences"))+ kinds = [event["type"] for event in sent if event["type"] != "batch_state"]++ assert kinds == ["token", "row_done", "token", "row_done", "gen_done"], f"{sent}"+++def test_a_count_is_refused_while_the_batch_decodes(monkeypatch):+ """Dispatched generations hold no orchestrator lock, so the loop keeps its promise."""+ backend = _Backend(script = {("r1", None): ["a", None]})+ backend.count_chat_tokens, backend.active_model_name = (lambda *a, **k: 7), "m"+ count = {"type": "count_tokens", "request_id": "c1", "messages": []}+ cmds = [_cmd("r1", parallel_slots = 2), count, IDLE, IDLE, IDLE, count, IDLE]++ sent = _run_loop(monkeypatch, backend, cmds).sent+ counts = [m for m in sent if m["type"] == "count_tokens_response"]++ assert [m.get("error") for m in counts] == [+ "A generation is in progress.",+ None,+ ], f"the count was served mid-decode, or never ran once the batch drained: {counts}"+ assert counts[1]["input_tokens"] == 7diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 17b25dc70c4..4d7aa9386e6 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -7410,7 +7410,7 @@ def test_saved_parallel_slots_reach_an_api_load(monkeypatch):assert rec.calls[0].n_parallel == 8-def test_parallel_slots_are_stored_and_gated_on_gguf():+def test_parallel_slots_are_stored_and_reach_either_backend():override = settings.normalize_model_override({"n_parallel": 8})assert override == {"n_parallel": 8}# Blank, out of range and non-integer all mean "follow the server-wide default".@@ -7419,9 +7419,13 @@ def test_parallel_slots_are_stored_and_gated_on_gguf():gguf = settings.model_override_load_kwargs(override, is_gguf = True)assert gguf["n_parallel"] == 8- # A safetensors load has no llama-server slots, exactly as the picker gates it.- assert "n_parallel" not in settings.model_override_load_kwargs(override, is_gguf = False)+ safetensors = settings.model_override_load_kwargs(override, is_gguf = False)+ assert safetensors["n_parallel"] == 8+ for flag in ("n_batch", "n_ubatch"):+ stored = settings.normalize_model_override({flag: 512})+ assert flag not in settings.model_override_load_kwargs(stored, is_gguf = False)LoadRequest(model_path = "unsloth/B-GGUF", **gguf)+ LoadRequest(model_path = "unsloth/B", **safetensors)def test_override_route_persists_parallel_slots(override_store):@@ -10047,11 +10051,13 @@ def test_auto_switch_loads_an_unloaded_mlx_model(monkeypatch):"unsloth/Qwen3-MLX", "/srv/models/Qwen3-MLX", (), is_gguf = False)monkeypatch.setattr(resolver, "_scan", (time.monotonic(), {"unsloth/qwen3-mlx": entry}))+ settings.set_model_override("unsloth/Qwen3-MLX", n_parallel = 8)_run_hook("unsloth/Qwen3-MLX")assert [c.model_path for c in calls] == ["/srv/models/Qwen3-MLX"]assert calls[0].gguf_variant is None+ assert calls[0].n_parallel == 8# The alias lands on the orchestrator, leaving the llama.cpp backend untouched.assert orchestrator._openai_advertised_id == "unsloth/Qwen3-MLX"assert getattr(llama, "_openai_advertised_id", None) is Nonediff --git a/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py b/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.pyindex 37d986a416d..a78cf864a45 100644--- a/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py+++ b/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py@@ -17,8 +17,11 @@ from core.inference.orchestrator import InferenceOrchestratordef _idle_orchestrator(models, loading = ()):"""An orchestrator whose unload round-trip is stubbed to succeed."""o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._stop_ledger = None+ o._pending_teardowns = Noneo._gen_lock = threading.Lock()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)o._drain_event = threading.Event()o._unload_pending = Falseo.models = dict(models)@@ -29,7 +32,7 @@ def _idle_orchestrator(models, loading = ()):o.cancel_load = lambda _name: Falseo._ensure_subprocess_alive = lambda: Trueo._cancel_generation = lambda: None- o._wait_dispatcher_idle = lambda: True+ o._wait_worker_idle = lambda timeout = None: Trueo._drain_queue = lambda: Noneo._send_cmd = lambda _cmd: Noneo._wait_response = lambda _token: Nonediff --git a/studio/backend/tests/test_orchestrator_load_cancel_event.py b/studio/backend/tests/test_orchestrator_load_cancel_event.pyindex d6f9766a885..2b14c8a8a28 100644--- a/studio/backend/tests/test_orchestrator_load_cancel_event.py+++ b/studio/backend/tests/test_orchestrator_load_cancel_event.py@@ -10,6 +10,8 @@ from core.inference.orchestrator import InferenceOrchestratordef test_safetensors_load_event_interrupts_worker_wait(monkeypatch):orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orchestrator._stop_ledger = None+ orchestrator._pending_teardowns = Noneorchestrator.__dict__.update(loading_models = set(), active_model_name = None, models = {}, _proc = None, _resp_queue = object())diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.pyindex 491c998a45d..9b90f1039bb 100644--- a/studio/backend/tests/test_orchestrator_unload_cancel.py+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py@@ -6,6 +6,7 @@ hanging the UI. ``unload_model`` now cancels first (the mp.Event the worker checeach token) and takes ``_gen_lock`` before the unload round-trip."""+import inspectimport queueimport threadingimport time@@ -27,9 +28,26 @@ class _Unsloth:return None # no Unsloth load in flight -> Unsloth fast path skipped-def _bare_orchestrator():+class _RecordOfStops:+ """Stands in for the ledger; only whether the worker reads it matters here."""++ def __init__(self, read_by_worker: bool):+ self.stopped: list = []+ self._read_by_worker = read_by_worker++ def stop(self, request_id):+ self.stopped.append(request_id)+ return True++ def read_by_worker(self) -> bool:+ return self._read_by_worker+++def _bare_orchestrator(reads_stops = False):"""An orchestrator without the real __init__ subprocess/network."""o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._stop_ledger = _RecordOfStops(True) if reads_stops else None+ o._pending_teardowns = Noneo._gen_lock = threading.Lock()o._send_order_lock = threading.Lock()o._subprocess_shutdown_lock = threading.Lock()@@ -44,12 +62,13 @@ def _bare_orchestrator():o._dispatcher_thread = Noneo._dispatcher_stop = threading.Event()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)+ o._unload_pending = False+ o._worker_reserved_for = Noneo._mailbox_lock = threading.Lock()o._mailboxes = {}o._direct_mailboxes = {}o._request_cancel_events = {}- o._unload_pending = False- o._exclusive_tts_pending = Falseo.active_model_name = "m"o.models = {"m": {}}o.loading_models = set()@@ -117,7 +136,9 @@ def test_worker_closes_cancelled_generator_before_gen_done():cancel,)- assert [item["type"] for item in responses.items] == ["gen_done"]+ assert [item["type"] for item in responses.items if item["type"] != "batch_state"] == [+ "gen_done"+ ]def test_unload_cancels_inflight_generation_then_unloads(monkeypatch):@@ -176,7 +197,13 @@ def test_unload_falls_back_to_shutdown_when_generation_wont_yield(monkeypatch):monkeypatch.setattr(orch_mod, "_UNLOAD_GEN_LOCK_TIMEOUT", 0.2)shutdown = []monkeypatch.setattr(o, "_shutdown_subprocess", lambda timeout = 5: shutdown.append(timeout))- monkeypatch.setattr(o, "_send_cmd", lambda cmd: pytest.fail("must not send unload when wedged"))+ monkeypatch.setattr(+ o,+ "_send_cmd",+ lambda cmd: pytest.fail("must not send unload when wedged")+ if cmd.get("type") == "unload"+ else None,+ )# A wedged worker never releases _gen_lock, even after the cancel.o._gen_lock.acquire()@@ -210,7 +237,11 @@ def test_unload_tears_down_when_compare_dispatcher_wedged(monkeypatch):monkeypatch.setattr(o, "_shutdown_subprocess", lambda timeout = 5: shutdown.append(timeout))monkeypatch.setattr(o, "_drain_queue", lambda: [])monkeypatch.setattr(- o, "_send_cmd", lambda cmd: pytest.fail("must not send unload with a wedged dispatcher")+ o,+ "_send_cmd",+ lambda cmd: pytest.fail("must not send unload with a wedged dispatcher")+ if cmd.get("type") == "unload"+ else None,)monkeypatch.setattr(o,@@ -743,7 +774,13 @@ def test_unload_clears_drain_event_even_on_wedged_teardown(monkeypatch):o = _bare_orchestrator()monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)monkeypatch.setattr(orch_mod, "_UNLOAD_GEN_LOCK_TIMEOUT", 0.2)- monkeypatch.setattr(o, "_send_cmd", lambda cmd: pytest.fail("must not send when wedged"))+ monkeypatch.setattr(+ o,+ "_send_cmd",+ lambda cmd: pytest.fail("must not send when wedged")+ if cmd.get("type") == "unload"+ else None,+ )# A wedged worker never releases _gen_lock; unload tears the subprocess down. The# real teardown nulls _drain_event, so emulate that so the finally exercises its guard.@@ -761,6 +798,20 @@ def test_unload_clears_drain_event_even_on_wedged_teardown(monkeypatch):# ----------------------------------------------------------------------------+def _announcing_lock(lock, reached):+ """The held lock, saying so when someone starts waiting for it."""++ class _Announcing:+ def __enter__(self):+ reached.set()+ return lock.__enter__()++ def __exit__(self, *exc):+ return lock.__exit__(*exc)++ return _Announcing()++def test_generation_rechecks_model_after_lock_wait(monkeypatch):# A request passes the pre-lock active-model check, then blocks on _gen_lock while# an unload clears/swaps the model. Even if _unload_pending was already reset (the@@ -773,13 +824,11 @@ def test_generation_rechecks_model_after_lock_wait(monkeypatch):)reached_lock = threading.Event()- # _wait_dispatcher_idle runs after the pre-lock check and before acquiring the lock;- # signalling here means the generator captured the model and is about to block.- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda: (reached_lock.set(), True)[1])-o.active_model_name = "m"o._unload_pending = False- o._gen_lock.acquire() # stand in for an in-flight unload holding the lock+ held = o._gen_lock+ held.acquire() # stand in for an in-flight unload holding the lock+ o._gen_lock = _announcing_lock(held, reached_lock)out: list = []@@ -791,7 +840,7 @@ def test_generation_rechecks_model_after_lock_wait(monkeypatch):assert reached_lock.wait(timeout = 5)# Unload finished: model swapped, pending already cleared. Release the lock.o.active_model_name = "other"- o._gen_lock.release()+ held.release()t.join(timeout = 5)assert out and any("unloaded" in chunk.lower() for chunk in out)@@ -805,11 +854,11 @@ def test_generation_rechecks_model_when_unloaded_to_none(monkeypatch):o, "_send_cmd", lambda cmd: pytest.fail("must not generate after the model was unloaded"))reached_lock = threading.Event()- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda: (reached_lock.set(), True)[1])-o.active_model_name = "m"o._unload_pending = False- o._gen_lock.acquire()+ held = o._gen_lock+ held.acquire()+ o._gen_lock = _announcing_lock(held, reached_lock)out: list = []t = threading.Thread(@@ -818,7 +867,7 @@ def test_generation_rechecks_model_when_unloaded_to_none(monkeypatch):t.start()assert reached_lock.wait(timeout = 5)o.active_model_name = None- o._gen_lock.release()+ held.release()t.join(timeout = 5)assert out and any("unloaded" in chunk.lower() for chunk in out)@@ -1487,13 +1536,11 @@ def test_stale_unload_does_not_hide_an_update_refusal():# ----------------------------------------------------------------------------# A dispatched (compare-mode) request that races an unload must not orphan its-# mailbox after _wait_dispatcher_idle stops the dispatcher.# ----------------------------------------------------------------------------def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypatch):# The request passes the pre-work _unload_pending check, then an unload sets- # _unload_pending and _wait_dispatcher_idle stops the dispatcher (mailboxes empty)# before this request registers its mailbox. The recheck under _mailbox_lock must# make it bail, or the worker's skipped-generate reply has nothing to route it and# the compare stream hangs on an orphaned mailbox.@@ -1506,7 +1553,6 @@ def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypamonkeypatch.setattr(o, "_start_dispatcher", lambda: None)# Flip the unload flag after the pre-work check (626) but before mailbox- # registration -- exactly the window _wait_dispatcher_idle exploits.def flip(*a, **k):o._unload_pending = Truereturn {"type": "generate", "request_id": "r1"}@@ -1568,8 +1614,7 @@ def test_dispatched_bails_when_model_swapped_before_mailbox_registration(monkeypdef test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(monkeypatch):# Same window, but the unload was a same-model reload so active_model_name is- # unchanged; the give-away is that the dispatcher was stopped. Registering a- # mailbox with no dispatcher to route the reply would hang the compare stream.+ monkeypatch.setattr(orch_mod, "_DISPATCH_IDLE_TIMEOUT", 0.05)o = _bare_orchestrator()o._mailbox_lock = threading.Lock()o._mailboxes = {}@@ -1590,7 +1635,7 @@ def test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(moout = list(o._generate_dispatched(messages = [{"role": "user", "content": "hi"}]))- assert any("unloaded" in chunk.lower() for chunk in out)+ assert any("nothing is routing replies" in chunk.lower() for chunk in out), outassert o._mailboxes == {}, "must not leave an orphaned mailbox"@@ -2243,9 +2288,9 @@ def test_dispatched_bail_stops_orphan_dispatcher_it_started(monkeypatch):def fake_start():started["v"] = Trueo._dispatcher_thread = _AliveDispatcher()- return True # _start_dispatcher returns True for the caller that spawned it+ return o._dispatcher_thread # what _start_dispatcher hands the caller that spawned it- def fake_stop():+ def fake_stop(thread = None):stopped["v"] = Trueo._dispatcher_thread = None@@ -2446,6 +2491,7 @@ def test_concurrent_start_dispatcher_spawns_exactly_one():o._dispatcher_thread = Noneo._dispatcher_stop = threading.Event()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)n = 32# A barrier aligns every thread on the check-then-spawn window: without the lifecycle@@ -2467,10 +2513,9 @@ def test_concurrent_start_dispatcher_spawns_exactly_one():t.join(timeout = 5)try:- # _start_dispatcher returns True only for the caller that actually spawned a thread.- # Exactly one caller may win; every other must observe the dispatcher alive and bail.- assert results.count(True) == 1, f"expected exactly one spawn, got {results.count(True)}"- assert results.count(False) == n - 1+ spawned = [result for result in results if result is not None]+ assert len(spawned) == 1, f"expected exactly one spawn, got {len(spawned)}"+ assert results.count(None) == n - 1# And exactly one live dispatcher thread exists -- no orphan racing resp_queue.live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()@@ -2512,11 +2557,12 @@ def test_start_dispatcher_refuses_while_unload_pending():o._dispatcher_thread = Noneo._dispatcher_stop = threading.Event()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)o._unload_pending = Truestarted = o._start_dispatcher()- assert started is False, "must not start a dispatcher while an unload is pending"+ assert started is None, "must not start a dispatcher while an unload is pending"assert o._dispatcher_thread is None, "no dispatcher thread may be created"live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()]assert live == [], "no dispatcher may exist to consume the unloaded reply"@@ -2534,11 +2580,12 @@ def test_start_dispatcher_resumes_after_unload_clears():o._dispatcher_thread = Noneo._dispatcher_stop = threading.Event()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)o._unload_pending = Falsetry:assert (- o._start_dispatcher() is True+ o._start_dispatcher() is o._dispatcher_thread is not None), "a fresh dispatcher must start once no unload is pending"assert o._dispatcher_thread is not None and o._dispatcher_thread.is_alive()finally:@@ -2564,6 +2611,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():o._request_cancel_events = {}o._dispatcher_stop = threading.Event()o._dispatcher_lifecycle_lock = threading.Lock()+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)o._unload_pending = Falsestart_queued = threading.Event() # release the stop's join once the start is queued behind it@@ -2584,7 +2632,6 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():def unload_side():# unload_model's sequence: set _unload_pending under the lifecycle lock, then stop- # the idle dispatcher (also under the lock, via _wait_dispatcher_idle).with o._dispatcher_lifecycle_lock:o._unload_pending = Trueo._stop_dispatcher()@@ -2611,7 +2658,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():u.join(timeout = 5)c.join(timeout = 5)- assert started_result.get("v") is False, "the queued start must refuse while unloading"+ assert started_result.get("v") is None, "the queued start must refuse while unloading"assert o._dispatcher_thread is None, "the stop cleared it and the queued start spawned nothing"live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()]assert live == [], "no fresh dispatcher may be left to consume the unloaded reply"@@ -2889,7 +2936,6 @@ def test_generation_stopped_while_queued_is_never_sent(monkeypatch):# gen_done without one) still held up its siblings.o = _bare_orchestrator()monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda *a, **k: None)monkeypatch.setattr(o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped"))@@ -2960,6 +3006,74 @@ def test_a_scoped_load_cancel_that_never_reports_back_releases_the_load():inf._scoped_load_cancel_tombstones.clear()+def test_a_cancelled_row_stream_shows_no_token_a_single_reply_would_have_dropped():+ o = _bare_orchestrator()+ cancel = threading.Event()+ cancel.set()+ events = [+ {"type": "token", "request_id": "r1", "row": 0, "text": "buffered"},+ {"type": "row_done", "request_id": "r1", "row": 0},+ {"type": "row_done", "request_id": "r1", "row": 1},+ {"type": "gen_done", "request_id": "r1"},+ ]++ seen = list(+ o._consume_token_stream(+ lambda timeout: events.pop(0) if events else None,+ lambda: None,+ crash_context = "generation",+ request_id = "r1",+ cancel_event = cancel,+ mark_started = False,+ rows = 2,+ )+ )++ assert seen == [(0, None), (1, None)]+++def _real_send_order_lock():+ """The send-order lock exactly as InferenceOrchestrator.__init__ makes it."""+ source = inspect.getsource(InferenceOrchestrator.__init__)+ line = next(one for one in source.splitlines() if "self._send_order_lock = " in one)+ return eval(line.split("=", 1)[1].strip(), {"threading": threading})+++def test_a_recovering_caller_resets_the_worker_without_deadlocking_on_itself():+ """The reset decides under the send-order lock and then sends under it."""+ from types import SimpleNamespace++ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)+ backend._send_order_lock = _real_send_order_lock()+ backend._mailbox_lock = threading.Lock()+ backend._request_cancel_events = {}+ backend._cancel_event = threading.Event()+ backend._proc = SimpleNamespace(is_alive = lambda: True)+ sent = []+ backend._send_cmd = sent.append+ backend._teardown_going_out = backend._teardown_not_sent = lambda: None+ backend._owns_worker = lambda event: True+ done = threading.Event()++ def reset():+ backend.reset_generation_state(threading.Event())+ done.set()++ worker = threading.Thread(target = reset, daemon = True)+ worker.start()+ worker.join(timeout = 5)++ assert done.is_set(), "the reset deadlocked on the lock it already held"+ assert sent == [{"type": "reset"}]+++def test_the_idle_wait_is_bounded_when_no_timeout_is_asked_for():+ default = (+ inspect.signature(InferenceOrchestrator._wait_worker_idle).parameters["timeout"].default+ )+ assert isinstance(default, float) and default > 0, default++def test_shutdown_cancels_loads_that_have_not_reached_the_backend():"""A /load between admission and the backend call holds nothing the backend'sshutdown flag can see: it can sit in the lifecycle gate or preflight for@@ -3382,6 +3496,9 @@ def test_a_shutdown_that_begins_during_the_spawn_reaps_the_new_worker():class _Ctx:Queue = staticmethod(lambda: mock.Mock())Event = staticmethod(lambda: mock.Mock())+ Lock = staticmethod(lambda: mock.Mock())+ Array = staticmethod(lambda *a, **k: mock.Mock())+ Value = staticmethod(lambda *a, **k: mock.Mock())@staticmethoddef Process(**kw):@@ -3859,3 +3976,16 @@ def test_both_generation_paths_forward_the_image_ordinal():assert "image_ordinal = image_ordinal" in inspect.getsource(method), (f"{method.__name__} accepts the ordinal but does not put it on the " "worker command")+++def test_a_count_during_a_dispatched_generation_is_refused_at_once():+ o = _bare_orchestrator(reads_stops = True)+ o._mailboxes["running"] = queue.Queue()+ sent = []+ o._send_cmd = sent.append++ with pytest.raises(RuntimeError, match = "generation is in progress"):+ o.count_chat_tokens([{"role": "user", "content": "hi"}])++ assert sent == [], "the count reached a worker that reads no command until the reply ends"+ assert o._gen_lock.acquire(blocking = False), "the refusal kept the generation lock"diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.pyindex bd56d419759..e2bae9045e3 100644--- a/studio/backend/tests/test_parallel_slots_per_load.py+++ b/studio/backend/tests/test_parallel_slots_per_load.py@@ -13,6 +13,7 @@ wiring behind the /load, /validate and /status echoes.from __future__ import annotations+import importlib.utilimport inspectimport reimport struct@@ -26,10 +27,10 @@ _BACKEND_DIR = str(Path(__file__).resolve().parent.parent)if _BACKEND_DIR not in sys.path:sys.path.insert(0, _BACKEND_DIR)-# Same external-dep stubs as the other llama_cpp unit tests.-_loggers_stub = _types.ModuleType("loggers")-_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)-sys.modules.setdefault("loggers", _loggers_stub)+if importlib.util.find_spec("loggers") is None:+ _loggers_stub = _types.ModuleType("loggers")+ _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)+ sys.modules.setdefault("loggers", _loggers_stub)_structlog_stub = _types.ModuleType("structlog")_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")@@ -346,7 +347,7 @@ def test_route_resolves_slots_once_before_dedupe_guard_and_load():assert active_intent < resolved_intent < resolved_dedupeassert resolved_dedupe < guard < load_call# Both immutable intents and the guard share the value; resolution runs once.- assert load_impl.count("n_parallel = _n_parallel") == 3+ assert load_impl.count("n_parallel = _n_parallel") == 4assert load_impl.count("_resolve_parallel_slots(request, fastapi_request)") == 1assert "fastapi_request.app.state" not in load_impl@@ -574,3 +575,44 @@ def test_training_guard_keeps_slots_for_an_unclassified_gguf(monkeypatch, tmp_paone = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = None)many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = None)assert many > one+++def test_the_unsloth_load_hands_its_requested_slots_to_the_runtime():+ """The resolved width reaches the non-GGUF load too, not only llama-server's."""+ load_impl = _load_impl_source()+ call = load_impl.index("asyncio.to_thread(\n backend.load_model,")+ kwargs = load_impl[call : load_impl.index("\n )", call)]+ assert "n_parallel = _n_parallel" in kwargs+ assert "request.n_parallel" not in load_impl+ assert "clamp_parallel_slots" not in load_impl+++def test_a_load_reply_carries_the_two_window_facts_and_the_width_that_selects_them():+ """The three window facts are read as a set, so a reply carrying some answers nothing."""+ from models.inference import LoadResponse, _InferenceRuntimeFields+ from routes.inference import _unsloth_serving_fields++ info = {+ "context_length_enforced": True,+ "context_unbounded_when_batched": True,+ "parallel_slots": 4,+ "can_batch": True,+ }++ def reply(**overrides):+ return LoadResponse(+ status = "loaded",+ model = "m",+ display_name = "m",+ inference = {},+ **_unsloth_serving_fields({**info, **overrides}),+ ).model_dump()++ body = reply()+ assert (+ body["context_length_enforced"],+ body["context_unbounded_when_batched"],+ body["parallel_slots"],+ ) == (True, True, 4)+ assert reply(can_batch = False, context_unbounded_when_batched = False)["parallel_slots"] == 1+ assert _InferenceRuntimeFields.model_fields["context_unbounded_when_batched"].default is Falsediff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.pyindex d48bb3bf078..fe5cb3a255e 100644--- a/studio/backend/tests/test_presence_penalty.py+++ b/studio/backend/tests/test_presence_penalty.py@@ -206,6 +206,10 @@ def test_orchestrator_cmd_carries_all_sampling_params():from core.inference.orchestrator import InferenceOrchestratoro = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o.models, o.active_model_name = {}, None+ o._stop_ledger = o._pending_teardowns = None+ o._stop_ledger = None+ o._pending_teardowns = Nonecmd = o._build_generate_cmd("req1",None,@@ -273,6 +277,8 @@ def test_orchestrator_cmd_carries_the_tool_protocol_flag():from core.inference.orchestrator import InferenceOrchestratoro = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o.models, o.active_model_name = {}, None+ o._stop_ledger = o._pending_teardowns = Nonebase = dict(messages = [{"role": "user", "content": "hi"}], tools = [])assert (o._build_generate_cmd("r", None, tool_protocol_active = True, **base)["tool_protocol_active"]@@ -303,7 +309,7 @@ def test_the_worker_gates_the_tool_protocol_flag_on_the_backend_signature():from core.inference import worker- src = inspect.getsource(worker._handle_generate)+ src = inspect.getsource(worker._generation_kwargs)gated = src[src.index("for gated in (") : src.index("for gated in (") + 200]assert '"tool_protocol_active"' in gated, "the flag must be gated on _backend_declares"@@ -379,6 +385,8 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_ifrom core.inference.worker import _handle_generateo = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o.models, o.active_model_name = {}, None+ o._stop_ledger = o._pending_teardowns = Noneassert "video_base64" not in o._build_generate_cmd("r", None, messages = [])cmd = o._build_generate_cmd("r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"@@ -438,12 +446,14 @@ def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():built = []o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o.models, o.active_model_name = {}, None+ o._stop_ledger = o._pending_teardowns = Noneo._gen_lock = threading.Lock()o._unload_pending = False- o._exclusive_tts_pending = False+ o._worker_reserved_for = Noneo.active_model_name = "m"o._ensure_subprocess_alive = lambda: True- o._wait_dispatcher_idle = lambda: None+ o._wait_worker_idle = lambda *a, **k: Trueo._start_dispatcher = lambda: Falsedef _build(*_args, **kwargs):diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.pyindex f4ee2590387..2902c8ac46c 100644--- a/studio/backend/tests/test_safetensors_capability_advertise.py+++ b/studio/backend/tests/test_safetensors_capability_advertise.py@@ -683,6 +683,7 @@ def test_orchestrator_mirrors_chat_template_info_into_models_dict():from core.inference.orchestrator import InferenceOrchestratororch = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orch._stop_ledger = Noneorch.models = {}orch.active_model_name = Noneorch.loading_models = set()@@ -738,6 +739,7 @@ def test_orchestrator_missing_chat_template_info_falls_back_to_all_false():from routes.inference import _detect_safetensors_featuresorch = InferenceOrchestrator.__new__(InferenceOrchestrator)+ orch._stop_ledger = Noneorch.models = {}orch.active_model_name = "unsloth/Qwen3-0.6B"diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 61302f63714..324cac6f9c7 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -82,6 +82,7 @@ class _ScriptedBackend:self._responder = responderself._stats = statsself.calls: list = []+ self.batch_calls: list = []self.reset_count = 0def generate_chat_response(@@ -104,6 +105,25 @@ class _ScriptedBackend:for snap in snapshots:yield snap+ def generate_chat_batch(+ self,+ rows,+ *,+ stats_holder = None,+ **kwargs,+ ):+ """Every choice's first reply in one command, as the real bridge does."""+ self.batch_calls.append({"rows": rows, "shared": kwargs})+ reported: list = []+ for index, row in enumerate(rows):+ holder: dict = {}+ for snapshot in self.generate_chat_response(stats_holder = holder, **{**kwargs, **row}):+ yield index, snapshot+ reported.append(holder.get("stats"))+ yield index, None+ if stats_holder is not None:+ stats_holder["stats"] = reported+def reset_generation_state(self, caller_cancel_event = None):self.reset_count += 1@@ -631,24 +651,6 @@ def test_what_this_backend_can_serve_reaches_it_rather_than_being_refused(monkeyassert body["choices"][0]["message"]["content"] == "hi"-def test_n_serves_one_full_generation_per_choice(monkeypatch):- """Each choice is its own sampling run, as on the llama-server path: the- backend is asked once per choice rather than one reply being copied, and the- prompt they share is not re-counted. Two runs may sample the same text; what- is guaranteed is that each was generated."""- turns = iter(["first", "second"])- backend = _ScriptedBackend(- lambda messages, tools: [next(turns)],- stats = {"usage": {"prompt_tokens": 7, "completion_tokens": 3}},- )- body = _json_body(_call(_request(n = 2), monkeypatch, backend, supports_tools = False))- assert [c["index"] for c in body["choices"]] == [0, 1]- assert [c["message"]["content"] for c in body["choices"]] == ["first", "second"]- assert len(backend.calls) == 2 # a generation per choice, not one reused- # The shared prompt is counted once; only generated tokens accumulate.- assert _totals(body) == {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13}--def _totals(body):return {k: body["usage"][k] for k in ("prompt_tokens", "completion_tokens", "total_tokens")}@@ -1432,6 +1434,49 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):assert backend.calls[0]["image"] is not None+def test_a_turn_asking_for_several_replies_sends_them_as_one_batch(monkeypatch):+ """One command carries every choice, each with its own seed."""+ from routes.inference import _choice_seed++ backend = _ScriptedBackend(_fixed("hi"))+ body = _json_body(_call(_request(stream = False, n = 3, seed = 11), monkeypatch, backend))++ assert len(body["choices"]) == 3+ assert len(backend.batch_calls) == 1, "the choices did not go out together"+ call = backend.batch_calls[0]+ effective = [row.get("seed", call["shared"].get("seed")) for row in call["rows"]]+ assert effective == [11, _choice_seed(11, 1), _choice_seed(11, 2)], effective+++class _StoppedAfterFirstRowBackend(_ScriptedBackend):+ """A backend that cannot batch: rows run apart and a Stop skips the rest."""++ def generate_chat_batch(+ self,+ rows,+ *,+ stats_holder = None,+ cancel_event = None,+ **kwargs,+ ):+ self.batch_calls.append({"rows": rows, "shared": kwargs})+ yield 0, "partial"+ cancel_event.set()+ yield 0, None+ for row in range(1, len(rows)):+ yield row, None+ if stats_holder is not None:+ stats_holder["stats"] = [{"completion_tokens": 1}] + [None] * (len(rows) - 1)+++def test_a_stop_during_the_first_choice_returns_no_empty_choices(monkeypatch):+ backend = _StoppedAfterFirstRowBackend(_fixed("unused"))+ body = _json_body(_call(_request(stream = False, n = 3), monkeypatch, backend))++ assert len(backend.batch_calls) == 1+ assert [c["message"]["content"] for c in body["choices"]] == ["partial"], body["choices"]++_RF_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}_RF_FORMAT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _RF_SCHEMA}}_MARKUP_DOC = '{"city":"<think>Paris</think>"}'diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.pyindex 8e0182793c4..29acb1072e4 100644--- a/studio/backend/tests/test_shutdown_preserves_live_worker.py+++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py@@ -43,8 +43,10 @@ class _FakeProc:def _bare_inference():o = InferenceOrchestrator.__new__(InferenceOrchestrator)+ o._stop_ledger = None+ o._pending_teardowns = Noneo._subprocess_shutdown_lock = threading.Lock()- o._stop_dispatcher = lambda: None+ o._stop_dispatcher = lambda thread = None: Noneo._cancel_generation = lambda: Noneo._drain_queue = lambda: []diff --git a/studio/backend/tests/test_stream_errors.py b/studio/backend/tests/test_stream_errors.pyindex d62c635c8ba..8d272a41d58 100644--- a/studio/backend/tests/test_stream_errors.py+++ b/studio/backend/tests/test_stream_errors.py@@ -219,7 +219,12 @@ class TestSurvivesTheRouteLayer:consume = orch.InferenceOrchestrator._consume_token_streamstream = consume(- SimpleNamespace(_proc = None, _resp_queue = None, _mark_worker_started = lambda _event: None),+ SimpleNamespace(+ _proc = None,+ _resp_queue = None,+ _mark_worker_started = lambda _event: None,+ _forget_request = lambda *_a: None,+ ),lambda _timeout: payload,lambda: None,crash_context = "test",diff --git a/studio/backend/tests/test_warm_window_review_fixes.py b/studio/backend/tests/test_warm_window_review_fixes.pyindex c94276ef150..b2246772caf 100644--- a/studio/backend/tests/test_warm_window_review_fixes.py+++ b/studio/backend/tests/test_warm_window_review_fixes.py@@ -2227,6 +2227,7 @@ def test_an_offline_read_leaves_the_fetch_available():started = []backend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)+ backend._stop_ledger = Nonebackend._top_models_started = Falsewith mock.patch.object(orch, "hf_env_offline", lambda: True):backend._start_top_models_fetch()@@ -2250,6 +2251,7 @@ def test_a_slow_refresh_cannot_overwrite_a_newer_default_list():import utils.hardware.hardware as hw_modbackend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)+ backend._stop_ledger = Nonebackend._static_models_lock = threading.Lock()backend._static_models = ["gen0"]backend._static_models_generation = 0@@ -2282,6 +2284,7 @@ def test_a_refresh_whose_generation_moved_mid_build_does_not_commit():import utils.hardware.hardware as hw_modbackend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)+ backend._stop_ledger = Nonebackend._static_models_lock = threading.Lock()backend._static_models = ["old"]backend._static_models_generation = 1@@ -2310,6 +2313,7 @@ def test_an_ordinary_refresh_still_happens():import utils.hardware.hardware as hw_modbackend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)+ backend._stop_ledger = Nonebackend._static_models_lock = threading.Lock()backend._static_models = ["old"]backend._static_models_generation = 1diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.pyindex fc1238469fc..7aabfdd22af 100644--- a/studio/backend/utils/openai_auto_switch_settings.py+++ b/studio/backend/utils/openai_auto_switch_settings.py@@ -582,6 +582,7 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di("llama_extra_args", "llama_extra_args"),("kv_cache_dtype", "cache_type_kv"),("mlx_kv_bits", "mlx_kv_bits"),+ ("n_parallel", "n_parallel"),("speculative_type", "speculative_type"),("spec_draft_n_max", "spec_draft_n_max"),("reasoning_budget", "reasoning_budget"),@@ -594,8 +595,6 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dikwargs[target] = override[source]if is_gguf:- if override.get("n_parallel") is not None:- kwargs["n_parallel"] = override["n_parallel"]if override.get("n_batch") is not None:kwargs["n_batch"] = override["n_batch"]if override.get("n_ubatch") is not None:diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 412ed360759..4b572ef98df 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2340,6 +2340,9 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["activeGgufVariant","activeModelIsLocal","loadedContextLength",+ "loadedContextEnforced",+ "loadedContextUnboundedWhenBatched",+ "loadedParallelSlots","maxContextLength","nativeContextLength","loadedIsGguf",@@ -3173,6 +3176,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{// A DSpark sidecar is ~11 GB, and Auto reaches it.speculative_type?: string | null;spec_draft_n_max?: number | null;+ n_parallel?: number | null;}): Promise<boolean> {// Before the POST, so an abort costs nothing: no request was sent.options?.abortSignal?.throwIfAborted();@@ -3387,6 +3391,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{cache_type_kv: config.kvCacheDtype,tensor_parallel: effectiveTensorParallel,disable_vision: effectiveDisableVision,+ n_parallel: config.nParallel ?? null,speculative_type: effectiveSpeculativeType,spec_draft_n_max: effectiveSpecDraftNMax,...(candidate.kind === "gguf"@@ -3395,7 +3400,6 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{gpu_memory_mode: effectiveGpuMemoryMode,// A remembered manual DiffusionGemma split (0 especially) must not be refused as a full-GGUF occupant.gpu_layers: effectiveGpuLayers,- n_parallel: config.nParallel ?? null,reasoning_budget: isDiffusion ? -1 : config.reasoningBudget,reasoning_budget_message: isDiffusion? ""@@ -3456,6 +3460,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{: "",tensor_parallel: effectiveTensorParallel,disable_vision: effectiveDisableVision,+ n_parallel: config.nParallel ?? null,// GGUF-only; the split ratio is never remembered (it is bound to an exact GPU set), so// llama.cpp's free-VRAM default stays in charge....(candidate.kind === "gguf"@@ -3464,8 +3469,6 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{gpu_layers: effectiveGpuLayers,n_cpu_moe: effectiveNCpuMoe,gpu_ids: effectiveGpuIds ?? undefined,- // Per-model too, or the auto-load reverts a remembered override.- n_parallel: config.nParallel ?? null,...(config.nBatch != null ? { n_batch: config.nBatch } : {}),...(config.nUbatch != null ? { n_ubatch: config.nUbatch } : {}),// Remembered like the rest of this block, or the auto-load reverts the override.@@ -3545,13 +3548,12 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{display_name: loadResp.display_name ?? candidate.id,is_gguf: loadResp.is_gguf ?? candidate.kind === "gguf",});+ const committedSlots =+ (loadResp.is_diffusion ?? false) ? null : (config.nParallel ?? null);if (candidate.kind === "gguf") {// The saved Context Length, not fitMaxSeqLength: the wire value is Auto-resolved on a// same-model reload, so pinning it turns Auto into a number the user never set.const keepCustomCtx = resolveExplicitCtxPin(config.customContextLength);- // Diffusion ignores --parallel, so counting slots there would mint a phantom override.- const committedSlots =- (loadResp.is_diffusion ?? false) ? null : (config.nParallel ?? null);// same rule for the batch sizesconst committedNBatch =(loadResp.is_diffusion ?? false) ? null : (config.nBatch ?? null);@@ -3643,9 +3645,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{kvCacheDtype: loadResp.cache_type_kv ?? null,loadedKvCacheDtype: loadResp.cache_type_kv ?? null,...mlxRuntimeStateFrom(loadResp),- // GGUF-only and never sent here: a staged override would be saved for a model that cannot use it.- nParallel: null,- loadedNParallel: null,+ nParallel: committedSlots,+ loadedNParallel: committedSlots,reasoningBudget: -1,loadedReasoningBudget: -1,loadedReasoningBudgetRequested: -1,diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsxindex 70d49b7afc3..f2d2d9ec8e5 100644--- a/studio/frontend/src/features/chat/chat-page.tsx+++ b/studio/frontend/src/features/chat/chat-page.tsx@@ -2303,6 +2303,12 @@ export function ChatPage({);const contextUsage = useChatRuntimeStore((state) => state.contextUsage);const loadedIsGguf = useChatRuntimeStore((state) => state.loadedIsGguf);+ const loadedContextUnboundedWhenBatched = useChatRuntimeStore(+ (state) => state.loadedContextUnboundedWhenBatched,+ );+ const loadedParallelSlots = useChatRuntimeStore(+ (state) => state.loadedParallelSlots,+ );const loadedContextBudget = useChatRuntimeStore((state) => state.loadedContextBudget,);@@ -4259,6 +4265,8 @@ export function ChatPage({platformChatOnlyReason,)}contextEnforced={loadedContextEnforced}+ contextUnboundedWhenBatched={loadedContextUnboundedWhenBatched}+ parallelSlots={loadedParallelSlots}contextBudget={loadedContextBudget}className="h-[var(--studio-chat-control-height,34px)]"/>diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex 8fd491d0fa4..2dfe699584e 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -1439,8 +1439,6 @@ export function useChatModelRuntime() {useChatRuntimeStore.getState().params.checkpoint;const pendingConfig =typeof selection !== "string" ? selection.config : undefined;- // nativePathToken is excluded: a leased file is named by a label two files can share, and only a- // completed load writes the lease, so adopting would keep a stale token.if (!forceReload && !nativePathToken) {const residentStatus = await getInferenceStatus().catch(() => null);// Warm before reconciling the remembered GPU pick below: load-on-selection can run before any@@ -1504,8 +1502,6 @@ export function useChatModelRuntime() {// no saved record the caller has already run applyModelLoadConfigToRuntime(null), which resets the// store to DEFAULT_PER_MODEL_CONFIG, and performLoad reads the store for the rest.residentRuntimeMatchesConfig(status, comparedConfig, {- // What the applier fills an unset field with, so the comparison is against what /load would send- // rather than against silence.speculativeType: readPersistedSpeculativeType(),gpuMemoryMode: readPersistedGpuMemoryMode(),gpuLayers: GPU_LAYERS_AUTO,@@ -2476,8 +2472,7 @@ export function useChatModelRuntime() {mlx_kv_bits: loadMlxKvBits ?? null,speculative_type: loadSpeculativeType,spec_draft_n_max: loadSpecDraftNMax,- // GGUF-only: slots mean nothing for a transformers load.- n_parallel: isGguf ? loadNParallel : null,+ n_parallel: loadNParallel,reasoning_budget:isGguf && !targetIsDiffusion ? loadReasoningBudget : -1,reasoning_budget_message:@@ -2589,11 +2584,9 @@ export function useChatModelRuntime() {const loadedSpec = normalizeSpeculativeType(loadResponse.speculative_type,);- // Slots the load actually committed: non-GGUF never sends them and diffusion ignores --parallel,- // so a click-time count would mint a phantom override.const committedSlots =- (loadResponse.is_gguf ?? false) &&- !(loadResponse.is_diffusion ?? false)+ ((loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)) ||+ (loadResponse.is_mlx ?? false)? (loadNParallel ?? null): null;// same rule for the batch sizes: gguf-only llama-server flagsdiff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex dbfaf636daf..c528c2a6ca9 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -253,10 +253,9 @@ export function applyActiveModelStatusToStore(: hydratingExistingModel)? resolveResidentInitialConfig(checkpointId, status.gguf_variant ?? null): null;- const rememberedNParallel =- status.is_gguf && remembered?.remembered- ? (remembered.config.nParallel ?? null)- : null;+ const rememberedNParallel = remembered?.remembered+ ? (remembered.config.nParallel ?? null)+ : null;const rememberedNBatch =status.is_gguf && remembered?.remembered? (remembered.config.nBatch ?? null)@@ -568,18 +567,14 @@ export function applyActiveModelStatusToStore(mlxKvQuantNote: status.mlx_kv_quant_note ?? null,}),// Baseline only, never the control: the echo is the RESOLVED count and would pin a blank- // "server default" control. The rollback re-sends the baseline, so without this a rollback- // after a tab reload loses the override. Refresh on every echo: another client- // can reload the same model with a different count....(seedLoadParams &&status.requested_parallel_slots != null && {loadedNParallel: status.requested_parallel_slots,}),- // A slotless model must not keep the previous GGUF's baseline, since the rollback re-sends- // it. /status omits the echo for non-GGUF and nulls it for diffusion, so an absent field- // on a GGUF means an older backend.+ // A slotless load must not keep the previous model's baseline, since the rollback+ // re-sends it. /status nulls the echo for a load that decodes one reply at a time....(seedLoadParams &&- (status.is_gguf === false || status.requested_parallel_slots === null) && {+ status.requested_parallel_slots === null && {loadedNParallel: null,}),// Per-model: a change underneath this tab blanks the control like performLoad's cross-modeldiff --git a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts b/studio/frontend/src/features/chat/lib/context-usage-bar-state.tsindex 08bc92a68e8..c4368950e64 100644--- a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts+++ b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts@@ -25,6 +25,8 @@ export type ContextUsageBarInput = {// MLX keeps generating past the window instead of stopping there, so it needs the// opposite advice from llama.cpp once a conversation outgrows the limit.isMlx?: boolean;+ contextUnboundedWhenBatched?: boolean;+ parallelSlots?: number | null;/** context_length_enforced as the load reported it; null where it does not answer. */contextEnforced?: boolean | null;contextBudget?: number | null;@@ -52,6 +54,8 @@ function contextLimitAdvice(total: number,isMlx: boolean | undefined,enforced: boolean | null | undefined,+ unboundedWhenBatched: boolean | undefined,+ slots: number | null | undefined,budget: number | null | undefined,): ContextLimitAdvice {if ((used / total) * 100 <= 85) return "none";@@ -61,7 +65,13 @@ function contextLimitAdvice(// nothing rotates and nothing stops, so neither of the other two is true of it. An// unjudged MLX window says the same thing operationally: the probe could not build a// cache, so none was bounded and it grows exactly as a confirmed false one does.- if (enforced === false || (isMlx && enforced == null)) return "unenforced-limit";+ if (+ enforced === false ||+ (isMlx && enforced == null) ||+ (unboundedWhenBatched && (slots ?? 1) > 1)+ ) {+ return "unenforced-limit";+ }if (!isMlx) return "stops-at-limit";return used > total ? "mlx-past-limit" : "mlx-near-limit";}@@ -88,6 +98,8 @@ export function deriveContextUsageBar({completionTokens,isMlx,contextEnforced,+ contextUnboundedWhenBatched,+ parallelSlots,contextBudget,}: ContextUsageBarInput): ContextUsageBarState | null {const limit = typeof total === "number" && total > 0 ? total : null;@@ -138,6 +150,8 @@ export function deriveContextUsageBar({limit,isMlx,contextEnforced,+ contextUnboundedWhenBatched,+ parallelSlots,contextBudget,),};diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.tsindex 81898ea1cce..eecb3bc01b2 100644--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts@@ -112,9 +112,6 @@ export type StandingConfigDefaults = {type SettingCheck = {/** Placement, which the backend rewrites wholesale on a preserved CPU fallback. */placement?: true;- /** One of the two fields `_mlx_runtime_settings_match` compares. The non-GGUF branch of /load- * checks identity and those, then answers already_loaded, so nothing else here may decide- * against a safetensors or MLX resident. */mlxComparable?: true;/** Placement the diffusion branch of `_runtime_matches_intent` replaces wholesale with one* `_diffusion_manual_ngl` comparison. */@@ -244,10 +241,6 @@ const requestedGpuMemoryMode = (const cleanTemplate = (value: string | null | undefined): string | null =>value?.trim() ? value : null;-/** Mirrors the fields `_runtime_matches_intent` reloads for, plus the MLX pair- * `_mlx_runtime_settings_match` compares. Nearly every check is unconditionally pinned: a- * config reaches here only after `applyModelLoadConfigToRuntime` resolved each field with `??- * null`, so an unset field asks for the default. Only `llamaExtraArgs` is optional. */const SETTING_CHECKS: SettingCheck[] = [{// Resolved, not compared raw: an unset length is Auto, which the load sends as 0 for a cross-model@@ -293,8 +286,7 @@ const SETTING_CHECKS: SettingCheck[] = [},{chatOnly: true,- // Unknown default: null against the status's resolved count is a reload, the safe direction.- // defaultParallelSlots is the EFFECTIVE count, so a build that clamps to one slot reloads.+ mlxComparable: true,pinned: () => true,agrees: (c, s, standing) =>(c.nParallel ?? standing.parallelSlots) ===@@ -591,8 +583,6 @@ export function residentRuntimeMatchesConfig(const diffusion = status.is_diffusion === true;return SETTING_CHECKS.every((check) =>- // The non-GGUF branch of /load checks identity and the MLX pair, then answers already_loaded, so- // no llama.cpp invocation field may decide against one.(status.is_gguf === false && !check.mlxComparable) ||(diffusion && check.chatOnly) ||(diffusion && check.ggufPlacement) ||@@ -602,3 +592,4 @@ export function residentRuntimeMatchesConfig(check.agrees(config, status, standing),);}+diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsxindex 8f124f4787f..d672d22ee02 100644--- a/studio/frontend/src/features/chat/shared-composer.tsx+++ b/studio/frontend/src/features/chat/shared-composer.tsx@@ -1675,6 +1675,7 @@ export function SharedComposer({: "",tensor_parallel: effectiveTensorParallel,disable_vision: effectiveDisableVision,+ n_parallel: ownConfig.nParallel ?? null,force_cancel_active:compareStopDecision?.forceCancelActive ?? false,...(targetIsGguf@@ -1684,9 +1685,6 @@ export function SharedComposer({n_cpu_moe: effectiveNCpuMoe,tensor_split: compareLoadKnobs.splitRatio ?? undefined,gpu_ids: effectiveSelectedGpuIds ?? undefined,- n_parallel: ownConfig.nParallel ?? null,- // Only when this panel has read the stored value: omitted, the load inherits it, which keeps- // CLI-set flags working....(ownConfig.llamaExtraArgs !== undefined? // biome-ignore lint/style/useNamingConvention: API schema{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }@@ -1747,7 +1745,7 @@ export function SharedComposer({// Slots this compare load committed. Diffusion ignores --parallel, so a// count there would mint a phantom override a preset carries onto a GGUF.const committedSlots =- targetIsGguf && !(resp.is_diffusion ?? false)+ ((resp.is_gguf ?? false) && !(resp.is_diffusion ?? false)) || (resp.is_mlx ?? false)? (ownConfig.nParallel ?? null): null;// same rule for the batch sizesdiff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 4848a8c4248..9dbaaf27440 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -2292,6 +2292,8 @@ type ChatRuntimeStore = {/** Whether loadedContextLength actually bounds the cache. Null where the backend does* not answer, which is not the same as a confirmed false. */loadedContextEnforced: boolean | null;+ loadedContextUnboundedWhenBatched: boolean;+ loadedParallelSlots: number | null;loadedContextBudget: number | null;modelRequiresTrustRemoteCode: boolean;supportsReasoning: boolean;@@ -2412,8 +2414,9 @@ type ChatRuntimeStore = {/** User --spec-draft-n-max override (null = platform default). */specDraftNMax: number | null;loadedSpecDraftNMax: number | null;- /** --parallel slots override for GGUF loads (null = server default). Never re-seeded from an- * echo: the resolved count would pin a blank control. */+ /** User reply-width override, for either backend (null = server default). GGUF spends it+ * on llama-server's --parallel slots; MLX on how many replies decode together.+ * Never re-seeded from an echo: the resolved count would pin a blank control. */nParallel: number | null;/** Slots the last successful load sent (null = default); a rollback re-sends them so a failed* switch cannot lose the override. */@@ -4103,6 +4106,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({loadedIsGguf: null,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextUnboundedWhenBatched: false,+ loadedParallelSlots: null,loadedContextBudget: null,modelRequiresTrustRemoteCode: false,supportsReasoning: false,diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex 54886c5a4e8..6927e7a5c3f 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -252,6 +252,7 @@ export interface LoadModelResponse {max_context_length?: number | null;native_context_length?: number | null;context_length_enforced?: boolean | null;+ context_unbounded_when_batched?: boolean;supports_reasoning?: boolean;reasoning_style?:| "enable_thinking"@@ -296,7 +297,6 @@ export interface LoadModelResponse {gpu_ids?: number[] | null;/** User-requested GPU placement pool before fit-time narrowing. */requested_gpu_ids?: number[] | null;- /** Slots the load was invoked with (else the --parallel default). Null for non-GGUF loads. */requested_parallel_slots?: number | null;reasoning_budget?: number;reasoning_budget_message?: string;@@ -305,7 +305,6 @@ export interface LoadModelResponse {requested_reasoning_budget?: number;// biome-ignore lint/style/useNamingConvention: API schemarequested_reasoning_budget_message?: string;- /** Slots llama-server actually runs, after any fit-time reduction. Null for non-GGUF loads. */parallel_slots?: number | null;/** batch size (--batch-size) the load was invoked with; null = default */requested_n_batch?: number | null;@@ -376,6 +375,7 @@ export interface InferenceStatusResponse {max_context_length?: number | null;native_context_length?: number | null;context_length_enforced?: boolean | null;+ context_unbounded_when_batched?: boolean;cache_type_kv?: string | null;mlx_kv_bits?: number | null;mlx_kv_bits_requested?: number | null;@@ -409,7 +409,6 @@ export interface InferenceStatusResponse {gpu_ids?: number[] | null;/** User-requested GPU placement pool before fit-time narrowing. */requested_gpu_ids?: number[] | null;- /** Slots the active load was invoked with (else the --parallel default). Null when no GGUF model is loaded. */requested_parallel_slots?: number | null;reasoning_budget?: number;reasoning_budget_message?: string;@@ -418,7 +417,6 @@ export interface InferenceStatusResponse {requested_reasoning_budget?: number;// biome-ignore lint/style/useNamingConvention: API schemarequested_reasoning_budget_message?: string;- /** Slots llama-server actually runs, after any fit-time reduction. Null when no GGUF model is loaded. */parallel_slots?: number | null;/** batch size (--batch-size) the active load was invoked with; null = default */requested_n_batch?: number | null;diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 8bec2615fee..f14861a1b54 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -1078,6 +1078,61 @@ function AdvancedSettingsToggle({);}+const GGUF_PARALLEL_HINT =+ "Decode slots (--parallel) for concurrent requests. Leave blank for the server " ++ "default. More slots share the context pool and use more VRAM.";++const MLX_PARALLEL_HINT =+ "Chat replies this model decodes at once (--parallel). Leave blank for the " ++ "default. Replies sharing a decode finish sooner together, but each one holds " ++ "its own context in memory for as long as it runs, and nothing reduces the " ++ "number to fit — raise it only if the memory is there.";++function ParallelSlotsRow({+ config,+ update,+ hint,+}: {+ config: PerModelConfig;+ update: (patch: Partial<PerModelConfig>) => void;+ hint: string;+}) {+ return (+ <div className={ROW_CLASS}>+ <div className="flex min-w-0 items-center gap-1.5">+ <span className={LABEL_CLASS}>Parallel Slots</span>+ <InfoHint>{hint}</InfoHint>+ </div>+ <input+ type="number"+ min={N_PARALLEL_MIN}+ max={N_PARALLEL_MAX}+ step={1}+ value={config.nParallel ?? ""}+ placeholder="auto"+ onChange={(event) => {+ const raw = event.target.value;+ if (raw === "") {+ update({ nParallel: null });+ return;+ }+ const parsed = Number.parseInt(raw, 10);+ if (Number.isFinite(parsed)) {+ update({+ nParallel: Math.max(+ N_PARALLEL_MIN,+ Math.min(N_PARALLEL_MAX, parsed),+ ),+ });+ }+ }}+ aria-label="Parallel decode slots"+ className={NUMBER_INPUT_CLASS}+ />+ </div>+ );+}+function MlxAdvancedSettings({config,update,@@ -1138,6 +1193,9 @@ function MlxAdvancedSettings({) : null}</div>)}+ {servedByMlx && (+ <ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />+ )}<div className="space-y-1"><ChatTemplateSettingconfig={config}@@ -1433,42 +1491,7 @@ function GgufAdvancedSettings({</div>)}- <div className={ROW_CLASS}>- <div className="flex min-w-0 items-center gap-1.5">- <span className={LABEL_CLASS}>Parallel Slots</span>- <InfoHint>- Decode slots (--parallel) for concurrent requests. Leave blank for- the server default. More slots share the context pool and use more- VRAM.- </InfoHint>- </div>- <input- type="number"- min={N_PARALLEL_MIN}- max={N_PARALLEL_MAX}- step={1}- value={config.nParallel ?? ""}- placeholder="auto"- onChange={(event) => {- const raw = event.target.value;- if (raw === "") {- update({ nParallel: null });- return;- }- const parsed = Number.parseInt(raw, 10);- if (Number.isFinite(parsed)) {- update({- nParallel: Math.max(- N_PARALLEL_MIN,- Math.min(N_PARALLEL_MAX, parsed),- ),- });- }- }}- aria-label="Parallel decode slots"- className={NUMBER_INPUT_CLASS}- />- </div>+ <ParallelSlotsRow config={config} update={update} hint={GGUF_PARALLEL_HINT} />{!isDiffusion && (<div className="space-y-1">diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.tsindex 0141ea6f327..8b2bbdbb9af 100644--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts@@ -205,6 +205,8 @@ export function loadedContextFields(resp: {native_context_length?: number | null;max_context_length?: number | null;context_length_enforced?: boolean | null;+ context_unbounded_when_batched?: boolean;+ parallel_slots?: number | null;mlx_context_budget?: number | null;} | null): {loadedContextLength: number | null;@@ -213,6 +215,8 @@ export function loadedContextFields(resp: {loadedIsGguf: boolean | null;loadedIsMlx: boolean | null;loadedContextEnforced: boolean | null;+ loadedContextUnboundedWhenBatched: boolean;+ loadedParallelSlots: number | null;loadedContextBudget: number | null;} {if (!resp) {@@ -223,6 +227,8 @@ export function loadedContextFields(resp: {loadedIsGguf: null,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextUnboundedWhenBatched: false,+ loadedParallelSlots: null,loadedContextBudget: null,};}@@ -237,6 +243,8 @@ export function loadedContextFields(resp: {loadedIsGguf: false,loadedIsMlx: resp.is_mlx ?? null,loadedContextEnforced: null,+ loadedContextUnboundedWhenBatched: false,+ loadedParallelSlots: null,loadedContextBudget: null,};}@@ -251,6 +259,11 @@ export function loadedContextFields(resp: {// llama.cpp allocates what it reports, so GGUF is enforced by construction.// Everything else answers for itself, or says nothing.loadedContextEnforced: isGguf ? true : (resp.context_length_enforced ?? null),+ // Read from the same response as the other two so the three never mix across loads.+ loadedContextUnboundedWhenBatched: isGguf+ ? false+ : (resp.context_unbounded_when_batched ?? false),+ loadedParallelSlots: resp.parallel_slots ?? null,loadedContextBudget: isGguf ? null : (resp.mlx_context_budget ?? null),};}diff --git a/studio/frontend/tests/helpers/store-stubs/auth.ts b/studio/frontend/tests/helpers/store-stubs/auth.tsindex c9dc028703f..f7f484b6035 100644--- a/studio/frontend/tests/helpers/store-stubs/auth.ts+++ b/studio/frontend/tests/helpers/store-stubs/auth.ts@@ -19,3 +19,7 @@ export function authFetch(input: string, init?: RequestInit): Promise<Response>if (!handler) throw new Error("authFetch: no network in tests");return Promise.resolve(handler(input, init));}++export async function prepareHfTokenForUse(): Promise<null> {+ return null;+}diff --git a/studio/frontend/tests/helpers/store-stubs/model-picker.ts b/studio/frontend/tests/helpers/store-stubs/model-picker.tsindex a53900604b4..722a737e1b3 100644--- a/studio/frontend/tests/helpers/store-stubs/model-picker.ts+++ b/studio/frontend/tests/helpers/store-stubs/model-picker.ts@@ -1,7 +1,6 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-// The picker barrel reaches the whole chat UI; the store only needs these three.export function applyPerModelConfigToRuntime(): void {}@@ -12,3 +11,25 @@ export function currentRuntimePerModelConfig(): Record<string, unknown> {export function perModelConfigsEqual(): boolean {return true;}++let residentConfig: unknown = null;++export function setResidentInitialConfig(value: unknown): void {+ residentConfig = value;+}++export function resolveResidentInitialConfig(): unknown {+ return residentConfig;+}++export function loadedContextFields(): Record<string, unknown> {+ return {};+}++export function savedContextPin(): null {+ return null;+}++export function adoptCachedRepoConfig(): null {+ return null;+}diff --git a/studio/frontend/tests/helpers/store-stubs/settings-http.ts b/studio/frontend/tests/helpers/store-stubs/settings-http.tsindex 3f248741a4b..f43a377d86f 100644--- a/studio/frontend/tests/helpers/store-stubs/settings-http.ts+++ b/studio/frontend/tests/helpers/store-stubs/settings-http.ts@@ -157,3 +157,7 @@ export async function authFetch(}),} as Response;}++export function getAuthSessionEpoch(): number {+ return 0;+}diff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.tsindex d0cd8bfd426..a4bdd7a59b5 100644--- a/studio/frontend/tests/mlx-context-helpers.test.ts+++ b/studio/frontend/tests/mlx-context-helpers.test.ts@@ -5,6 +5,7 @@// coverage regexes the source, which passes with the body deleted; these call them.import assert from "node:assert/strict";+import { readFileSync } from "node:fs";import test from "node:test";import {@@ -61,6 +62,8 @@ test("an MLX response carries a window without a native one", () => {loadedIsGguf: false,loadedIsMlx: true,loadedContextEnforced: null,+ loadedContextUnboundedWhenBatched: false,+ loadedParallelSlots: null,loadedContextBudget: null,});// Transformers, which sizes nothing, still contributes no window.@@ -71,6 +74,8 @@ test("an MLX response carries a window without a native one", () => {loadedIsGguf: false,loadedIsMlx: null,loadedContextEnforced: null,+ loadedContextUnboundedWhenBatched: false,+ loadedParallelSlots: null,loadedContextBudget: null,});assert.equal(loadedContextFields(null).loadedIsGguf, null);@@ -220,6 +225,12 @@ test("the usage bar names the three ways a window can end", () => {assert.equal(at(30000, { contextEnforced: undefined }), "unenforced-limit");// Only for MLX: nothing else installs a window whose bound could go unjudged.assert.equal(at(30000, { isMlx: false, contextEnforced: null }), "stops-at-limit");+ const batched = { contextUnboundedWhenBatched: true };+ assert.equal(at(30000, { ...batched, parallelSlots: 4 }), "unenforced-limit");+ assert.equal(at(40000, { ...batched, parallelSlots: 4 }), "unenforced-limit");+ assert.equal(at(30000, { ...batched, parallelSlots: 1 }), "mlx-near-limit");+ assert.equal(at(30000, { ...batched, parallelSlots: null }), "mlx-near-limit");+ assert.equal(at(30000, { parallelSlots: 4 }), "mlx-near-limit");});test("an outgoing self-sizing window does not become the next model's request", () => {@@ -313,3 +324,55 @@ test("a load keeps the pin it was built from, wherever the record held it", () =}),);});++test("a load response's two window facts and width reach the bar together", () => {+ const wide = loadedContextFields({+ is_mlx: true,+ context_length: 32768,+ native_context_length: 32768,+ context_length_enforced: true,+ context_unbounded_when_batched: true,+ parallel_slots: 4,+ });+ assert.equal(wide.loadedContextUnboundedWhenBatched, true);+ assert.equal(wide.loadedParallelSlots, 4);++ assert.equal(+ deriveContextUsageBar({+ used: 30000,+ total: 32768,+ isMlx: true,+ contextEnforced: wide.loadedContextEnforced,+ contextUnboundedWhenBatched: wide.loadedContextUnboundedWhenBatched,+ parallelSlots: wide.loadedParallelSlots,+ })?.advice,+ "unenforced-limit",+ );++ const gguf = loadedContextFields({+ is_gguf: true,+ context_length: 32768,+ native_context_length: 32768,+ context_unbounded_when_batched: true,+ parallel_slots: 4,+ });+ assert.equal(gguf.loadedContextUnboundedWhenBatched, false);+});++test("a background load cannot leave the visible model reading another model's window", () => {+ const source = readFileSync(+ new URL("../src/features/chat/api/chat-adapter.ts", import.meta.url),+ "utf8",+ );+ const list = source.slice(+ source.indexOf("const VISIBLE_MODEL_RUNTIME_KEYS = ["),+ source.indexOf("] as const satisfies"),+ );+ for (const key of [+ "loadedContextEnforced",+ "loadedContextUnboundedWhenBatched",+ "loadedParallelSlots",+ ]) {+ assert.match(list, new RegExp(`"${key}"`), `${key} is not preserved`);+ }+});diff --git a/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.ts b/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.tsnew file mode 100644index 00000000000..7109cc017a3--- /dev/null+++ b/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.ts@@ -0,0 +1,92 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+++import assert from "node:assert/strict";+import { register } from "node:module";+import test from "node:test";++import { installLocalStorageFake } from "./helpers/kit.ts";++installLocalStorageFake();+register("./store-settings-resolver.mjs", import.meta.url);++const { useChatRuntimeStore } = await import(+ "../src/features/chat/stores/chat-runtime-store.ts"+);+const { applyActiveModelStatusToStore } = await import(+ "../src/features/chat/lib/apply-inference-status-to-store.ts"+);++const MODEL = "unsloth/Qwen3-8B";++function statusFor(requested: number | null, effective = requested) {+ return {+ active_model: MODEL,+ model_identifier: MODEL,+ is_gguf: false,+ requested_parallel_slots: requested,+ parallel_slots: effective,+ } as never;+}++test("a non-GGUF status carries the width it reports into the rollback baseline", () => {+ useChatRuntimeStore.setState({+ modelLoading: false,+ loadedNParallel: null,+ nParallel: null,+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },+ });++ applyActiveModelStatusToStore(statusFor(8, 1), { previousCheckpoint: MODEL });++ assert.equal(+ useChatRuntimeStore.getState().loadedNParallel,+ 8,+ "the width the load was invoked with is what a rollback must re-send",+ );+});++test("a status reporting no width clears the baseline it would otherwise re-send", () => {+ useChatRuntimeStore.setState({+ modelLoading: false,+ loadedNParallel: 8,+ nParallel: null,+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },+ });++ applyActiveModelStatusToStore(statusFor(null, null), { previousCheckpoint: MODEL });++ assert.equal(useChatRuntimeStore.getState().loadedNParallel, null);+});++test("a remembered non-GGUF width is adopted into the control, not just the baseline", async () => {+ const { setResidentInitialConfig } = await import(+ "./helpers/store-stubs/model-picker.ts"+ );+ setResidentInitialConfig({ remembered: true, config: { nParallel: 2 } });+ useChatRuntimeStore.setState({+ modelLoading: false,+ loadedNParallel: null,+ nParallel: null,+ params: { ...useChatRuntimeStore.getState().params, checkpoint: "unsloth/Other" },+ });++ applyActiveModelStatusToStore(statusFor(2), { previousCheckpoint: "unsloth/Other" });+ setResidentInitialConfig(null);++ assert.equal(useChatRuntimeStore.getState().nParallel, 2);+});++test("the baseline follows a width the server changed underneath this tab", () => {+ useChatRuntimeStore.setState({+ modelLoading: false,+ loadedNParallel: 2,+ nParallel: 2,+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },+ });++ applyActiveModelStatusToStore(statusFor(4), { previousCheckpoint: MODEL });++ assert.equal(useChatRuntimeStore.getState().loadedNParallel, 4);+});diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.tsindex ba4f3f1d64e..93d50341747 100644--- a/studio/frontend/tests/resident-config-match.test.ts+++ b/studio/frontend/tests/resident-config-match.test.ts@@ -733,9 +733,7 @@ test("a non-GGUF resident is not judged on a GGUF invocation field", () => {}),false,);- // And a non-GGUF resident answers for the two fields the backend actually compares,- // which is all _mlx_runtime_settings_match looks at. cache_type_kv is deliberately not- // among them: it is a llama.cpp flag, and the non-GGUF branch never reads it.+ // Non-GGUF matches only what the backend acts on; cache_type_kv is a llama.cpp flag it never reads.assert.equal(matches({ ...DEFAULTS, is_gguf: false, cache_type_kv: "q8_0" }, BLANK),true,@@ -1138,9 +1136,9 @@ test("a diffusion pick is reduced to its lowest GPU, as the backend reduces it",test("no llama.cpp invocation field decides against a non-GGUF resident", () => {// The non-GGUF branch of /load checks identity and _mlx_runtime_settings_match, then- // answers already_loaded. Every other field here is a llama.cpp flag it never reads, so- // a persisted Manual mode, tensor split, slot count or batch size raised the prompt for- // a load that could not have changed anything.+ // answers already_loaded. Every field here is a llama.cpp flag it never reads, so a+ // persisted Manual mode, tensor split or batch size raised the prompt for a load that+ // could not have changed anything.const resident = { ...DEFAULTS, is_gguf: false };assert.equal(matches(resident, {@@ -1149,7 +1147,6 @@ test("no llama.cpp invocation field decides against a non-GGUF resident", () =>gpuLayers: 20,nCpuMoe: 8,tensorParallel: true,- nParallel: 4,nBatch: 2048,nUbatch: 512,selectedGpuIds: [1],@@ -1165,6 +1162,19 @@ test("no llama.cpp invocation field decides against a non-GGUF resident", () =>);});+test("a resident decoding at another width is not adopted, whichever backend", () => {+ for (const is_gguf of [true, false]) {+ assert.equal(+ matches({ ...DEFAULTS, is_gguf, requested_parallel_slots: 4 }, { ...BLANK, nParallel: 2 }),+ false,+ );+ assert.equal(+ matches({ ...DEFAULTS, is_gguf, requested_parallel_slots: 2 }, { ...BLANK, nParallel: 2 }),+ true,+ );+ }+});+test("a diffusion resident is judged on its NGL, not on the placement fields", () => {// The diffusion branch of _runtime_matches_intent replaces the placement comparison with// one _diffusion_manual_ngl check, and an older shim that dropped a manual NGL leavesdiff --git a/studio/frontend/tests/resident-status-baselines.test.ts b/studio/frontend/tests/resident-status-baselines.test.tsindex 9c29f54509b..f9e36c242c1 100644--- a/studio/frontend/tests/resident-status-baselines.test.ts+++ b/studio/frontend/tests/resident-status-baselines.test.ts@@ -207,7 +207,10 @@ test("slot baselines ignore in-flight loads and absent GGUF echoes", () => {test("slotless statuses still clear the baseline without editing the control", () => {const previous = { ...loaded, nParallel: 8 };- for (const echo of [{ is_gguf: false }, { requested_parallel_slots: null }]) {+ for (const echo of [+ { is_gguf: false, requested_parallel_slots: null },+ { requested_parallel_slots: null },+ ]) {const next = applySlots(previous, echo, true, false);assert.equal(next.loadedNParallel, null);assert.equal(next.nParallel, 8);diff --git a/studio/frontend/tests/store-settings-resolver.mjs b/studio/frontend/tests/store-settings-resolver.mjsindex d5387c5b639..08450cf3397 100644--- a/studio/frontend/tests/store-settings-resolver.mjs+++ b/studio/frontend/tests/store-settings-resolver.mjs@@ -7,6 +7,7 @@ import { resolve as resolveBundler } from "./bundler-resolver.mjs";const STUBS = new Map([["@/features/auth", "./helpers/store-stubs/settings-http.ts"],+ ["@/features/hf-auth", "./helpers/store-stubs/auth.ts"],["@/features/hub", "./helpers/store-stubs/hub.ts"],["@/features/model-picker", "./helpers/store-stubs/model-picker.ts"],["@/config/env", "./helpers/store-stubs/env.ts"],diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.pyindex 71cc2bd9b12..d3bb35fc03b 100644--- a/tests/studio/test_model_picker_contracts.py+++ b/tests/studio/test_model_picker_contracts.py@@ -2297,8 +2297,8 @@ def test_parallel_slots_setting_wired_end_to_end():# Click-time snapshot, /load body, validate preflight, cross-model reset and# failed-switch rollback all carry the value.assert "pendingLoadConfig?.nParallel" in runtime- # GGUF-gated, like the compare pane: a transformers load has no slots.- assert "n_parallel: isGguf ? loadNParallel : null," in runtime+ # Both backends take a width: llama-server slots, or the MLX batch width.+ assert "n_parallel: loadNParallel," in runtimeassert "n_parallel: validateNParallel," in runtimeassert "loadNParallel = pendingLoadConfig?.nParallel ?? null;" in runtimeassert "n_parallel: rollbackState.loadedNParallel," in runtime@@ -2342,9 +2342,10 @@ def test_parallel_slots_reach_an_api_load_through_the_server_mirror():assert "n_parallel = payload.n_parallel," in routestore = _read_backend("utils/openai_auto_switch_settings.py")assert 'entry["n_parallel"] = n_parallel' in store- # GGUF-only, like the picker: a safetensors load has no llama-server slots.- gguf_block = store.split(" if is_gguf:", 1)[1]- assert 'kwargs["n_parallel"] = override["n_parallel"]' in gguf_block+ # Ungated: MLX sizes its batch by the same width llama-server sizes its slots by.+ shared, gguf_block = store.split(" if is_gguf:", 1)+ assert '("n_parallel", "n_parallel"),' in shared+ assert "n_parallel" not in gguf_block.split("\n\n", 1)[0]def test_parallel_slots_control_cleared_when_the_load_never_sent_them():@@ -2367,9 +2368,9 @@ def test_parallel_slots_control_cleared_when_the_load_never_sent_them():# The cached-GGUF branch keeps the remembered override via the gated local...assert "nParallel: committedSlots," in gguf_branchassert "nParallel: null," not in gguf_branch- # ...- assert "nParallel: null," in non_gguf_branch- assert "loadedNParallel: null," in non_gguf_branch+ # ...and so does the non-GGUF branch, now that an MLX load takes a width too.+ assert "nParallel: committedSlots," in non_gguf_branch+ assert "loadedNParallel: committedSlots," in non_gguf_branchfresh_default = adapter.split(" return { loaded: false, blockedByTrustRemoteCode: false };", 1@@ -2385,10 +2386,11 @@ def test_hydration_clears_the_slot_baseline_for_a_slotless_model():"""The baseline is what a rollback re-sends and what preset capture reads, so a modelthat cannot have slots must not inherit the previous GGUF's count."""src = _read("features/chat/lib/apply-inference-status-to-store.ts")+ # A non-GGUF load reports its width too, so only the explicit null echo means slotless.assert (- "(status.is_gguf === false || status.requested_parallel_slots === null) && {" in src- ), "the slotless clear must key on is_gguf or an explicit null echo"- clear = src.index("status.is_gguf === false || status.requested_parallel_slots === null")+ "status.requested_parallel_slots === null && {" in src+ ), "the slotless clear must key on an explicit null echo"+ clear = src.index("status.requested_parallel_slots === null && {")assert "loadedNParallel: null," in src[clear : clear + 200]# Never `!= null`: that also matches the absent field an older backend sends.assert "status.requested_parallel_slots !== null && {" not in src@@ -2488,8 +2490,8 @@ def test_hydration_restores_a_remembered_slot_override():": hydratingExistingModel)" in status), "storage is read on a fresh store or a model change, never on a steady poll"assert (- "const rememberedNParallel = status.is_gguf && remembered?.remembered" in status- ), "slots are a llama.cpp knob; reading MLX's record must not seed one"+ "const rememberedNParallel = remembered?.remembered" in status+ ), "a remembered width seeds the control on either backend"assert ("...(seedLoadParams && (slotsUnseeded || slotsModelChanged) &&" in status), "the seed fires in both cases the clear leaves the control blank"