Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Batched serving on the MLX path: several replies decoding at once (#10310)

Long Yixing1232012651a9
diff --git a/studio/backend/core/inference/llama_server_args.py b/studio/backend/core/inference/llama_server_args.py
index caadada73d7..cd11322285f 100644
--- a/studio/backend/core/inference/llama_server_args.py
+++ b/studio/backend/core/inference/llama_server_args.py
@@ -27,6 +27,19 @@ logger = logging.getLogger(__name__)
PARALLEL_MIN = 1
PARALLEL_MAX = 64
+PARALLEL_DEFAULT = 4
+
+
+def clamp_parallel_slots(n_parallel) -> int:
+ if n_parallel is None:
+ return PARALLEL_DEFAULT
+ try:
+ asked = int(n_parallel)
+ except (TypeError, ValueError):
+ return PARALLEL_DEFAULT
+ return max(PARALLEL_MIN, min(PARALLEL_MAX, asked))
+
+
# --batch-size / --ubatch-size range, mirrored by N_BATCH_MIN/MAX in per-model-config.ts
BATCH_MIN = 1
BATCH_MAX = 65536
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 64bc72512c7..851e14b36f1 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -15,6 +15,7 @@ import time
from collections import OrderedDict
from contextlib import ExitStack, closing, contextmanager, nullcontext
from typing import Optional, Generator
+from core.inference.worker import RowRefused
from core.inference import context_refusal
from core.inference.message_content import content_to_text
from core.inference.native_tool_tokens import (
@@ -2501,6 +2502,24 @@ def _make_mlx_presence_penalty_processor(penalty: float):
return _processor
+def _row_logits_processors(processors):
+ """A reply's processors, each starting at the prompt's last token."""
+ if not processors:
+ return []
+
+ def _shown(processor):
+ state = {"offset": None}
+
+ def _narrowed(tokens, logits):
+ if state["offset"] is None:
+ state["offset"] = max(int(tokens.shape[0]) - 1, 0)
+ return processor(tokens[state["offset"] :], logits)
+
+ return _narrowed
+
+ return [_shown(processor) for processor in processors]
+
+
def _make_mlx_frequency_penalty_processor(penalty: float):
"""Frequency penalty as an mlx_lm/mlx_vlm logits processor. Identical to the presence processor
except the scatter *accumulates*, so a token repeated N times in the completion is charged N
@@ -2706,6 +2725,647 @@ class _VLMMediaBlock:
return entries
+class _RowStream:
+ def __init__(
+ self,
+ *,
+ prefix = "",
+ sequences = (),
+ normalizer = None,
+ decode = None,
+ ):
+ self._prefix = prefix
+ self._sequences = sequences
+ self._normalizer = normalizer
+ self._decode = decode
+ self.token_ids = []
+ self.sampled = ""
+ self.stopped = False
+ self._released = 0
+ self._cut = 0
+ self._output = prefix
+
+ def feed(
+ self,
+ token = None,
+ text = "",
+ ):
+ if token is not None:
+ self.token_ids.append(token)
+ if self._decode is not None:
+ self.sampled = self._decode(self.token_ids)
+ else:
+ self.sampled += text or ""
+ if self._sequences:
+ self._cut, self.stopped = _mlx_stop_cut(self.sampled, self._sequences)
+ else:
+ self._cut = len(self.sampled)
+ if self._normalizer is not None:
+ return self._normalize(self.sampled[self._released : self._cut])
+ if not self._sequences:
+ return [self._prefix + self.sampled]
+ if self._decode is not None:
+ return []
+ if self._cut > self._released:
+ self._released = self._cut
+ return [self._prefix + self.sampled[: self._cut]]
+ return []
+
+ def settle(self):
+ if not self._sequences:
+ return []
+ if not self.stopped:
+ self._cut = len(self.sampled)
+ if self._normalizer is not None:
+ return self._normalize(self.sampled[self._released : self._cut])
+ if self._decode is not None:
+ settled = self._prefix + self.sampled[: self._cut]
+ return [settled] if settled != self._prefix else []
+ if not self.stopped and self._released < len(self.sampled):
+ return [self._prefix + self.sampled]
+ return []
+
+ def drain(self, *, cancelled):
+ if self._normalizer is None:
+ return []
+ drained = cancelled and not self.stopped
+ tail = self._normalizer.drain() if drained else self._normalizer.finish()
+ return self._append(tail)
+
+ def _normalize(self, raw):
+ self._released = self._cut
+ return self._append(self._normalizer.feed(raw))
+
+ def _append(self, delta):
+ if not delta:
+ return []
+ self._output += delta
+ return [self._output]
+
+
+def _messages_with_system_prompt(messages, system_prompt):
+ full = [{"role": "system", "content": system_prompt}] if system_prompt else []
+ full.extend(messages)
+ return full
+
+
+def _engine_fields(engine, name):
+ return getattr(getattr(engine, name, None), "__dataclass_fields__", ())
+
+
+def _asks_for_row_processors(request):
+ return (
+ _mlx_sampling_processors(
+ repetition_penalty = request.get("repetition_penalty", 1.0),
+ presence_penalty = request.get("presence_penalty", 0.0),
+ frequency_penalty = request.get("frequency_penalty", 0.0),
+ logit_bias = request.get("logit_bias"),
+ )
+ is not None
+ )
+
+
+def _row_processor_gap(engine, requests):
+ """Why these replies' penalties cannot travel into a batch, or None."""
+ if not any(_asks_for_row_processors(request) for request in requests):
+ return None
+ unavailable = getattr(engine, "row_logits_processors_unavailable_reason", None)
+ return unavailable() if callable(unavailable) else None
+
+
+def _request_batch_gap(request):
+ """What a reply asks for that only the one-at-a-time decode serves, or None."""
+ if request.get("images") or request.get("images_base64"):
+ return "a reply replays tool images"
+ if request.get("video") is not None or request.get("video_base64"):
+ return "a reply carries a video clip"
+ if request.get("tools") or request.get("tool_protocol_active"):
+ return "a reply runs the tool protocol"
+ if request.get("response_format") is not None:
+ return "a reply is constrained to a response format"
+ return None
+
+
+def _batched_vision_engine_gap(engine):
+ def fields(name):
+ return _engine_fields(engine, name)
+
+ if not hasattr(engine, "stream_batch"):
+ return "the installed unsloth-zoo has no batched vision generation"
+ if "seed" not in fields("SamplingParams"):
+ return "the installed unsloth-zoo cannot seed a batched reply"
+ if "prompt_token_count" not in fields("GenerationResult"):
+ return "the installed unsloth-zoo does not report what a batched prompt cost"
+ if "logits_processors" not in fields("GenerationRequest"):
+ return "the installed unsloth-zoo cannot carry a penalty into a batched reply"
+ return None
+
+
+class _TextRowPlan:
+ __slots__ = (
+ "prompt",
+ "think_prefix",
+ "stream",
+ "sampler",
+ "processors",
+ "max_tokens",
+ "detokenizes",
+ )
+
+ def __init__(
+ self,
+ *,
+ prompt,
+ think_prefix,
+ stream,
+ sampler,
+ processors,
+ max_tokens,
+ detokenizes = False,
+ ):
+ self.prompt = prompt
+ self.think_prefix = think_prefix
+ self.stream = stream
+ self.detokenizes = detokenizes
+ self.sampler = sampler
+ self.processors = processors
+ self.max_tokens = max_tokens
+
+
+class _TextBatchRow:
+ __slots__ = (
+ "handle",
+ "plan",
+ "uid",
+ "cache_key",
+ "all_tokens",
+ "prompt_len",
+ "cached",
+ "detokenizer",
+ "generated",
+ "reason",
+ "processing_at",
+ "ready_at",
+ )
+
+ def __init__(self, *, handle, plan, cache_key, all_tokens, prompt_len, cached, detokenizer):
+ self.handle = handle
+ self.plan = plan
+ self.uid = None
+ self.cache_key = cache_key
+ self.all_tokens = all_tokens
+ self.prompt_len = prompt_len
+ self.cached = cached
+ self.detokenizer = detokenizer
+ self.generated = 0
+ self.reason = None
+ self.processing_at = None
+ self.ready_at = None
+
+
+def _row_kwargs(request):
+ """The sampling and render options one batched reply is planned with."""
+ return dict(
+ temperature = request.get("temperature", 0.7),
+ top_p = request.get("top_p", 0.9),
+ top_k = request.get("top_k", 40),
+ min_p = request.get("min_p", 0.0),
+ max_new_tokens = request.get("max_new_tokens", 256),
+ repetition_penalty = request.get("repetition_penalty", 1.0),
+ tools = request.get("tools"),
+ enable_thinking = request.get("enable_thinking"),
+ reasoning_effort = request.get("reasoning_effort"),
+ preserve_thinking = request.get("preserve_thinking"),
+ continue_final_message = request.get("continue_final_message", False),
+ presence_penalty = request.get("presence_penalty", 0.0),
+ seed = request.get("seed"),
+ frequency_penalty = request.get("frequency_penalty", 0.0),
+ logit_bias = request.get("logit_bias"),
+ stop = request.get("stop"),
+ )
+
+
+class _TextBatchSession:
+ """mlx-lm's batch with the reply set left open."""
+
+ def __init__(
+ self,
+ backend,
+ *,
+ width,
+ adapter_state = None,
+ owns_model = False,
+ ):
+ from mlx_lm.generate import BatchGenerator
+
+ self.backend = backend
+ self._adapter_state = adapter_state
+ self._rows = {}
+ self._by_uid = {}
+ self._settled = {}
+ self._held = ExitStack()
+ try:
+ if owns_model:
+ self._held.enter_context(backend._generation_lock)
+ self._held.enter_context(
+ _temporary_mlx_adapter_state(backend._model, adapter_state)
+ )
+ self.generator = BatchGenerator(
+ backend._model,
+ stop_tokens = [
+ [token] for token in _mlx_stop_token_ids(backend._tokenizer, backend._model)
+ ],
+ completion_batch_size = width,
+ prefill_batch_size = width,
+ )
+ except BaseException:
+ self._held.close()
+ raise
+
+ @property
+ def rows_in_flight(self) -> int:
+ return len(self._rows)
+
+ @property
+ def handles(self):
+ return list(self._rows)
+
+ @property
+ def ending(self):
+ return []
+
+ def take_stats(self, handle):
+ """The usage a retired reply settled with, once."""
+ return self._settled.pop(handle, None)
+
+ def admit(self, request, handle):
+ """Take one reply in, answering with the text preceding its first token."""
+ backend = self.backend
+ plan = backend._plan_text_row(
+ _messages_with_system_prompt(
+ request.get("messages") or [],
+ request.get("system_prompt", ""),
+ ),
+ **_row_kwargs(request),
+ )
+ prefix, remainder, cache, cache_key, all_tokens = backend._batch_prompt_row(
+ plan.prompt,
+ self._adapter_state,
+ )
+ row = _TextBatchRow(
+ handle = handle,
+ plan = plan,
+ cache_key = cache_key,
+ all_tokens = all_tokens,
+ prompt_len = len(remainder),
+ cached = len(prefix),
+ detokenizer = backend._tokenizer.detokenizer if plan.detokenizes else None,
+ )
+ uids = self.generator.insert(
+ [remainder],
+ max_tokens = [plan.max_tokens],
+ caches = [cache],
+ all_tokens = [prefix],
+ samplers = [plan.sampler],
+ logits_processors = [_row_logits_processors(plan.processors)],
+ state_machines = None,
+ )
+ if len(uids) != 1:
+ self.generator.remove(list(uids))
+ raise RuntimeError(
+ f"mlx-lm admitted {len(uids)} rows for one reply; a batch cannot "
+ "tell whose tokens are whose."
+ )
+ row.uid = uids[0]
+ self._rows[handle] = row
+ self._by_uid[row.uid] = row
+ logger.info(
+ "Admitted reply %s to the batch: prompt=%d, cached=%d, max_tokens=%d, model=%s",
+ handle,
+ row.prompt_len,
+ row.cached,
+ plan.max_tokens,
+ type(backend._model).__name__,
+ )
+ return plan.think_prefix
+
+ def step(self):
+ """Report what the batch produced, retiring every reply that ended."""
+ began = time.perf_counter()
+ prompt_events, events = self.generator.next()
+ now = time.perf_counter() if (events or prompt_events) else None
+ for event in prompt_events:
+ row = self._by_uid.get(event.uid)
+ if row is not None and row.processing_at is None:
+ row.processing_at = began
+ for event in events:
+ row = self._by_uid.get(event.uid)
+ if row is None:
+ continue
+ if row.processing_at is None:
+ row.processing_at = began
+ if row.ready_at is None:
+ row.ready_at = now
+ yield from self._feed(row, event)
+
+ def _feed(self, row, event):
+ stream = row.plan.stream
+ text = None
+ if event.finish_reason is None:
+ if row.detokenizer is not None:
+ row.detokenizer.add_token(event.token)
+ text = row.detokenizer.last_segment
+ else:
+ text = ""
+ if row.detokenizer is not None:
+ if event.finish_reason == "length":
+ row.detokenizer.add_token(event.token)
+ row.detokenizer.finalize()
+ text = row.detokenizer.last_segment
+ row.generated += 1
+ yield from (
+ (row.handle, snapshot) for snapshot in stream.feed(token = event.token, text = text)
+ )
+ if not (stream.stopped or event.finish_reason is not None):
+ return
+ row.reason = "stop" if stream.stopped else event.finish_reason
+ banked = self.generator.remove(
+ [row.uid],
+ return_prompt_caches = event.finish_reason is None,
+ )
+ self.backend._bank_row_cache(
+ row.cache_key,
+ event,
+ banked.get(row.uid),
+ row.all_tokens,
+ stream,
+ )
+ yield from self._retire(row, cancelled = False)
+
+ def withdraw(self, handles):
+ rows = [self._rows[handle] for handle in handles if handle in self._rows]
+ if not rows:
+ return
+ taken = self.generator.remove([row.uid for row in rows], return_prompt_caches = True)
+ for row in rows:
+ row.reason = "stop"
+ self.backend._bank_row_cache(
+ row.cache_key,
+ None,
+ taken.get(row.uid),
+ row.all_tokens,
+ row.plan.stream,
+ )
+ yield from self._retire(row, cancelled = True)
+
+ def _retire(self, row, *, cancelled):
+ self._rows.pop(row.handle, None)
+ self._by_uid.pop(row.uid, None)
+ stream = row.plan.stream
+ yield from ((row.handle, snap) for snap in stream.settle())
+ yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
+ ready_at = row.ready_at or time.perf_counter()
+ decoded = time.perf_counter() - ready_at
+ prefill_s = ready_at - (row.processing_at or ready_at)
+ self._settled[row.handle] = _build_generation_stats(
+ row.prompt_len,
+ (row.prompt_len / prefill_s) if prefill_s > 0 else 0.0,
+ row.generated,
+ row.generated / max(decoded, 1e-9),
+ row.cached,
+ finish_reason = row.reason,
+ )
+ yield row.handle, None
+
+ def close(self):
+ try:
+ try:
+ if self._by_uid:
+ self.generator.remove(list(self._by_uid))
+ finally:
+ self.generator.close()
+ finally:
+ self._rows.clear()
+ self._by_uid.clear()
+ self._settled.clear()
+ self._held.close()
+
+
+class _VLMRowPlan:
+ __slots__ = (
+ "prompt",
+ "images",
+ "think_prefix",
+ "stream",
+ "kwargs",
+ "sampling",
+ "max_tokens",
+ "processors",
+ )
+
+ def __init__(
+ self,
+ *,
+ prompt,
+ images,
+ think_prefix,
+ stream,
+ kwargs,
+ sampling,
+ max_tokens,
+ processors = None,
+ ):
+ self.prompt = prompt
+ self.images = images
+ self.think_prefix = think_prefix
+ self.stream = stream
+ self.kwargs = kwargs
+ self.sampling = sampling
+ self.max_tokens = max_tokens
+ self.processors = processors
+
+
+class _VisionBatchRow:
+ __slots__ = (
+ "handle",
+ "plan",
+ "row",
+ "generated",
+ "reason",
+ "prompt_tokens",
+ "admitted_at",
+ "ready_at",
+ "cancelled",
+ )
+
+ def __init__(self, *, handle, plan, row):
+ self.handle = handle
+ self.plan = plan
+ self.row = row
+ self.generated = 0
+ self.reason = None
+ self.prompt_tokens = 0
+ self.admitted_at = time.perf_counter()
+ self.ready_at = None
+ self.cancelled = False
+
+
+class _VisionBatchSession:
+ def __init__(
+ self,
+ backend,
+ *,
+ width,
+ adapter_state = None,
+ owns_model = False,
+ ):
+ from unsloth_zoo.mlx.generate import BatchStream, GenerationDefaults
+
+ self.backend = backend
+ self._adapter_state = adapter_state
+ self._rows = {}
+ self._by_row = {}
+ self._settled = {}
+ self._held = ExitStack()
+ try:
+ if owns_model:
+ self._held.enter_context(backend._generation_lock)
+ self._held.enter_context(
+ _temporary_mlx_adapter_state(backend._model, adapter_state)
+ )
+ self.stream = BatchStream(
+ backend._model,
+ backend._processor,
+ defaults = GenerationDefaults(
+ prefill_batch_size = 1,
+ completion_batch_size = width,
+ ),
+ )
+ except BaseException:
+ self._held.close()
+ raise
+
+ @property
+ def rows_in_flight(self) -> int:
+ return len(self._rows)
+
+ @property
+ def handles(self):
+ return list(self._rows)
+
+ @property
+ def ending(self):
+ return [handle for handle, row in self._rows.items() if row.cancelled]
+
+ def take_stats(self, handle):
+ return self._settled.pop(handle, None)
+
+ def admit(self, request, handle):
+ from unsloth_zoo.mlx.generate import (
+ BatchRowRefused,
+ GenerationRequest,
+ SamplingParams,
+ )
+
+ backend = self.backend
+ plan = backend._plan_vlm_request(request)
+ if plan.images:
+ backend._release_vlm_snapshots()
+ try:
+ row_number = self.stream.add(
+ GenerationRequest(
+ prompt = plan.prompt,
+ image = plan.images[0] if plan.images else None,
+ max_tokens = plan.max_tokens,
+ sampling = SamplingParams(**plan.sampling),
+ logits_processors = plan.processors,
+ )
+ )
+ except BatchRowRefused as refusal:
+ raise RowRefused(str(refusal)) from refusal
+ row = _VisionBatchRow(handle = handle, plan = plan, row = row_number)
+ self._rows[handle] = row
+ self._by_row[row_number] = row
+ logger.info(
+ "Admitted vision reply %s to the batch: images=%d, max_tokens=%d, model=%s",
+ handle,
+ len(plan.images or ()),
+ plan.max_tokens,
+ type(backend._model).__name__,
+ )
+ return plan.think_prefix
+
+ def step(self):
+ now = None
+ for event in self.stream.step():
+ row = self._by_row.get(event.index)
+ if row is None:
+ continue
+ if now is None:
+ now = time.perf_counter()
+ if row.ready_at is None:
+ row.ready_at = now
+ if event.delta and not row.cancelled:
+ yield from (
+ (row.handle, snapshot) for snapshot in row.plan.stream.feed(text = event.delta)
+ )
+ if row.plan.stream.stopped:
+ row.cancelled = True
+ row.reason = "stop"
+ if event.result is not None:
+ result = event.result
+ if not row.cancelled:
+ row.reason = result.finish_reason
+ row.prompt_tokens = result.prompt_token_count
+ row.generated = len(result.token_ids) + (result.finish_reason == "stop")
+ yield from self._retire(row, cancelled = row.cancelled)
+ elif row.cancelled:
+ yield from self._retake(row)
+
+ def withdraw(self, handles):
+ for handle in list(handles):
+ row = self._rows.get(handle)
+ if row is None:
+ continue
+ row.cancelled = True
+ row.reason = "stop"
+ yield from self._retake(row)
+
+ def _retake(self, row):
+ managed = self.stream.withdraw(row.row)
+ if managed is not None:
+ row.prompt_tokens = managed.prompt_token_count
+ row.generated = len(managed.token_ids)
+ yield from self._retire(row, cancelled = True)
+
+ def _retire(self, row, *, cancelled):
+ self._rows.pop(row.handle, None)
+ self._by_row.pop(row.row, None)
+ stream = row.plan.stream
+ yield from ((row.handle, snap) for snap in stream.settle())
+ yield from ((row.handle, snap) for snap in stream.drain(cancelled = cancelled))
+ ready_at = row.ready_at or time.perf_counter()
+ decoded = time.perf_counter() - ready_at
+ prefill_s = ready_at - row.admitted_at
+ self._settled[row.handle] = _build_generation_stats(
+ row.prompt_tokens,
+ (row.prompt_tokens / prefill_s) if prefill_s > 0 else 0.0,
+ row.generated,
+ row.generated / max(decoded, 1e-9),
+ finish_reason = row.reason or "stop",
+ )
+ yield row.handle, None
+
+ def close(self):
+ try:
+ self.stream.close()
+ finally:
+ self._rows.clear()
+ self._by_row.clear()
+ self._settled.clear()
+ self._held.close()
+
+
class MLXInferenceBackend:
def __init__(self):
self.models = {}
@@ -2716,6 +3376,7 @@ class MLXInferenceBackend:
self._generation_lock = threading.Lock()
# usage, timings and terminal reason of the latest generation, shipped on gen_done.
self.last_generation_stats = None
+ self.last_batch_generation_stats = []
self._model = None
self._model_fusion = ExitStack()
@@ -3392,6 +4053,7 @@ class MLXInferenceBackend:
# unbounded, None nothing could be built to judge. Without it the API reports a limit a client cannot tell
# from an enforced one.
"context_length_enforced": _ctx_enforced,
+ "context_unbounded_when_batched": self._vision_batch_is_available(),
"mlx_context_budget": self._kv_context_budget,
"mlx_kv_bits": self._kv_quant["kv_bits"],
"mlx_kv_bits_requested": self._kv_quant["requested_kv_bits"],
@@ -3589,6 +4251,24 @@ class MLXInferenceBackend:
full_messages.extend(messages)
return full_messages
+ def _conversation_to_render(self, messages, system_prompt, image):
+ """The turns this request renders, whichever way its reply is decoded."""
+ if self._is_vlm and image is not None:
+ from core.inference.chat_template_helpers import (
+ chat_render_target as _chat_render_target,
+ )
+ full_messages = messages_with_attached_image(
+ messages,
+ system_prompt = system_prompt,
+ structured_content = (
+ self._processor is not None
+ and _chat_render_target(self._processor) is self._processor
+ ),
+ )
+ else:
+ full_messages = self._with_system_prompt(messages, system_prompt)
+ return full_messages
+
def count_chat_tokens(
self,
messages,
@@ -4091,6 +4771,109 @@ class MLXInferenceBackend:
if stopped:
self._mark_stopped()
+ def _plan_text_row(
+ self,
+ messages,
+ *,
+ temperature,
+ top_p,
+ top_k,
+ min_p,
+ max_new_tokens,
+ repetition_penalty,
+ tools = None,
+ enable_thinking = None,
+ reasoning_effort = None,
+ preserve_thinking = None,
+ continue_final_message = False,
+ presence_penalty = 0.0,
+ seed = None,
+ frequency_penalty = 0.0,
+ logit_bias = None,
+ stop = None,
+ tool_protocol_active = None,
+ ) -> "_TextRowPlan":
+ from mlx_lm.sample_utils import make_sampler
+
+ from core.inference.chat_template_helpers import detect_think_prefill
+
+ render_result = self._render_text_prompt(
+ messages,
+ tools = tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ )
+ prompt = render_result.prompt
+ reasoning_channel_markers = render_result.reasoning_channel_markers
+ _resumed_partial = bool(continue_final_message and trailing_assistant_text(messages))
+
+ think_prefix = detect_think_prefill(
+ prompt,
+ getattr(self._tokenizer, "all_special_tokens", None),
+ preserves_think_close = (
+ bool(tools) or tool_protocol_active or reasoning_channel_markers is not None
+ )
+ and decoder_preserves_token(
+ self._tokenizer, "</think>", reasoning_control_tokens(reasoning_channel_markers)
+ ),
+ )
+ if seed is None:
+ sampler = make_sampler(
+ temp = temperature,
+ top_p = top_p,
+ top_k = int(top_k or 0),
+ min_p = float(min_p or 0.0),
+ min_tokens_to_keep = 1,
+ )
+ else:
+ sampler = _make_seeded_mlx_sampler(
+ seed,
+ temp = temperature,
+ top_p = top_p,
+ top_k = int(top_k or 0),
+ min_p = float(min_p or 0.0),
+ )
+ logits_processors = _mlx_sampling_processors(
+ repetition_penalty = repetition_penalty,
+ presence_penalty = presence_penalty,
+ frequency_penalty = frequency_penalty,
+ logit_bias = logit_bias,
+ )
+
+ normalizer = (
+ make_reasoning_normalizer(
+ reasoning_channel_markers,
+ in_reasoning = prompt_opens_reasoning_channel(
+ prompt, reasoning_channel_markers, _resumed_partial
+ ),
+ )
+ if reasoning_channel_markers is not None
+ else None
+ )
+ row = _RowStream(
+ prefix = think_prefix,
+ sequences = _mlx_stop_sequences(stop),
+ normalizer = normalizer,
+ decode = (
+ None
+ if normalizer is not None
+ else lambda ids: self._tokenizer.decode(ids, skip_special_tokens = True)
+ ),
+ )
+ return _TextRowPlan(
+ prompt = prompt,
+ think_prefix = think_prefix,
+ stream = row,
+ detokenizes = normalizer is not None,
+ sampler = sampler,
+ processors = logits_processors,
+ max_tokens = (
+ self._unset_generation_budget(prompt) if max_new_tokens is None else max_new_tokens
+ ),
+ )
+
def _render_vlm_prompt(
self,
messages,
@@ -4223,6 +5006,118 @@ class MLXInferenceBackend:
)
return prompt, chat_target
+ def _plan_vlm_request(self, request) -> "_VLMRowPlan":
+ image = request.get("image")
+ messages = request.get("messages") or []
+ conversation = self._conversation_to_render(
+ messages, request.get("system_prompt", ""), image
+ )
+ return self._plan_vlm_row(conversation, image, **_row_kwargs(request))
+
+ def _plan_vlm_row(
+ self,
+ messages,
+ image,
+ *,
+ temperature,
+ top_p,
+ top_k,
+ min_p,
+ max_new_tokens,
+ repetition_penalty,
+ tools = None,
+ enable_thinking = None,
+ reasoning_effort = None,
+ preserve_thinking = None,
+ continue_final_message = False,
+ presence_penalty = 0.0,
+ seed = None,
+ frequency_penalty = 0.0,
+ logit_bias = None,
+ stop = None,
+ ) -> "_VLMRowPlan":
+ images = [image] if image is not None else None
+ prompt, chat_target = self._render_vlm_prompt(
+ messages,
+ images,
+ tools = tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ )
+
+ from core.inference.chat_template_helpers import detect_think_prefill
+
+ think_prefix = detect_think_prefill(
+ prompt, getattr(chat_target, "all_special_tokens", None)
+ )
+ markers = detect_reasoning_channel_markers(chat_target, tools = tools)
+ normalizer = (
+ make_reasoning_normalizer(
+ markers,
+ in_reasoning = prompt_opens_reasoning_channel(
+ prompt,
+ markers,
+ bool(continue_final_message and trailing_assistant_text(messages)),
+ ),
+ )
+ if markers is not None
+ else None
+ )
+ row = _RowStream(
+ prefix = think_prefix,
+ sequences = _mlx_stop_sequences(stop),
+ normalizer = normalizer,
+ )
+ if max_new_tokens is None:
+ max_new_tokens = self._unset_generation_budget(prompt)
+ if images:
+ max_new_tokens = min(max_new_tokens, UNSET_GENERATION_BUDGET)
+ vlm_kwargs = dict(
+ max_tokens = max_new_tokens,
+ temperature = temperature,
+ top_p = top_p,
+ top_k = int(top_k or 0),
+ min_p = float(min_p or 0.0),
+ )
+ vlm_kwargs.update(self._kv_window_generate_kwargs())
+ if seed is not None:
+ vlm_kwargs["sampler"] = _make_seeded_mlx_sampler(
+ seed,
+ temp = temperature,
+ top_p = top_p,
+ top_k = int(top_k or 0),
+ min_p = float(min_p or 0.0),
+ )
+ processors = _mlx_sampling_processors(
+ repetition_penalty = repetition_penalty,
+ presence_penalty = presence_penalty,
+ frequency_penalty = frequency_penalty,
+ logit_bias = logit_bias,
+ )
+ if processors:
+ # Never the repetition_penalty shortcut: mlx-vlm would build that processor
+ # itself, outside the narrowing every other path applies.
+ vlm_kwargs["logits_processors"] = _row_logits_processors(processors)
+
+ return _VLMRowPlan(
+ prompt = prompt,
+ images = images,
+ think_prefix = think_prefix,
+ stream = row,
+ kwargs = vlm_kwargs,
+ sampling = {
+ "temperature": vlm_kwargs["temperature"],
+ "top_p": vlm_kwargs["top_p"],
+ "top_k": max(vlm_kwargs["top_k"], 0),
+ "min_p": vlm_kwargs["min_p"],
+ "seed": seed,
+ },
+ max_tokens = vlm_kwargs["max_tokens"],
+ processors = processors,
+ )
+
def _generate_vlm(
self,
messages,
@@ -4555,6 +5450,295 @@ class MLXInferenceBackend:
if stopped:
self._mark_stopped()
+ def _kv_policy_batch_reason(self):
+ if self._kv_quant_bits() is not None or getattr(self, "_kv_context_budget", None):
+ return "the load quantizes or budgets its KV cache, which a batch does not carry"
+ return None
+
+ def batch_unavailable_reason(self, requests):
+ if self._model is None:
+ return "no model is loaded"
+ for request in requests:
+ reason = _request_batch_gap(request)
+ if reason is not None:
+ return reason
+ reason = self._kv_policy_batch_reason()
+ if reason is not None:
+ return reason
+ if len(requests) < 2:
+ return "fewer than two replies were requested"
+ if self._is_vlm:
+ reason = self._vlm_batch_unavailable_reason(requests)
+ if reason is not None:
+ return reason
+ if any(_mlx_stop_sequences(request.get("stop")) for request in requests):
+ return "a reply asks for stop sequences"
+ return None
+ return self._text_batch_unavailable_reason()
+
+ def resident_unavailable_reason(self, request):
+ if self._model is None:
+ return "no model is loaded"
+ reason = _request_batch_gap(request)
+ if reason is not None:
+ return reason
+ reason = self._kv_policy_batch_reason()
+ if reason is not None:
+ return reason
+ if self._is_vlm:
+ return self._vlm_resident_unavailable_reason(request)
+ return self._text_batch_unavailable_reason()
+
+ def _text_batch_unavailable_reason(self):
+ """Why the installed mlx-lm cannot batch this load's replies, or None."""
+ try:
+ from unsloth_zoo.mlx.generate import stream_unavailable_reason
+ except ImportError:
+ return "the installed unsloth-zoo cannot keep a batch open"
+ return stream_unavailable_reason(self._model, self._tokenizer)
+
+ def _vlm_resident_unavailable_reason(self, request):
+ reason = self._vlm_batch_unavailable_reason([request])
+ if reason is not None:
+ return reason
+ try:
+ from unsloth_zoo.mlx.generate import stream_unavailable_reason
+ except ImportError:
+ return "the installed unsloth-zoo cannot keep a vision batch open"
+ return stream_unavailable_reason(self._model, self._processor)
+
+ def open_resident_batch(
+ self,
+ *,
+ width,
+ adapter_state = None,
+ ):
+ reason = self.resident_unavailable_reason({})
+ if reason is not None:
+ raise RuntimeError(f"MLX batched generation is unavailable: {reason}")
+ session = _VisionBatchSession if self._is_vlm else _TextBatchSession
+ return session(
+ self,
+ width = width,
+ adapter_state = adapter_state,
+ owns_model = True,
+ )
+
+ def _vision_batch_is_available(self) -> bool:
+ """Whether this load will decode vision replies together, asked without a request."""
+ if not self._is_vlm:
+ return False
+ return (
+ self.batch_unavailable_reason([{}, {}]) is None
+ or self.resident_unavailable_reason({}) is None
+ )
+
+ def _vlm_batch_unavailable_reason(self, requests):
+ try:
+ from unsloth_zoo.mlx import generate as engine
+ except ImportError:
+ return "the installed unsloth-zoo has no MLX generation"
+ gap = _batched_vision_engine_gap(engine)
+ if gap is not None:
+ return gap
+ gap = _row_processor_gap(engine, requests)
+ if gap is not None:
+ return gap
+ if not getattr(self._model, "_is_vlm_model", False):
+ return "this model was not loaded as a vision model"
+ return None
+
+ def generate_chat_batch(
+ self,
+ requests,
+ *,
+ cancel_event = None,
+ _adapter_state = None,
+ ):
+ reason = self.batch_unavailable_reason(requests)
+ if reason is not None:
+ raise RuntimeError(f"MLX batched generation is unavailable: {reason}")
+
+ self.last_batch_generation_stats = [None] * len(requests)
+ self.last_generation_stats = None
+
+ run = self._generate_vlm_batch if self._is_vlm else self._generate_text_batch
+ yield from run(requests, cancel_event = cancel_event, _adapter_state = _adapter_state)
+
+ def _generate_text_batch(
+ self,
+ requests,
+ *,
+ cancel_event = None,
+ _adapter_state = None,
+ ):
+ def report(events):
+ for row, snapshot in events:
+ if snapshot is None:
+ self.last_batch_generation_stats[row] = session.take_stats(row)
+ yield row, snapshot
+
+ with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):
+ session = _TextBatchSession(
+ self,
+ width = len(requests),
+ adapter_state = _adapter_state,
+ )
+ try:
+ prefixes = [
+ (row, session.admit(request, row)) for row, request in enumerate(requests)
+ ]
+ for row, prefix in prefixes:
+ if prefix:
+ yield row, prefix
+ while session.rows_in_flight:
+ yield from report(session.step())
+ if cancel_event is not None and cancel_event.is_set():
+ yield from report(session.withdraw(list(session.handles)))
+ finally:
+ session.close()
+
+ def _generate_vlm_batch(
+ self,
+ requests,
+ *,
+ cancel_event = None,
+ _adapter_state = None,
+ ):
+ import time
+
+ from unsloth_zoo.mlx.generate import (
+ GenerationDefaults,
+ GenerationRequest,
+ SamplingParams,
+ stream_batch,
+ )
+
+ plans = [self._plan_vlm_request(request) for request in requests]
+ batch = [
+ GenerationRequest(
+ prompt = plan.prompt,
+ image = plan.images[0] if plan.images else None,
+ max_tokens = plan.max_tokens,
+ sampling = SamplingParams(**plan.sampling),
+ logits_processors = plan.processors,
+ )
+ for plan in plans
+ ]
+ defaults = GenerationDefaults(
+ max_tokens = max(plan.max_tokens for plan in plans),
+ prefill_batch_size = len(plans),
+ completion_batch_size = len(plans),
+ )
+
+ with self._generation_lock, _temporary_mlx_adapter_state(self._model, _adapter_state):
+ if any(plan.images for plan in plans):
+ self._release_vlm_snapshots()
+ logger.info(
+ "Generating %d vision replies as one batch: images=%d, max_tokens=%s, model=%s",
+ len(plans),
+ sum(1 for plan in plans if plan.images),
+ [plan.max_tokens for plan in plans],
+ type(self._model).__name__,
+ )
+ for row, plan in enumerate(plans):
+ if plan.think_prefix:
+ yield row, plan.think_prefix
+ started = time.perf_counter()
+ prefilled_at: list[float | None] = [None] * len(plans)
+ counted: dict[int, tuple[int, int]] = {}
+ pending = set(range(len(plans)))
+ events = stream_batch(self._model, self._processor, batch, defaults = defaults)
+
+ def _retire(row, result, *, cancelled):
+ stream = plans[row].stream
+ yield from ((row, snap) for snap in stream.settle())
+ yield from ((row, snap) for snap in stream.drain(cancelled = cancelled))
+ ready_at = prefilled_at[row] or time.perf_counter()
+ decoded = time.perf_counter() - ready_at
+ prefill_s = ready_at - started
+ seen_prompt, seen_generated = counted.get(row, (0, 0))
+ prompt_n = result.prompt_token_count if result is not None else seen_prompt
+ generated = (
+ len(result.token_ids) + (result.finish_reason == "stop")
+ if result is not None
+ else seen_generated
+ )
+ self.last_batch_generation_stats[row] = _build_generation_stats(
+ prompt_n,
+ (prompt_n / prefill_s) if prefill_s > 0 else 0.0,
+ generated,
+ generated / max(decoded, 1e-9),
+ finish_reason = (result.finish_reason if result is not None else "stop"),
+ )
+ yield row, None
+
+ try:
+ for event in events:
+ row = event.index
+ if row not in pending:
+ continue
+ if prefilled_at[row] is None:
+ prefilled_at[row] = time.perf_counter()
+ counted[row] = (
+ getattr(event, "prompt_tokens", 0),
+ getattr(event, "generated_tokens", 0),
+ )
+ if event.delta:
+ yield from (
+ (row, snapshot) for snapshot in plans[row].stream.feed(text = event.delta)
+ )
+ if event.result is not None:
+ pending.discard(row)
+ yield from _retire(row, event.result, cancelled = False)
+ if cancel_event is not None and cancel_event.is_set():
+ break
+ if pending and not (cancel_event is not None and cancel_event.is_set()):
+ raise RuntimeError(
+ f"Batched vision generation ended with {len(pending)} of "
+ f"{len(plans)} replies unfinished."
+ )
+ for row in sorted(pending):
+ yield from _retire(row, None, cancelled = True)
+ finally:
+ close = getattr(events, "close", None)
+ if callable(close):
+ close()
+
+ def _batch_prompt_row(self, prompt, adapter_state):
+ rest, cache, key, tokens, cached_n = self._prepare_prompt_cache(prompt, adapter_state)
+ if cache is None or tokens is None:
+ tokens = self._encode_prompt(prompt)
+ rest, cached_n, key = tokens, 0, None
+ cache = self._make_row_cache()
+ return list(tokens[:cached_n]), list(rest), cache, key, list(tokens)
+
+ def _make_row_cache(self):
+ """A row's own cache: mlx-lm applies ``max_kv_size`` only to caches it builds."""
+ from mlx_lm.models.cache import make_prompt_cache
+
+ window = getattr(self, "_kv_cache_window", None)
+ if window is None:
+ return make_prompt_cache(self._model)
+ return make_prompt_cache(self._model, max_kv_size = window)
+
+ def _bank_row_cache(self, cache_key, event, withdrawn, prompt_tokens, stream):
+ history = self._prompt_cache_history
+ if history is None or cache_key is None:
+ return
+ cache = getattr(event, "prompt_cache", None)
+ tokens = getattr(event, "all_tokens", None)
+ if cache is None and withdrawn is not None:
+ cache, tokens = withdrawn
+ if cache is None:
+ return
+ if not tokens:
+ tokens = prompt_tokens + stream.token_ids
+ try:
+ history.insert(cache_key, list(tokens), cache)
+ except Exception as exc:
+ logger.debug("MLX prompt cache insert failed: %s", exc)
+
def generate_audio_input_response(
self,
messages,
@@ -4687,7 +5871,6 @@ class MLXInferenceBackend:
max_new_tokens,
),
)
- # As in _generate_text: what was withheld is ordinary text now.
if sequences and not stopped:
delta = sampled[released:]
if normalizer is not None:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 2ec80c61d12..71fcf282611 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -9,6 +9,7 @@ Pattern follows core/training/training.py."""
import atexit
import base64
+import contextlib
import os
import signal
from loggers import get_logger
@@ -30,6 +31,7 @@ from core.inference.audio_errors import (
AudioBackendUnsupportedError,
AudioGenerationCancelledError,
)
+from core.inference.worker import PendingTeardowns, StopLedger
from utils.hardware import get_device, prepare_gpu_selection
from utils.utils import hf_env_offline, is_metal_queue_dead
@@ -191,10 +193,13 @@ _CTX = mp.get_context("spawn")
_DISPATCH_READ_TIMEOUT = 30.0
+
+_STOP_NOTICE_INTERVAL = 0.1
_DISPATCH_POLL_INTERVAL = 0.5
_DISPATCH_STOP_TIMEOUT = 5.0
_DISPATCH_IDLE_TIMEOUT = 30.0
_DISPATCH_DRAIN_TIMEOUT = 5.0
+_CANCELLED_ROWS_GRACE = 5.0
# Only bounds the Transformers subprocess path; llama.cpp TTS never reaches here. 120s was tuned against GGUF speeds
# and killed real work: a safetensors LoRA on a mid-range GPU needs minutes for the same clip a GGUF returns in
@@ -370,6 +375,7 @@ def _mirrored_model_entry(model_info: dict, model_name: str) -> dict:
"max_context_length": model_info.get("max_context_length"),
"requested_context_length": model_info.get("requested_context_length"),
"context_length_enforced": model_info.get("context_length_enforced"),
+ "context_unbounded_when_batched": model_info.get("context_unbounded_when_batched"),
"mlx_context_budget": model_info.get("mlx_context_budget"),
}
@@ -390,6 +396,8 @@ class InferenceOrchestrator:
# Set for the whole unload; the worker never clears it (unlike _cancel_event), so a generate queued behind the
# cancelled one is skipped, not run.
self._drain_event: Any = None
+ self._stop_ledger: Any = None
+ self._pending_teardowns: Any = None
self._gen_lock = threading.Lock() # Serializes generation
# Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the running generation or is
# queued behind it (the worker's event is shared).
@@ -397,16 +405,11 @@ class InferenceOrchestrator:
self._executing_cancel_events: list = []
self._active_cancel_lock = threading.Lock()
# Held across claim + _send_cmd so claim order matches the subprocess dequeue order, which _owns_worker relies
- # on.
- self._send_order_lock = threading.Lock()
+ self._send_order_lock = threading.RLock()
# Set during a switch so a generation winning the _gen_lock handoff bails instead of starting on the outgoing
# model
self._unload_pending = False
- # Blocking TTS has no token response that can safely establish worker ownership while it is queued behind
- # compare-mode work. Reserve the dispatcher admission lane for the whole TTS request so its shared cancel
- # event cannot hit a sibling. Mutated under _dispatcher_lifecycle_lock while holding _gen_lock.
- self._exclusive_tts_pending = False
- self._exclusive_vram_probe_pending = False
+ self._worker_reserved_for: Optional[str] = None
# Dispatcher state for compare mode (adapter-controlled requests): bypass _gen_lock, send commands directly,
# read from per-request mailboxes routed by a dispatcher thread on request_id.
@@ -426,6 +429,7 @@ class InferenceOrchestrator:
# and each spawn one, orphaning the extra thread (self._dispatcher_thread tracks only the last). The orphan
# later steals the "unloaded" reply off resp_queue and hangs unload_model.
self._dispatcher_lifecycle_lock = threading.Lock()
+ self._worker_released = threading.Condition(self._dispatcher_lifecycle_lock)
# Local state mirrors (updated from subprocess responses)
self.active_model_name: Optional[str] = None
@@ -495,6 +499,23 @@ class InferenceOrchestrator:
self._top_models_started = True
threading.Thread(target = self._fetch_top_models, daemon = True, name = "top-models").start()
+ def set_parallel_slots(self, n_parallel) -> int:
+ from core.inference.llama_server_args import clamp_parallel_slots
+
+ slots = clamp_parallel_slots(n_parallel)
+ entry = self.models.get(self.active_model_name or "")
+ if entry is not None:
+ entry["parallel_slots"] = slots
+ return slots
+
+ @property
+ def effective_parallel_slots(self) -> int:
+ from core.inference.llama_server_args import PARALLEL_DEFAULT
+
+ entry = self.models.get(self.active_model_name or "") or {}
+ slots = entry.get("parallel_slots")
+ return slots if isinstance(slots, int) and slots > 0 else PARALLEL_DEFAULT
+
@property
def default_models(self) -> list[str]:
self._refresh_static_models_if_stale()
@@ -614,6 +635,8 @@ class InferenceOrchestrator:
self._resp_queue = _CTX.Queue()
self._cancel_event = _CTX.Event()
self._drain_event = _CTX.Event()
+ self._stop_ledger = StopLedger(_CTX)
+ self._pending_teardowns = PendingTeardowns(_CTX)
# Built into a local FIRST, and started through that local. A shutdown can
# observe a not-yet-alive child, clear self._proc and finish its sweep while
@@ -626,6 +649,8 @@ class InferenceOrchestrator:
"resp_queue": self._resp_queue,
"cancel_event": self._cancel_event,
"drain_event": self._drain_event,
+ "stop_ledger": self._stop_ledger,
+ "pending_teardowns": self._pending_teardowns,
"config": config,
}
if self._stderr_capture is not None:
@@ -684,10 +709,18 @@ class InferenceOrchestrator:
logger.info("Inference subprocess started (pid=%s)", _spawned_proc.pid)
def _cancel_generation(self) -> None:
- """Cancel any ongoing generation in the subprocess (instant)."""
if self._cancel_event is not None:
self._cancel_event.set()
+ def _cancel_every_generation(self) -> None:
+ self._teardown_going_out()
+ self._cancel_generation()
+ try:
+ self._send_cmd({"type": "cancel"})
+ except Exception:
+ self._teardown_not_sent()
+ logger.debug("Could not tell the worker to let go of its batch", exc_info = True)
+
def is_worker_alive(self) -> bool:
"""True while the inference subprocess is running, even with no model active (a failed load
can leave a live worker holding sidecar modules)."""
@@ -708,12 +741,12 @@ class InferenceOrchestrator:
return None
with self._dispatcher_lifecycle_lock:
- if self._exclusive_tts_pending or getattr(self, "_exclusive_vram_probe_pending", False):
+ if self._worker_reserved_for is not None:
return None
- self._exclusive_vram_probe_pending = True
+ self._worker_reserved_for = "model switch is checking GPU memory"
acquired = False
try:
- if not self._wait_dispatcher_idle():
+ if not self._wait_worker_idle():
return None
acquired = self._gen_lock.acquire(timeout = 5.0)
if not acquired:
@@ -767,7 +800,7 @@ class InferenceOrchestrator:
if acquired:
self._gen_lock.release()
with self._dispatcher_lifecycle_lock:
- self._exclusive_vram_probe_pending = False
+ self._worker_reserved_for = None
def _shutdown_subprocess(self, timeout: float = 10.0) -> bool:
with self._subprocess_shutdown_lock:
@@ -787,6 +820,7 @@ class InferenceOrchestrator:
self._proc = None
return True
+ self._teardown_going_out()
self._cancel_generation()
time.sleep(0.5)
@@ -795,7 +829,7 @@ class InferenceOrchestrator:
try:
self._cmd_queue.put({"type": "shutdown"})
except (OSError, ValueError):
- pass
+ self._teardown_not_sent()
try:
self._proc.join(timeout = timeout)
@@ -841,6 +875,8 @@ class InferenceOrchestrator:
self._resp_queue = None
self._cancel_event = None
self._drain_event = None
+ self._stop_ledger = None
+ self._pending_teardowns = None
self._reset_worker_scoped_state()
logger.info("Inference subprocess shut down")
return True
@@ -925,11 +961,7 @@ class InferenceOrchestrator:
return not expected_free_gb
def _reset_worker_scoped_state(self) -> None:
- """Drop bookkeeping that only means anything for the worker that just died. Ownership is
- scoped by cancel-event identity alone, so a consumer still blocked on its mailbox when
- the process was replaced stayed recorded as the executor, and a generation on the fresh
- worker then failed _owns_worker and could not be stopped. Mailboxes go too: nothing will
- ever route to them, and a stale one reads as compare activity to the unload path."""
+ """Drop bookkeeping that only means anything for the worker that just died."""
with self._active_cancel_lock:
self._active_cancel_events.clear()
self._executing_cancel_events.clear()
@@ -1014,6 +1046,14 @@ class InferenceOrchestrator:
except Exception:
return ""
+ def _teardown_going_out(self) -> None:
+ if self._pending_teardowns is not None:
+ self._pending_teardowns.sending()
+
+ def _teardown_not_sent(self) -> None:
+ if self._pending_teardowns is not None:
+ self._pending_teardowns.unsent()
+
def _ensure_subprocess_alive(self) -> bool:
return self._proc is not None and self._proc.is_alive()
@@ -1207,7 +1247,11 @@ class InferenceOrchestrator:
except (EOFError, OSError, ValueError):
return events
- def _direct_reader(self, request_id: str):
+ def _direct_reader(
+ self,
+ request_id: str,
+ cancel_event = None,
+ ):
"""Response reader for a _gen_lock generation, safe once compare exists.
The dispatcher and this reader would otherwise both consume _resp_queue. A dispatcher
@@ -1221,6 +1265,8 @@ class InferenceOrchestrator:
mailbox = _WorkerMailbox(self._proc)
with self._mailbox_lock:
self._direct_mailboxes[request_id] = mailbox
+ if cancel_event is not None:
+ self._request_cancel_events[request_id] = cancel_event
def read_one(timeout: float = 1.0):
try:
@@ -1276,6 +1322,8 @@ class InferenceOrchestrator:
def release() -> None:
with self._mailbox_lock:
self._direct_mailboxes.pop(request_id, None)
+ if cancel_event is not None:
+ self._request_cancel_events.pop(request_id, None)
return read_one, drain, release
@@ -1321,6 +1369,7 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ rows: Optional[list] = None,
video_b64: Optional[str] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
@@ -1343,6 +1392,7 @@ class InferenceOrchestrator:
"presence_penalty": presence_penalty,
"frequency_penalty": frequency_penalty,
"logit_bias": logit_bias,
+ "parallel_slots": self.effective_parallel_slots,
}
if seed is not None:
cmd["seed"] = seed
@@ -1365,6 +1415,8 @@ class InferenceOrchestrator:
cmd["preserve_thinking"] = preserve_thinking
if continue_final_message:
cmd["continue_final_message"] = True
+ if rows:
+ cmd["rows"] = rows
if tool_protocol_active is not None:
cmd["tool_protocol_active"] = tool_protocol_active
return cmd
@@ -1375,29 +1427,53 @@ class InferenceOrchestrator:
drain_on_cancel,
*,
crash_context: str,
+ request_id: str = "",
cancel_event = None,
stats_holder: Optional[dict] = None,
read_timeout: float = 30.0,
mark_started: bool = True,
- ) -> Generator[str, None, None]:
- """Yield tokens from a response stream until gen_done/gen_error. ``read_one(timeout)``
- returns the next response (or None on timeout) and owns the queue choice -- the shared
- resp_queue under _gen_lock, or a per-request mailbox on the dispatcher path -- so this
- loop stays agnostic of which queue is read. On cancel, ``drain_on_cancel()`` consumes the
- cancel ack from that same source so stale events don't leak into the next request."""
+ rows: Optional[int] = None,
+ ) -> Generator[Any, None, None]:
+ """Yield tokens from a response stream until gen_done/gen_error."""
# Latch this stream's subprocess/queue: if a wedged worker is torn down and a later load spawns a fresh one,
# bail rather than re-block on the new queue under _gen_lock (deadlock).
initial_proc = self._proc
initial_resp_queue = self._resp_queue
+ reading_on_until = None
+ stop_recorded = False
+ stop_sent = False
while True:
if self._proc is not initial_proc or self._resp_queue is not initial_resp_queue:
+ if stop_sent:
+ return
# No tail here whatever the lists say: the capture is the REPLACEMENT's.
detail = self._subprocess_crash_message(crash_context)
yield GenStreamError(f"Error: {detail}", public = True)
return
- resp = read_one(read_timeout)
+ if not stop_sent and cancel_event is not None:
+ if cancel_event.is_set():
+ stop_recorded, stop_sent = self._stop_and_signal(
+ request_id,
+ cancel_event,
+ stop_recorded,
+ may_signal = False,
+ )
+ if stop_sent and rows is not None and reading_on_until is None:
+ reading_on_until = time.monotonic() + _CANCELLED_ROWS_GRACE
+ timeout = read_timeout
+ if not stop_recorded and cancel_event is not None:
+ timeout = min(timeout, _STOP_NOTICE_INTERVAL)
+ if reading_on_until is not None:
+ remaining = reading_on_until - time.monotonic()
+ if remaining <= 0:
+ drain_on_cancel()
+ return
+ timeout = min(timeout, remaining)
+ resp = read_one(timeout)
if resp is None:
if not self._ensure_subprocess_alive():
+ if stop_sent:
+ return
# Only the request the worker was RUNNING gets its last words; the rest were queued behind it.
detail = self._subprocess_crash_message(
crash_context, with_worker_output = self._owns_worker(cancel_event)
@@ -1409,32 +1485,52 @@ class InferenceOrchestrator:
rtype = resp.get("type", "")
if rtype == "status":
continue
- # The worker is answering THIS request, so it is the one executing: only now may its cancel event speak
- # for the shared worker one. The dispatched path opts out: its dispatcher already did this in worker
- # order, which a mailbox read can lag behind.
- if mark_started:
+ if mark_started and not stop_sent:
self._mark_worker_started(cancel_event)
# Subprocess-level error (no request_id); request-scoped failures arrive as gen_error below
if rtype == "error" and not resp.get("request_id"):
+ if stop_sent:
+ return
yield GenStreamError(f"Error: {resp.get('error', 'Unknown error')}")
return
- if rtype == "token":
- if cancel_event is not None and cancel_event.is_set():
- # Same rule as reset_generation_state: the shared worker event may only be set by the generation
- # the worker is running. A dispatched request can still be draining stale mailbox tokens after the
- # dispatcher retired it, and signalling from here would end the next one instead. Tearing this
- # stream down is always safe, so the local drain happens either way.
- if self._owns_worker(cancel_event):
- self._cancel_generation()
+ if rtype == "token" and cancel_event is not None and cancel_event.is_set():
+ if not stop_sent:
+ stop_recorded, stop_sent = self._stop_and_signal(
+ request_id,
+ cancel_event,
+ stop_recorded,
+ )
+ if stop_sent and rows is not None and reading_on_until is None:
+ reading_on_until = time.monotonic() + _CANCELLED_ROWS_GRACE
+ if rows is None:
drain_on_cancel()
return
- yield resp.get("text", "")
- elif rtype == "gen_done":
+ # Several replies read on for their completions, but a token drawn
+ # after the Stop is one the same reply alone would never have shown.
+ continue
+
+ if rtype == "row_done" and rows is not None:
if stats_holder is not None:
+ reported = stats_holder.setdefault("stats", [None] * rows)
+ row = int(resp.get("row", 0))
+ if 0 <= row < len(reported):
+ reported[row] = resp.get("stats")
+ yield int(resp.get("row", 0)), None
+ elif rtype == "token":
+ if rows is not None:
+ yield int(resp.get("row", 0)), resp.get("text", "")
+ else:
+ yield resp.get("text", "")
+ elif rtype == "gen_done":
+ if stats_holder is not None and rows is None:
stats_holder["stats"] = resp.get("stats")
+ self._forget_request(request_id, cancel_event)
return
elif rtype == "gen_error":
+ self._forget_request(request_id, cancel_event)
+ if stop_sent:
+ return
_budget = resp.get("context_budget")
if _budget:
# Rebuilt rather than yielded as text: the route arms match on the type.
@@ -1448,28 +1544,13 @@ class InferenceOrchestrator:
)
return
- def _start_dispatcher(self) -> bool:
- """Start the dispatcher thread if not already running. The dispatcher reads the shared
- resp_queue and routes responses to per-request mailbox queues, letting multiple
- adapter-controlled (compare) requests be in-flight without holding _gen_lock. The whole
- check-then-spawn runs under _dispatcher_lifecycle_lock so concurrent compare requests
- (which bypass _gen_lock) can't both observe no live dispatcher and each spawn one.
- Returns True only for the caller that actually started a new thread; False if one was
- already alive."""
+ def _start_dispatcher(self) -> Optional[threading.Thread]:
+ """Start the dispatcher thread if not already running."""
with self._dispatcher_lifecycle_lock:
- # Refuse to start while an unload is in progress. unload_model sets _unload_pending under this same lock
- # before it stops the idle dispatcher, so a start queued behind that stop observes the unload here and
- # bails. Without this a fresh dispatcher would be spawned after the stop, become the resp_queue reader,
- # and consume the worker's "unloaded" reply (unroutable, so dropped) before unload_model's _wait_response
- # sees it -- hanging the unload 300s.
- if (
- self._unload_pending
- or self._exclusive_tts_pending
- or getattr(self, "_exclusive_vram_probe_pending", False)
- ):
- return False
+ if self._unload_pending or self._worker_reserved_for:
+ return None
if self._dispatcher_thread is not None and self._dispatcher_thread.is_alive():
- return False
+ return None
self._dispatcher_stop.clear()
self._dispatcher_thread = threading.Thread(
@@ -1479,16 +1560,15 @@ class InferenceOrchestrator:
)
self._dispatcher_thread.start()
logger.debug("Dispatcher thread started")
- return True
+ return self._dispatcher_thread
- def _stop_dispatcher(self) -> None:
- """Signal the dispatcher to stop and wait for it. Runs under _dispatcher_lifecycle_lock
- (paired with _start_dispatcher) so a stop can't interleave with a concurrent start.
- Callers must NOT hold _mailbox_lock here: this joins the dispatcher, and the dispatcher
- loop takes _mailbox_lock, so holding it would deadlock the join."""
+ def _stop_dispatcher(self, thread: Optional[threading.Thread] = None) -> None:
+ """Signal the dispatcher to stop and wait for it."""
with self._dispatcher_lifecycle_lock:
if self._dispatcher_thread is None:
return
+ if thread is not None and self._dispatcher_thread is not thread:
+ return
self._dispatcher_stop.set()
self._dispatcher_thread.join(timeout = _DISPATCH_STOP_TIMEOUT)
self._dispatcher_thread = None
@@ -1575,10 +1655,12 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ rows: Optional[list] = None,
+ expected_model: Optional[str] = None,
video: Optional[str] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
- ) -> Generator[str, None, None]:
+ ) -> Generator[Any, None, None]:
"""Dispatched generation, sending the command without holding _gen_lock. Uses a per-request
mailbox for tokens so two compare-mode requests can be queued at once. The subprocess
still runs commands sequentially, so GPU work stays serialized; this only avoids
@@ -1592,25 +1674,14 @@ class InferenceOrchestrator:
return
# Latch the target model so the recheck below can detect a switch that completed between _start_dispatcher and
# mailbox registration (mirrors the locked path's expected_model check).
- expected_model = self.active_model_name
+ if expected_model is None:
+ expected_model = self.active_model_name
# Switch in flight (unload waiting on _gen_lock). This path bypasses the lock, so without this early-out a
# compare request would enqueue a generate on the outgoing model and delay the switch.
if self._unload_pending:
yield GenStreamError("Error: model is being unloaded", public = True)
return
- if self._exclusive_tts_pending:
- yield GenStreamError("Error: audio generation is in progress", public = True)
- return
-
- # Ensure the dispatcher runs. _start_dispatcher serializes concurrent starters under
- # _dispatcher_lifecycle_lock and returns True only for the caller that spawned the thread, so at most one
- # dispatcher ever exists even when two compare requests race here. Derive dispatcher_preexisting from that
- # atomic result (not a separate unlocked is_alive() read): if THIS call started the dispatcher and then bails
- # on a racing unload, it must stop it again (see the unloading bail below).
- started = self._start_dispatcher()
- dispatcher_preexisting = not started
-
request_id = str(uuid.uuid4())
image_b64 = self._pil_to_base64(image) if image is not None else None
@@ -1643,90 +1714,89 @@ class InferenceOrchestrator:
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
seed = seed,
+ rows = rows,
video_b64 = video,
)
mailbox = _WorkerMailbox(self._proc)
- with self._mailbox_lock:
- # _unload_pending alone is not enough: an unload that ran fully since _start_dispatcher clears it in its
- # finally and stops the dispatcher, so it reads False here though the dispatcher is gone and the model
- # swapped. Also bail when the active model changed or the dispatcher died: a mailbox with no dispatcher to
- # route gen_done/gen_error hangs the compare stream.
- dispatcher_alive = (
- self._dispatcher_thread is not None and self._dispatcher_thread.is_alive()
- )
- unloading = (
- self._unload_pending
- or self.active_model_name != expected_model
- or not dispatcher_alive
- )
- tts_reserved = self._exclusive_tts_pending
- probe_reserved = getattr(self, "_exclusive_vram_probe_pending", False)
- blocked = unloading or tts_reserved or probe_reserved
+ admission_deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT
+ while True:
+ self._wait_while_worker_held(cancel_event, deadline = admission_deadline)
+ if cancel_event is not None and cancel_event.is_set():
+ return
+
+ started = self._start_dispatcher()
+
+ with self._mailbox_lock:
+ dispatcher_alive = (
+ self._dispatcher_thread is not None
+ and self._dispatcher_thread.is_alive()
+ and not self._dispatcher_stop.is_set()
+ )
+ unloading = self._unload_pending or self.active_model_name != expected_model
+ reserved_for = self._worker_reserved_for
+ blocked = unloading or bool(reserved_for) or not dispatcher_alive
+ if not blocked:
+ self._mailboxes[request_id] = mailbox
+ if cancel_event is not None:
+ self._request_cancel_events[request_id] = cancel_event
+ orphaned_dispatcher = unloading and started is not None and not self._mailboxes
+ if orphaned_dispatcher:
+ self._dispatcher_stop.set()
if not blocked:
- self._mailboxes[request_id] = mailbox
- if cancel_event is not None:
- self._request_cancel_events[request_id] = cancel_event
- # When bailing without a mailbox, note whether any OTHER compare request still routes through the
- # dispatcher; if none and this call started it, stop it below.
- orphaned_dispatcher = blocked and not dispatcher_preexisting and not self._mailboxes
- if blocked:
- # A racing unload can pass its _wait_dispatcher_idle() while the dispatcher was stopped, then set
- # _unload_pending. The one we just started would otherwise linger with no mailboxes, race unload_model's
- # _wait_response for the "unloaded" reply off resp_queue, and drop it as unroutable -- hanging the unload
- # 300s. Stop it here so the unload stays the sole resp_queue reader. Outside _mailbox_lock:
+ break
# _stop_dispatcher joins the dispatcher, which itself takes that lock.
if orphaned_dispatcher:
- self._stop_dispatcher()
- detail = (
- "Error: audio generation is in progress"
- if tts_reserved
- else "Error: model switch is checking GPU memory"
- if probe_reserved
- else "Error: model is being unloaded"
- )
+ self._stop_dispatcher(started)
+ if not unloading and time.monotonic() < admission_deadline:
+ time.sleep(_STOP_NOTICE_INTERVAL)
+ continue
+ if reserved_for:
+ detail = f"Error: {reserved_for}"
+ elif unloading:
+ detail = "Error: model is being unloaded"
+ else:
+ detail = "Error: nothing is routing replies right now; try again"
yield GenStreamError(detail, public = True)
return
- # Claim before sending, like the locked path: dispatched runs are concurrent by design, so without this a Stop
- # on one saw no owner and reset the worker, ending its siblings. Claim and enqueue under one lock, or two
- # dispatcher threads interleave and claim order stops matching the subprocess's command order, which
- # _owns_worker reads.
- try:
- with self._send_order_lock:
- self._claim_worker(cancel_event)
- self._send_cmd(cmd)
- except RuntimeError as exc:
- self._release_worker(cancel_event)
- with self._mailbox_lock:
- self._mailboxes.pop(request_id, None)
- self._request_cancel_events.pop(request_id, None)
- yield GenStreamError(f"Error: {exc}")
- return
-
def read_mailbox(timeout):
try:
return self._read_mailbox(mailbox, timeout)
except queue.Empty:
return None
+ send_error = None
try:
- yield from self._consume_token_stream(
- read_mailbox,
- lambda: self._drain_mailbox(mailbox, timeout = 5.0),
- crash_context = "generation",
- cancel_event = cancel_event,
- stats_holder = stats_holder,
- read_timeout = _DISPATCH_READ_TIMEOUT,
- mark_started = False,
- )
+ try:
+ with self._send_order_lock:
+ if cancel_event is not None and cancel_event.is_set():
+ return
+ self._claim_worker(cancel_event)
+ self._send_cmd(cmd)
+ except RuntimeError as exc:
+ send_error = exc
+
+ if send_error is None:
+ yield from self._consume_token_stream(
+ read_mailbox,
+ lambda: self._drain_mailbox(mailbox, timeout = 5.0),
+ crash_context = "generation",
+ request_id = request_id,
+ cancel_event = cancel_event,
+ stats_holder = stats_holder,
+ read_timeout = _DISPATCH_READ_TIMEOUT,
+ mark_started = False,
+ rows = len(rows) if rows else None,
+ )
finally:
- # Normally already retired by the dispatcher at gen_done; this covers streams that end without one
- # (cancel, disconnect, a dead subprocess).
+ # Covers streams that end without a gen_done (never sent, cancel, disconnect, dead worker).
self._release_worker(cancel_event)
with self._mailbox_lock:
self._mailboxes.pop(request_id, None)
self._request_cancel_events.pop(request_id, None)
+ if send_error is not None:
+ yield GenStreamError(f"Error: {send_error}")
def _drain_mailbox(
self,
@@ -1747,36 +1817,62 @@ class InferenceOrchestrator:
return
logger.warning("Timed out draining mailbox after cancel")
- def _wait_dispatcher_idle(self, cancel_event = None) -> bool:
- """Wait for all dispatched requests to complete, then stop dispatcher. Returns True if the
- dispatcher was stopped (all mailboxes drained, or no dispatcher was running), and False
- if it was left running because compare requests were still active after
- _DISPATCH_IDLE_TIMEOUT. Callers must reserve admission before using this as an exclusive
- handoff; TTS does so under _gen_lock, while older direct paths call it before the lock."""
- if self._dispatcher_thread is None or not self._dispatcher_thread.is_alive():
- return True
+ @contextlib.contextmanager
+ def _reserve_worker(self, reason: str):
+ """Hold the worker for one caller, answering everything else with ``reason``."""
+ with self._worker_released:
+ if self._worker_reserved_for is not None:
+ raise RuntimeError(f"The worker is already held: {self._worker_reserved_for}")
+ self._worker_reserved_for = reason
+ try:
+ yield
+ finally:
+ with self._worker_released:
+ self._worker_reserved_for = None
+ self._worker_released.notify_all()
+
+ def _wait_while_worker_held(
+ self,
+ cancel_event = None,
+ deadline = None,
+ ) -> None:
+ if deadline is None:
+ deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT
+ with self._worker_released:
+ while self._worker_reserved_for is not None:
+ if cancel_event is not None and cancel_event.is_set():
+ return
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ return
+ self._worker_released.wait(min(remaining, _STOP_NOTICE_INTERVAL))
+
+ def _replies_in_flight(self) -> int:
+ with self._mailbox_lock:
+ return len(self._mailboxes) + len(self._direct_mailboxes)
- # Wait for all mailboxes to be emptied (dispatched requests complete)
- deadline = time.monotonic() + _DISPATCH_IDLE_TIMEOUT
+ def _wait_worker_idle(
+ self,
+ cancel_event = None,
+ timeout: float = _DISPATCH_IDLE_TIMEOUT,
+ ) -> bool:
+ deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if cancel_event is not None and cancel_event.is_set():
return False
- with self._mailbox_lock:
- if not self._mailboxes:
- break
+ if not self._replies_in_flight():
+ break
time.sleep(0.1)
if cancel_event is not None and cancel_event.is_set():
return False
- # Only stop the dispatcher if all mailboxes drained
- with self._mailbox_lock:
- still_active = bool(self._mailboxes)
- if still_active:
+ remaining = self._replies_in_flight()
+ if remaining:
logger.warning(
- "Dispatcher still has %d active mailbox(es); "
- "leaving dispatcher running for compare requests",
- len(self._mailboxes),
+ "%d repl(ies) still in flight after the idle wait; leaving the dispatcher "
+ "running for them",
+ remaining,
)
return False
self._stop_dispatcher()
@@ -1791,54 +1887,53 @@ class InferenceOrchestrator:
if not self._ensure_subprocess_alive():
raise RuntimeError("Inference subprocess is not running")
- self._wait_dispatcher_idle()
- with self._mailbox_lock:
- if self._mailboxes:
- raise RuntimeError(
- "Cannot share distributed objects while compare requests are active"
- )
- request_id = str(uuid.uuid4())
- cmd = {
- "type": "share_object",
- "request_id": request_id,
- "object": obj,
- }
-
with self._gen_lock:
- self._send_cmd(cmd)
- deadline = None if timeout is None else time.monotonic() + timeout
- while deadline is None or time.monotonic() < deadline:
- remaining = 1.0 if deadline is None else max(0.1, deadline - time.monotonic())
- resp = self._read_resp(timeout = min(remaining, 1.0))
- if resp is None:
- if not self._ensure_subprocess_alive():
- raise RuntimeError(
- self._subprocess_crash_message(
- "sharing chat turn",
- with_worker_output = self._owns_worker(None),
+ with self._reserve_worker("a distributed share is in progress"):
+ if not self._wait_worker_idle():
+ raise RuntimeError(
+ "Cannot share distributed objects while a reply is still generating"
+ )
+ request_id = str(uuid.uuid4())
+ cmd = {
+ "type": "share_object",
+ "request_id": request_id,
+ "object": obj,
+ }
+
+ self._send_cmd(cmd)
+ deadline = None if timeout is None else time.monotonic() + timeout
+ while deadline is None or time.monotonic() < deadline:
+ remaining = 1.0 if deadline is None else max(0.1, deadline - time.monotonic())
+ resp = self._read_resp(timeout = min(remaining, 1.0))
+ if resp is None:
+ if not self._ensure_subprocess_alive():
+ raise RuntimeError(
+ self._subprocess_crash_message(
+ "sharing chat turn",
+ with_worker_output = self._owns_worker(None),
+ )
)
- )
- continue
+ continue
- rtype = resp.get("type", "")
- rid = resp.get("request_id")
- if rid and rid != request_id:
- logger.debug(
- "Skipping response for request_id=%s while sharing request_id=%s",
- rid,
- request_id,
- )
- continue
- if rtype == "shared":
- return resp.get("object")
- if rtype == "share_error":
- raise RuntimeError(resp.get("error", "Failed to share object"))
- if rtype == "error":
- raise RuntimeError(resp.get("error", "Subprocess error"))
- if rtype == "status":
- continue
+ rtype = resp.get("type", "")
+ rid = resp.get("request_id")
+ if rid and rid != request_id:
+ logger.debug(
+ "Skipping response for request_id=%s while sharing request_id=%s",
+ rid,
+ request_id,
+ )
+ continue
+ if rtype == "shared":
+ return resp.get("object")
+ if rtype == "share_error":
+ raise RuntimeError(resp.get("error", "Failed to share object"))
+ if rtype == "error":
+ raise RuntimeError(resp.get("error", "Subprocess error"))
+ if rtype == "status":
+ continue
- raise RuntimeError("Timeout waiting for distributed object share")
+ raise RuntimeError("Timeout waiting for distributed object share")
# Monotonic count of PUBLISHED loads; lets the install route detect a load (including a same-model reload) that
# completed while it waited on the gate. Bumped when the load result is published, not at load start: a start-time
@@ -1869,9 +1964,12 @@ class InferenceOrchestrator:
cache_environment: Optional[Mapping[str, str]] = None,
anonymous_hf_access: bool = False,
audio_codec_path: Optional[str] = None,
+ n_parallel: Optional[int] = None,
) -> bool:
- """Load a model for inference. Always spawns a fresh subprocess per load for a clean
- interpreter (no stale unsloth patches, torch.compile caches, or getsource failures)."""
+ """Load a model for inference."""
+ from core.inference.llama_server_args import clamp_parallel_slots
+
+ parallel_slots = clamp_parallel_slots(n_parallel)
from utils.transformers_version import needs_transformers_5
from utils.hf_xet_fallback import DownloadStallError
@@ -2115,6 +2213,10 @@ class InferenceOrchestrator:
self.models[self.active_model_name] = _mirrored_model_entry(
model_info, model_name
)
+ self.models[self.active_model_name]["parallel_slots"] = parallel_slots
+ self.models[self.active_model_name]["can_batch"] = model_info.get(
+ "can_batch"
+ )
# Lets the already-loaded shortcut tell a CPU request from the GPU
# model it would otherwise report as satisfied. Native audio only:
# marking anything else tells training a GPU model holds no VRAM.
@@ -2255,13 +2357,6 @@ class InferenceOrchestrator:
self.models.pop(model_name, None)
return True
- # The subprocess runs commands sequentially, so a bare unload queues behind a running generate (a 2-3 min
- # hang). Cancel first (via the mp.Event the worker polls each token), then take _gen_lock as sole resp_queue
- # reader (like GGUF). Set _unload_pending under _dispatcher_lifecycle_lock so it is ordered ahead of the
- # dispatcher stop that _wait_dispatcher_idle runs under the same lock: a compare request's _start_dispatcher
- # queued behind that stop then observes the unload and refuses to spawn a fresh dispatcher that would eat the
- # "unloaded" reply off resp_queue. This is a standalone acquisition (no _gen_lock held yet), so it keeps the
- # _gen_lock -> _dispatcher_lifecycle_lock order and can't deadlock.
with self._dispatcher_lifecycle_lock:
self._unload_pending = True
# Cancelling only the running generation isn't enough: the worker clears cancel_event at each generate start,
@@ -2270,7 +2365,7 @@ class InferenceOrchestrator:
if self._drain_event is not None:
self._drain_event.set()
try:
- self._cancel_generation()
+ self._cancel_every_generation()
acquired = self._gen_lock.acquire(timeout = _UNLOAD_GEN_LOCK_TIMEOUT)
if not acquired:
logger.warning(
@@ -2285,11 +2380,7 @@ class InferenceOrchestrator:
return True
try:
- # Stop the compare-mode dispatcher so it can't consume the "unloaded" reply off resp_queue before we
- # do. A dispatched generation bypasses _gen_lock, so a wedged one slips past the acquire above; if the
- # dispatcher is still active it owns resp_queue and the queued unload hangs _wait_response behind the
- # stuck generate.
- if not self._wait_dispatcher_idle():
+ if not self._wait_worker_idle(timeout = _DISPATCH_IDLE_TIMEOUT):
logger.warning(
"Unload: compare-mode dispatcher still active after idle "
"wait; shutting the inference subprocess down to free the model"
@@ -2300,12 +2391,17 @@ class InferenceOrchestrator:
self.active_model_name = None
return True
self._drain_queue()
- self._send_cmd(
- {
- "type": "unload",
- "model_name": model_name,
- }
- )
+ self._teardown_going_out()
+ try:
+ self._send_cmd(
+ {
+ "type": "unload",
+ "model_name": model_name,
+ }
+ )
+ except Exception:
+ self._teardown_not_sent()
+ raise
self._wait_response("unloaded")
self.models.pop(model_name, None)
@@ -2354,6 +2450,11 @@ class InferenceOrchestrator:
nowhere."""
if not self._gen_lock.acquire(blocking = False):
raise RuntimeError("Cannot count tokens while a generation is in progress")
+ with self._mailbox_lock:
+ dispatched = bool(self._mailboxes)
+ if dispatched:
+ self._gen_lock.release()
+ raise RuntimeError("Cannot count tokens while a generation is in progress")
request_id = str(uuid.uuid4())
read_one, _drain, release_mailbox = self._direct_reader(request_id)
try:
@@ -2466,6 +2567,70 @@ class InferenceOrchestrator:
video = video,
)
+ def generate_chat_batch(
+ self,
+ rows: list,
+ messages: list,
+ system_prompt: str = "",
+ image = None,
+ images: Optional[list] = None,
+ image_ordinal: Optional[int] = None,
+ temperature: float = 0.7,
+ top_p: float = 0.9,
+ top_k: int = 40,
+ min_p: float = 0.0,
+ max_new_tokens: int = 256,
+ repetition_penalty: float = 1.0,
+ cancel_event = None,
+ tools: Optional[list] = None,
+ enable_thinking: Optional[bool] = None,
+ reasoning_effort: Optional[str] = None,
+ preserve_thinking: Optional[bool] = None,
+ continue_final_message: bool = False,
+ tool_protocol_active: Optional[bool] = None,
+ stats_holder: Optional[dict] = None,
+ presence_penalty: float = 0.0,
+ seed: Optional[int] = None,
+ frequency_penalty: float = 0.0,
+ logit_bias: Optional[dict] = None,
+ stop: Optional[list] = None,
+ video: Optional[str] = None,
+ response_format: Optional[dict] = None,
+ reasoning_is_extracted: bool = False,
+ ) -> Generator[Any, None, None]:
+ """Ask for several replies to one prompt in a single command."""
+ yield from self._generate_inner(
+ messages = messages,
+ system_prompt = system_prompt,
+ image = image,
+ images = images,
+ image_ordinal = image_ordinal,
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ min_p = min_p,
+ max_new_tokens = max_new_tokens,
+ repetition_penalty = repetition_penalty,
+ cancel_event = cancel_event,
+ use_adapter = None,
+ tools = tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ tool_protocol_active = tool_protocol_active,
+ stats_holder = stats_holder,
+ presence_penalty = presence_penalty,
+ seed = seed,
+ frequency_penalty = frequency_penalty,
+ logit_bias = logit_bias,
+ stop = stop,
+ rows = rows,
+ video = video,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
+ )
+
def generate_chat_completion_with_tools(
self,
messages: list,
@@ -2700,10 +2865,11 @@ class InferenceOrchestrator:
frequency_penalty: float = 0.0,
logit_bias: Optional[dict] = None,
stop: Optional[list] = None,
+ rows: Optional[list] = None,
video: Optional[str] = None,
response_format: Optional[dict] = None,
reasoning_is_extracted: bool = False,
- ) -> Generator[str, None, None]:
+ ) -> Generator[Any, None, None]:
"""Inner generation logic: sends the command to the subprocess and yields tokens. Serialized
by _gen_lock (one generation at a time) so concurrent readers don't consume each other's
tokens off the shared resp_queue."""
@@ -2716,10 +2882,41 @@ class InferenceOrchestrator:
return
expected_model = self.active_model_name
- self._wait_dispatcher_idle()
+ if self._stop_ledger is not None and self._stop_ledger.read_by_worker():
+ yield from self._generate_dispatched(
+ messages = messages,
+ system_prompt = system_prompt,
+ image = image,
+ images = images,
+ image_ordinal = image_ordinal,
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ min_p = min_p,
+ max_new_tokens = max_new_tokens,
+ repetition_penalty = repetition_penalty,
+ cancel_event = cancel_event,
+ use_adapter = use_adapter,
+ tools = tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = continue_final_message,
+ tool_protocol_active = tool_protocol_active,
+ stats_holder = stats_holder,
+ presence_penalty = presence_penalty,
+ seed = seed,
+ frequency_penalty = frequency_penalty,
+ logit_bias = logit_bias,
+ stop = stop,
+ rows = rows,
+ expected_model = expected_model,
+ video = video,
+ response_format = response_format,
+ reasoning_is_extracted = reasoning_is_extracted,
+ )
+ return
- # Serialize generation: two concurrent readers on resp_queue would consume and drop each other's token events.
- # Hold _gen_lock across the cmd build + send + whole stream so we stay the sole resp_queue reader.
with self._gen_lock:
if self._unload_pending or self.active_model_name != expected_model:
yield GenStreamError("Error: model is being unloaded", public = True)
@@ -2756,14 +2953,11 @@ class InferenceOrchestrator:
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
seed = seed,
+ rows = rows,
video_b64 = video,
)
- # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the lock above, having
- # generated nothing -- cannot reset the generation this is starting. Claiming after the send left the
- # command running unclaimed. Released in the finally. Own mailbox: a compare request can start the
- # dispatcher while this is streaming, and it would otherwise consume our responses and drop them.
- read_one, drain, release_mailbox = self._direct_reader(request_id)
+ read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)
try:
try:
with self._send_order_lock:
@@ -2777,29 +2971,26 @@ class InferenceOrchestrator:
read_one,
lambda: drain(timeout = 5.0),
crash_context = "generation",
+ request_id = request_id,
cancel_event = cancel_event,
stats_holder = stats_holder,
+ rows = len(rows) if rows else None,
)
finally:
self._release_worker(cancel_event)
release_mailbox()
def _claim_worker(self, cancel_event) -> None:
- """Record this request as one the worker will run. Admission only: the subprocess executes
- generations one at a time, so a dispatched request sitting behind another in the command
- queue is claimed but not executing, and must not be able to signal the shared cancel
- event (that would end whichever request IS executing). _mark_worker_started promotes it
- once the worker answers it."""
+ """Record this request as one the worker will run."""
with self._active_cancel_lock:
self._active_cancel_events.append(cancel_event)
def _mark_worker_started(self, cancel_event) -> None:
- """Promote a claimed request to executing, on its first worker response. Sole executor: the
- subprocess runs one generation at a time, so answering this one means it has left the
- previous one behind."""
if cancel_event is None:
return
with self._active_cancel_lock:
+ if not any(ev is cancel_event for ev in self._active_cancel_events):
+ return
if self._executing_cancel_events[:1] != [cancel_event]:
self._executing_cancel_events[:] = [cancel_event]
@@ -2826,23 +3017,75 @@ class InferenceOrchestrator:
# oldest claim is the executor; anyone else here is queued behind it.
return self._active_cancel_events[0] is cancel_event
+ def _stop_and_signal(
+ self,
+ request_id: str,
+ cancel_event,
+ recorded: bool = False,
+ *,
+ may_signal: bool = True,
+ ) -> tuple[bool, bool]:
+ """End one request: by name where the worker reads stops, else via the shared event.
+ Reports (recorded, sent)."""
+ ledger = self._stop_ledger
+ sent = False
+ if ledger is not None:
+ if not recorded:
+ recorded = bool(
+ request_id and self._ensure_subprocess_alive() and ledger.stop(request_id)
+ )
+ sent = bool(recorded and ledger.read_by_worker())
+ if not sent and may_signal:
+ with self._active_cancel_lock:
+ answered = any(ev is cancel_event for ev in self._executing_cancel_events)
+ if answered:
+ self._cancel_generation()
+ sent = True
+ if sent:
+ self._release_worker(cancel_event)
+ return recorded, sent
+
+ def _forget_request(self, request_id: str, cancel_event) -> None:
+ """Retire a reply as it ends: a cancel event outliving its request stops a later one."""
+ if request_id:
+ with self._mailbox_lock:
+ self._request_cancel_events.pop(request_id, None)
+ self._release_worker(cancel_event)
+
+ def _request_of(self, cancel_event) -> Optional[str]:
+ if cancel_event is None:
+ return None
+ with self._mailbox_lock:
+ for request_id, event in self._request_cancel_events.items():
+ if event is cancel_event:
+ return request_id
+ return None
+
def reset_generation_state(self, caller_cancel_event = None):
- """Cancel any ongoing generation and reset state. ``caller_cancel_event`` scopes the reset
- to one request. The worker has a single cancel event and generation is serialized on
- _gen_lock, so a chat that is still queued has no generation of its own to reset: calling
- this from its Stop handler would kill whichever chat currently holds the lock. Pass the
- request's own event and the reset is dropped unless that request is the one running. Omit
- it for genuinely global resets (unload, switch)."""
- if caller_cancel_event is not None and not self._owns_worker(caller_cancel_event):
+ """Cancel any ongoing generation and reset state."""
+ if caller_cancel_event is not None:
+ request_id = self._request_of(caller_cancel_event)
+ if request_id is not None:
+ self._stop_and_signal(request_id, caller_cancel_event)
+ return
+ with self._send_order_lock:
+ if not self._owns_worker(caller_cancel_event):
+ return
+ self._reset_worker()
return
+ self._reset_worker()
+
+ def _reset_worker(self) -> None:
+ self._teardown_going_out()
self._cancel_generation()
if not self._ensure_subprocess_alive():
+ self._teardown_not_sent()
return
try:
with self._send_order_lock:
self._send_cmd({"type": "reset"})
except RuntimeError:
- pass
+ self._teardown_not_sent()
def generate_audio_response(
self,
@@ -2871,15 +3114,13 @@ class InferenceOrchestrator:
# idle wait is racy: a compare request can register between the wait and this command, leaving TTS queued
# without safe ownership of the worker's single shared cancel event.
with self._gen_lock:
- with self._dispatcher_lifecycle_lock:
- self._exclusive_tts_pending = True
- try:
- dispatcher_idle = self._wait_dispatcher_idle(cancel_event = cancel_event)
+ with self._reserve_worker("audio generation is in progress"):
+ idle = self._wait_worker_idle(cancel_event = cancel_event)
if cancel_event is not None and cancel_event.is_set():
raise AudioGenerationCancelledError("Audio generation cancelled")
- if not dispatcher_idle:
+ if not idle:
raise RuntimeError(
- "Cannot start audio generation while compare requests are active"
+ "Cannot start audio generation while a reply is still generating"
)
# Recheck after the dispatcher wait: unload can set its flag without _gen_lock, and a switch may have
@@ -2931,7 +3172,7 @@ class InferenceOrchestrator:
cmd["seed"] = int(seed)
# Same shared-queue hazard as _generate_inner: see _direct_reader.
- read_one, _drain, release_mailbox = self._direct_reader(request_id)
+ read_one, _drain, release_mailbox = self._direct_reader(request_id, cancel_event)
try:
# Claim before enqueueing so request-scoped reset ownership follows the same discipline as text
# and audio-input generation
@@ -3036,9 +3277,6 @@ class InferenceOrchestrator:
finally:
self._release_worker(cancel_event)
release_mailbox()
- finally:
- with self._dispatcher_lifecycle_lock:
- self._exclusive_tts_pending = False
def generate_whisper_response(
self,
@@ -3152,7 +3390,7 @@ class InferenceOrchestrator:
if stop:
cmd["stop"] = stop
- read_one, drain, release_mailbox = self._direct_reader(request_id)
+ read_one, drain, release_mailbox = self._direct_reader(request_id, cancel_event)
try:
try:
# Claim under the send lock, like _generate_inner: unclaimed, a compare request queued behind this
@@ -3168,6 +3406,7 @@ class InferenceOrchestrator:
read_one,
lambda: drain(timeout = 5.0),
crash_context = "audio input generation",
+ request_id = request_id,
cancel_event = cancel_event,
stats_holder = stats_holder,
)
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 502ffc914f4..705c54e631c 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -23,7 +23,7 @@ import time
import traceback
from io import BytesIO
from pathlib import Path
-from typing import Any
+from typing import Any, Optional
logger = get_logger(__name__)
from core.inference.audio_errors import AUDIO_UNSUPPORTED_CODE
@@ -33,6 +33,110 @@ from utils.hardware import apply_gpu_ids, is_apple_silicon
# Fresh spawned interpreter: re-apply the OS-trust-store injection.
from utils.native_tls import activate_native_tls
+
+_ID_BYTES = 36
+
+_SLOTS = 256
+
+
+class StopLedger:
+ """The request ids the parent has stopped, and whether the worker reads them."""
+
+ def __init__(self, ctx: Any):
+ self._lock = ctx.Lock()
+ self._slots = ctx.Array("c", _SLOTS * _ID_BYTES, lock = False)
+ self._read_by_worker = ctx.Value("b", 0, lock = False)
+ self._written = ctx.Value("l", 0, lock = False)
+
+ def worker_reads_this(self) -> None:
+ """Said once by the worker, as it enters the loop that reads names from here."""
+ self._read_by_worker.value = 1
+
+ def read_by_worker(self) -> bool:
+ return bool(self._read_by_worker.value)
+
+ def stop(self, request_id: str) -> bool:
+ """Record a stop. False for an id no slot could hold."""
+ entry = _entry(request_id)
+ if entry is None:
+ return False
+ with self._lock:
+ if not self._holds(entry):
+ start = (self._written.value % _SLOTS) * _ID_BYTES
+ self._slots[start : start + _ID_BYTES] = entry
+ self._written.value += 1
+ return True
+
+ def snapshot(self, since: int = -1) -> tuple[int, Optional[set]]:
+ written = self._written.value
+ if written == since:
+ return written, None
+ with self._lock:
+ written = self._written.value
+ raw = bytes(self._slots)
+ return written, {
+ raw[start : start + _ID_BYTES].rstrip(b"\0").decode("utf-8", "replace")
+ for start in range(0, min(written, _SLOTS) * _ID_BYTES, _ID_BYTES)
+ }
+
+ def _holds(self, entry: bytes) -> bool:
+ for slot in range(min(self._written.value, _SLOTS)):
+ start = slot * _ID_BYTES
+ if bytes(self._slots[start : start + _ID_BYTES]) == entry:
+ return True
+ return False
+
+
+def _entry(request_id: Optional[str]) -> Optional[bytes]:
+ """One slot's worth of bytes, or None for an id no slot could hold."""
+ if not request_id:
+ return None
+ entry = str(request_id).encode("utf-8", "replace")
+ if len(entry) > _ID_BYTES or b"\0" in entry:
+ return None
+ return entry.ljust(_ID_BYTES, b"\0")
+
+
+class PendingTeardowns:
+ """How many commands that end everything are on their way to the worker."""
+
+ def __init__(self, ctx: Any):
+ self._count = ctx.Value("l", 0)
+
+ def sending(self) -> None:
+ with self._count.get_lock():
+ self._count.value += 1
+
+ def unsent(self) -> None:
+ self._counted_off()
+
+ def taken(self) -> None:
+ self._counted_off()
+
+ def _counted_off(self) -> None:
+ """One fewer on its way, clamped: the two ends are different processes."""
+ with self._count.get_lock():
+ self._count.value = max(0, self._count.value - 1)
+
+ def any_in_flight(self) -> bool:
+ return self._count.value > 0
+
+
+class RowRefused(Exception):
+ """This batch will not take this reply, and is exactly as it was."""
+
+
+def narrow_load_reason(cmd: dict) -> Optional[str]:
+ """Why no batch of either kind may take this command, or None."""
+ width = int(cmd.get("parallel_slots") or 1)
+ if width <= 1:
+ return "this load decodes one reply at a time"
+ rows = len(cmd.get("rows") or [])
+ if rows > width:
+ return f"the load decodes {width} replies at once, and this asks for {rows}"
+ return None
+
+
activate_native_tls()
_SHARE_OBJECT_MAX_BYTES = 1 << 20
@@ -530,6 +634,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
"is_audio": getattr(mc, "is_audio", False),
"audio_type": getattr(mc, "audio_type", None),
"has_audio_input": getattr(mc, "has_audio_input", False),
+ "can_batch": _load_can_batch(backend),
}
_bm = getattr(backend, "models", {}) or {}
_entry = (
@@ -567,6 +672,7 @@ def _handle_load(backend, config: dict, resp_queue: Any) -> None:
{
k: _entry[k]
for k in (
+ "context_unbounded_when_batched",
"mlx_kv_bits",
"mlx_kv_bits_requested",
"mlx_kv_quant_eligibility",
@@ -659,6 +765,86 @@ def _drain_skip_generate(
return True
+def _abandon_held_commands(held: list, resp_queue: Any) -> None:
+ while held:
+ _abandon_one(held.pop(0), resp_queue)
+
+
+class _Stops:
+ def __init__(self, stop_ledger, resp_queue, batch, held: list):
+ self._ledger = stop_ledger
+ self._resp_queue = resp_queue
+ self._batch = batch
+ self._held = held
+ self._written = 0
+ self._stopped: set = set()
+
+ def answer(self) -> None:
+ self._refresh()
+ self._batch.drop_stopped(self)
+ for cmd in [held for held in self._held if held.get("request_id", "") in self]:
+ self._held.remove(cmd)
+ _abandon_one(cmd, self._resp_queue)
+
+ def _refresh(self) -> None:
+ if self._ledger is None:
+ return
+ self._written, stopped = self._ledger.snapshot(self._written)
+ if stopped is not None:
+ self._stopped = stopped
+
+ def __contains__(self, request_id) -> bool:
+ return request_id in self._stopped
+
+
+class _StopWhileItRuns:
+ def __init__(self, cancel_event, stops, request_id: str):
+ self._cancel_event = cancel_event
+ self._stops = stops
+ self._request_id = request_id
+
+ def is_set(self) -> bool:
+ if self._cancel_event is not None and self._cancel_event.is_set():
+ return True
+ self._stops.answer()
+ return self._request_id in self._stops
+
+
+_TEARDOWN_COMMANDS = frozenset({"cancel", "reset", "unload", "shutdown"})
+
+
+def _teardown_skip(cmd: dict, resp_queue: Any, pending_teardowns) -> bool:
+ if pending_teardowns is None or not pending_teardowns.any_in_flight():
+ return False
+ _abandon_one(cmd, resp_queue)
+ return True
+
+
+def _stopped_before_it_ran(cmd: dict, resp_queue: Any, stopped) -> bool:
+ stopped.answer()
+ if cmd.get("request_id", "") not in stopped:
+ return False
+ _abandon_one(cmd, resp_queue)
+ return True
+
+
+def _abandon_one(cmd: dict, resp_queue: Any) -> None:
+ logger.info(
+ "Abandoning held %s for request %s",
+ cmd.get("type", ""),
+ cmd.get("request_id", ""),
+ )
+ _send_response(
+ resp_queue,
+ {
+ "type": "gen_done",
+ "request_id": cmd.get("request_id", ""),
+ "cancelled": True,
+ "stats": None,
+ },
+ )
+
+
def _prepare_generate_audio(cmd, resp_queue: Any, cancel_event, drain_event) -> bool:
"""Clear stale cancellation and acknowledge when this TTS command owns the worker.
@@ -702,75 +888,82 @@ def _backend_declares(
return False
+def _dispatch_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
+ if cmd.get("rows"):
+ _handle_generate_rows(backend, cmd, resp_queue, cancel_event)
+ else:
+ _handle_generate(backend, cmd, resp_queue, cancel_event)
+
+
+def _generation_kwargs(backend, cmd: dict, cancel_event) -> dict:
+ image = None
+ image_b64 = cmd.get("image_base64")
+ if image_b64:
+ image = _resize_image(_decode_image(image_b64))
+ images = [
+ _resize_image(_decode_image(encoded))
+ for encoded in cmd.get("images_base64") or ()
+ if encoded
+ ]
+
+ gen_kwargs = {
+ "messages": cmd["messages"],
+ "system_prompt": cmd.get("system_prompt", ""),
+ "image": image,
+ "images": images,
+ "temperature": cmd.get("temperature", 0.7),
+ "top_p": cmd.get("top_p", 0.9),
+ "top_k": cmd.get("top_k", 40),
+ "min_p": cmd.get("min_p", 0.0),
+ "max_new_tokens": cmd.get("max_new_tokens", 256),
+ "repetition_penalty": cmd.get("repetition_penalty", 1.0),
+ "presence_penalty": cmd.get("presence_penalty", 0.0),
+ "cancel_event": cancel_event,
+ }
+
+ for opt_key in (
+ "tools",
+ "enable_thinking",
+ "reasoning_effort",
+ "preserve_thinking",
+ "continue_final_message",
+ ):
+ if opt_key in cmd:
+ gen_kwargs[opt_key] = cmd[opt_key]
+
+ if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):
+ gen_kwargs["image_ordinal"] = cmd["image_ordinal"]
+
+ for gated in (
+ "seed",
+ "frequency_penalty",
+ "logit_bias",
+ "stop",
+ "tool_protocol_active",
+ "response_format",
+ "reasoning_is_extracted",
+ ):
+ if gated in cmd and _backend_declares(backend, gated):
+ gen_kwargs[gated] = cmd[gated]
+ if cmd.get("video_base64"):
+ if not _backend_declares(backend, "video"):
+ raise RuntimeError("The loaded backend does not read video.")
+ gen_kwargs["video"] = cmd["video_base64"]
+
+ return gen_kwargs
+
+
def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
"""Handle a generate command: stream tokens back via resp_queue.
- cancel_event is an mp.Event the parent can set anytime (user stop, or new
- model load mid-generate); generation stops within 1-2 tokens.
+ cancel_event is asked between tokens; what it answers for depends on the caller --
+ the batching loop passes something reading both the record and the shared event, the
+ one-at-a-time loop the shared event alone. Nothing is asked during the prefill.
"""
request_id = cmd.get("request_id", "")
try:
- image = None
- image_b64 = cmd.get("image_base64")
- if image_b64:
- image = _resize_image(_decode_image(image_b64))
- images = [
- _resize_image(_decode_image(encoded))
- for encoded in cmd.get("images_base64") or ()
- if encoded
- ]
-
- gen_kwargs = {
- "messages": cmd["messages"],
- "system_prompt": cmd.get("system_prompt", ""),
- "image": image,
- "images": images,
- "temperature": cmd.get("temperature", 0.7),
- "top_p": cmd.get("top_p", 0.9),
- "top_k": cmd.get("top_k", 40),
- "min_p": cmd.get("min_p", 0.0),
- "max_new_tokens": cmd.get("max_new_tokens", 256),
- "repetition_penalty": cmd.get("repetition_penalty", 1.0),
- "presence_penalty": cmd.get("presence_penalty", 0.0),
- "cancel_event": cancel_event,
- }
-
- # Forward only present optional keys so the backend signature can evolve.
- for opt_key in (
- "tools",
- "enable_thinking",
- "reasoning_effort",
- "preserve_thinking",
- "continue_final_message",
- ):
- if opt_key in cmd:
- gen_kwargs[opt_key] = cmd[opt_key]
-
- if cmd.get("image_ordinal") is not None and _backend_declares(backend, "image_ordinal"):
- gen_kwargs["image_ordinal"] = cmd["image_ordinal"]
-
- # Not every backend declares these (transformers declares only ``stop``)
- # and none takes **kwargs, so forwarding unconditionally would turn a
- # backend's documented "ignores them" behavior into a TypeError.
- # ``tool_protocol_active`` rides here rather than above: MLX declares no such
- # parameter and takes no **kwargs, so an unconditional forward would raise.
- for gated in (
- "seed",
- "frequency_penalty",
- "logit_bias",
- "stop",
- "tool_protocol_active",
- "response_format",
- "reasoning_is_extracted",
- ):
- if gated in cmd and _backend_declares(backend, gated):
- gen_kwargs[gated] = cmd[gated]
- # A clip cannot be dropped like an unknown sampling knob: the answer would ignore it.
- if cmd.get("video_base64"):
- if not _backend_declares(backend, "video"):
- raise RuntimeError("The loaded backend does not read video.")
- gen_kwargs["video"] = cmd["video_base64"]
+ gen_kwargs = _generation_kwargs(backend, cmd, cancel_event)
use_adapter = cmd.get("use_adapter")
if use_adapter is not None:
@@ -785,7 +978,6 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
try:
for cumulative_text in generator:
- # cancel_event is an mp.Event - checked instantly, no queue polling.
if cancel_event.is_set():
logger.info("Generation cancelled for request %s", request_id)
break
@@ -854,18 +1046,360 @@ def _handle_count_tokens(backend, cmd: dict, resp_queue: Any) -> None:
)
-def _decline_count_tokens(cmd: dict, resp_queue: Any) -> None:
+def _decline_count_tokens(
+ cmd: dict,
+ resp_queue: Any,
+ reason: str = "Counting is not supported on the transformers backend.",
+) -> None:
"""Answer a count this backend cannot serve; dropping it costs the caller its timeout."""
_send_response(
resp_queue,
{
"type": "count_tokens_response",
"request_id": cmd.get("request_id"),
- "error": "Counting is not supported on the transformers backend.",
+ "error": reason,
},
)
+def _load_can_batch(backend) -> bool:
+ """Whether this load serves several replies at once at all."""
+ fixed = getattr(backend, "batch_unavailable_reason", None)
+ resident = getattr(backend, "resident_unavailable_reason", None)
+ return (callable(fixed) and fixed([{}, {}]) is None) or (
+ callable(resident) and resident({}) is None
+ )
+
+
+def _rows_apart(backend, requests, cancel_event, stats: list):
+ """A declined batch served reply by reply, as the batch's own (row, snapshot) events."""
+ for row, request in enumerate(requests):
+ if not cancel_event.is_set():
+ generator = backend.generate_chat_response(
+ **{
+ name: value
+ for name, value in request.items()
+ if _backend_declares(backend, name)
+ },
+ cancel_event = cancel_event,
+ )
+ try:
+ for cumulative_text in generator:
+ if cancel_event.is_set():
+ break
+ yield row, cumulative_text
+ finally:
+ close = getattr(generator, "close", None)
+ if callable(close):
+ close()
+ stats[row] = getattr(backend, "last_generation_stats", None)
+ yield row, None
+
+
+def _handle_generate_rows(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
+ request_id = cmd.get("request_id", "")
+ rows = cmd.get("rows") or []
+
+ try:
+ shared = _generation_kwargs(backend, cmd, cancel_event)
+ shared.pop("cancel_event", None)
+ requests = [{**shared, **row} for row in rows]
+
+ reason = narrow_load_reason(cmd)
+ if reason is None:
+ unavailable = getattr(backend, "batch_unavailable_reason", None)
+ reason = unavailable(requests) if callable(unavailable) else "backend cannot batch"
+ if reason is not None:
+ logger.info(
+ "Declining %d replies in one command for request_id=%s: %s",
+ len(requests),
+ request_id,
+ reason,
+ )
+ stats = [None] * len(requests)
+ events = _rows_apart(backend, requests, cancel_event, stats)
+ else:
+ logger.info(
+ "Starting batched generation for request_id=%s rows=%d",
+ request_id,
+ len(requests),
+ )
+ stats = None
+ events = backend.generate_chat_batch(requests, cancel_event = cancel_event)
+ try:
+ for row, snapshot in events:
+ if snapshot is None:
+ reported = backend.last_batch_generation_stats if stats is None else stats
+ message = {"type": "row_done", "stats": reported[row]}
+ else:
+ message = {"type": "token", "text": snapshot}
+ _send_response(resp_queue, {**message, "request_id": request_id, "row": row})
+ finally:
+ close = getattr(events, "close", None)
+ if callable(close):
+ close()
+
+ _send_response(
+ resp_queue,
+ {"type": "gen_done", "request_id": request_id, "stats": None},
+ )
+
+ except Exception as exc:
+ logger.error("Multi-reply generation error: %s", exc, exc_info = True)
+ _send_response(resp_queue, _generation_error_payload(request_id, exc))
+
+
+def _admitted_width(cmd: dict) -> int:
+ return max(1, int(cmd.get("parallel_slots") or 1))
+
+
+def _held_head_leaves_the_hold(batch: "_ResidentBatch", held: list) -> bool:
+ """Whether the head comes off the hold on this pass; the batch can still turn it away."""
+ if not held:
+ return False
+ if not batch.rows_in_flight:
+ return True
+ head = held[0]
+ return head.get("type") == "generate" and batch.unavailable_reason(head) is None
+
+
+class _ResidentBatch:
+ """The replies an MLX worker is decoding at once."""
+
+ def __init__(self, backend, resp_queue: Any):
+ self.backend = backend
+ self.resp_queue = resp_queue
+ self.session = None
+ self.width = None
+ self._owed: dict = {}
+ self._refused: set = set()
+
+ @property
+ def rows_in_flight(self) -> int:
+ return self.session.rows_in_flight if self.session is not None else 0
+
+ def _live(self, request_id: str) -> list:
+ if self.session is None:
+ return []
+ return [handle for handle in self.session.handles if handle[0] == request_id]
+
+ def unavailable_reason(self, cmd: dict) -> Optional[str]:
+ if cmd.get("request_id", "") in self._refused:
+ return "this batch has already refused these replies"
+ if cmd.get("use_adapter") is not None:
+ return "the reply asks for a particular adapter state"
+ reason = narrow_load_reason(cmd)
+ if reason is not None:
+ return reason
+ if self.width is not None and self.width != _admitted_width(cmd):
+ # A command at another width waits for the batch to drain: joining at the old
+ # one would refill the batch, so a narrowed load never reaches its new width.
+ return "the open batch is decoding at a different width"
+ if (
+ self.width is not None
+ and self.rows_in_flight + len(cmd.get("rows") or [None]) > self.width
+ ):
+ return "the open batch is full"
+ probe = getattr(self.backend, "resident_unavailable_reason", None)
+ if not callable(probe):
+ return "this backend has no batch a reply can join"
+ rows = cmd.get("rows") or []
+ for request in [{**cmd, **row} for row in rows] if rows else [cmd]:
+ reason = probe(request)
+ if reason is not None:
+ return reason
+ return None
+
+ def admit(self, cmd: dict, cancel_event) -> bool:
+ request_id = cmd.get("request_id", "")
+ rows = cmd.get("rows") or []
+ shared = _generation_kwargs(self.backend, cmd, cancel_event)
+ shared.pop("cancel_event", None)
+ requests = [{**shared, **row} for row in rows] if rows else [shared]
+
+ if self.session is None:
+ self.width = _admitted_width(cmd)
+ try:
+ self.session = self.backend.open_resident_batch(width = self.width)
+ except Exception as unopened:
+ logger.warning(
+ "No batch could be opened for request_id=%s, decoding it alone: %s",
+ request_id,
+ unopened,
+ )
+ self.width = None
+ return False
+ handles = [(request_id, row if rows else None) for row in range(len(requests))]
+ self._owed[request_id] = None
+ prefixes = []
+ try:
+ for request, handle in zip(requests, handles):
+ prefixes.append((handle, self.session.admit(request, handle)))
+ except RowRefused as refusal:
+ if not self._forget(request_id, withdraw = True):
+ self._fail_all(refusal)
+ raise
+ logger.info("Request_id=%s cannot join this batch: %s", request_id, refusal)
+ self._refused.add(request_id)
+ self._close_if_empty()
+ return False
+ except BaseException as exc:
+ if self._forget(request_id, withdraw = True):
+ self._close_if_empty()
+ else:
+ self._fail_all(exc)
+ raise
+ logger.info(
+ "Admitted request_id=%s (%d replies) to a batch of %d",
+ request_id,
+ len(requests),
+ self.rows_in_flight,
+ )
+ for handle, prefix in prefixes:
+ if prefix:
+ self._send_token(handle, prefix)
+ return True
+
+ def cancel(self, request_id: str) -> bool:
+ if request_id not in self._owed:
+ return False
+ live = self._live(request_id)
+ if not live:
+ return False
+ logger.info("Cancelling request_id=%s in a batch of %d", request_id, self.rows_in_flight)
+ try:
+ for handle, snapshot in self.session.withdraw(sorted(live, key = _handle_order)):
+ self._report(handle, snapshot)
+ except Exception as exc:
+ logger.error("Batched cancellation error: %s", exc, exc_info = True)
+ self._fail_all(exc)
+ return True
+ self._close_if_empty()
+ return True
+
+ def cancel_all(self) -> None:
+ for request_id in list(self._owed):
+ self.cancel(request_id)
+
+ def drop_stopped(self, stopped) -> None:
+ for request_id in list(self._owed):
+ if request_id in stopped:
+ self.cancel(request_id)
+
+ def step(self) -> None:
+ if self.session is None or not self.session.rows_in_flight:
+ return
+ try:
+ for handle, snapshot in self.session.step():
+ self._report(handle, snapshot)
+ except Exception as exc:
+ logger.error("Batched generation error: %s", exc, exc_info = True)
+ self._fail_all(exc)
+ return
+ self._close_if_empty()
+
+ def _fail_all(self, exc: BaseException) -> None:
+ """Give up on the batch: every request in it is owed an error, not silence."""
+ stack = traceback.format_exc(limit = 20)
+ for request_id in list(self._owed):
+ self._fail(request_id, exc, stack)
+ self.close()
+
+ def _close_if_empty(self) -> None:
+ if self.session is not None and not self.session.rows_in_flight:
+ self.close()
+
+ def close(self) -> None:
+ ending = (
+ {handle[0] for handle in self.session.ending} if self.session is not None else set()
+ )
+ for request_id in [r for r in self._owed if r in ending]:
+ self._end(request_id, cancelled = True)
+ session, self.session = self.session, None
+ self.width = None
+ self._owed.clear()
+ self._refused.clear()
+ if session is None:
+ return
+ try:
+ session.close()
+ except Exception:
+ logger.error("Could not close the resident batch", exc_info = True)
+
+ def _report(self, handle, snapshot) -> None:
+ request_id, row = handle
+ if request_id not in self._owed:
+ self._take_stats(handle)
+ return
+ if snapshot is not None:
+ self._send_token(handle, snapshot)
+ return
+ if row is not None:
+ _send_response(
+ self.resp_queue,
+ {
+ "type": "row_done",
+ "request_id": request_id,
+ "row": row,
+ "stats": self._take_stats(handle),
+ },
+ )
+ if not self._live(request_id):
+ self._end(request_id)
+
+ def _send_token(self, handle, text) -> None:
+ request_id, row = handle
+ event = {"type": "token", "request_id": request_id, "text": text}
+ if row is not None:
+ event["row"] = row
+ _send_response(self.resp_queue, event)
+
+ def _end(
+ self,
+ request_id: str,
+ *,
+ cancelled: bool = False,
+ ) -> None:
+ """Answer a request's gen_done; cancelled when it ends with its batch, not a reply."""
+ stats = self._take_stats((request_id, None))
+ self._forget(request_id, withdraw = False)
+ event = {"type": "gen_done", "request_id": request_id, "stats": stats}
+ if cancelled:
+ event["cancelled"] = True
+ _send_response(self.resp_queue, event)
+ logger.info("Ended request_id=%s (cancelled=%s)", request_id, cancelled)
+
+ def _fail(self, request_id: str, exc: BaseException, stack: str) -> None:
+ self._forget(request_id, withdraw = False)
+ _send_response(
+ self.resp_queue, {**_generation_error_payload(request_id, exc), "stack": stack}
+ )
+
+ def _forget(self, request_id: str, *, withdraw: bool) -> bool:
+ """Stop answering for a request. False where taking its rows back failed."""
+ live = self._live(request_id)
+ self._owed.pop(request_id, None)
+ taken_back = True
+ if withdraw and live:
+ try:
+ for _handle, _snapshot in self.session.withdraw(sorted(live, key = _handle_order)):
+ pass
+ except Exception:
+ logger.error("Could not withdraw rows for request_id=%s", request_id, exc_info = True)
+ taken_back = False
+ for handle in live:
+ self._take_stats(handle)
+ return taken_back
+
+ def _take_stats(self, handle):
+ return self.session.take_stats(handle) if self.session is not None else None
+
+
+def _handle_order(handle):
+ _request_id, row = handle
+ return -1 if row is None else row
+
+
def _handle_share_object(backend, cmd: dict, resp_queue: Any) -> None:
"""Share a small Python object across MLX distributed ranks."""
request_id = cmd.get("request_id", "")
@@ -1106,6 +1640,8 @@ def run_inference_process(
cancel_event,
config: dict,
drain_event = None,
+ stop_ledger = None,
+ pending_teardowns = None,
) -> None:
"""Subprocess entrypoint. Persistent — runs the command loop until shutdown.
@@ -1118,6 +1654,11 @@ def run_inference_process(
cancel_event (cleared at the start of every generate), it is never cleared
here, so a generate still queued behind a cancelled one is skipped rather
than run — the cancel survives the queue handoff.
+ stop_ledger: StopLedger in shared memory naming the requests the parent has
+ stopped. Read rather than received: a reply decoding beside others is stopped
+ by name, and a queue put is not readable the moment it returns.
+ pending_teardowns: counts the commands that end everything on their way here, so a
+ held command is answered rather than run in front of one. Read, for the same reason.
"""
# Apply request credentials before a Hugging Face import snapshots the environment.
_apply_worker_hf_token_environment(config)
@@ -1263,22 +1804,66 @@ def run_inference_process(
)
return
- # Enter the same command loop as the GPU path.
logger.info("MLX inference subprocess ready, entering command loop")
+ batch = _ResidentBatch(backend, resp_queue)
+ deferred: list[dict] = []
+ stops = _Stops(stop_ledger, resp_queue, batch, deferred)
+ if stop_ledger is not None:
+ stop_ledger.worker_reads_this()
while True:
- try:
- cmd = cmd_queue.get(timeout = 1.0)
- except _queue.Empty:
- continue
- except (EOFError, OSError):
- return
+ stops.answer()
+ tearing_down = pending_teardowns is not None and pending_teardowns.any_in_flight()
+ if not tearing_down:
+ batch.step()
+ from_deferred = False
+ if _held_head_leaves_the_hold(batch, deferred):
+ cmd = deferred.pop(0)
+ from_deferred = True
+ else:
+ try:
+ cmd = cmd_queue.get(
+ timeout = 0.0
+ if (batch.rows_in_flight or deferred) and not tearing_down
+ else 1.0
+ )
+ except _queue.Empty:
+ continue
+ except (EOFError, OSError):
+ batch.close()
+ return
if cmd is None:
continue
cmd_type = cmd.get("type", "")
+ if pending_teardowns is not None and cmd_type in _TEARDOWN_COMMANDS:
+ pending_teardowns.taken()
try:
if cmd_type == "generate":
if _drain_skip_generate(cmd, resp_queue, drain_event):
continue
+ if deferred and not from_deferred:
+ deferred.append(cmd)
+ continue
+ reason = batch.unavailable_reason(cmd)
+ if reason is None:
+ if _teardown_skip(cmd, resp_queue, pending_teardowns):
+ continue
+ if _stopped_before_it_ran(cmd, resp_queue, stops):
+ continue
+ if batch.admit(cmd, None):
+ continue
+ reason = "it does not prepare like the replies in the batch"
+ if batch.rows_in_flight:
+ logger.info(
+ "Holding request_id=%s until the batch takes it or drains: %s",
+ cmd.get("request_id", ""),
+ reason,
+ )
+ if from_deferred:
+ deferred.insert(0, cmd)
+ else:
+ deferred.append(cmd)
+ continue
+ batch.close()
cancel_event.clear()
# Re-check the drain after clearing: the parent sets drain_event
# then cancel_event for an unload, so if that pair landed between
@@ -1287,15 +1872,36 @@ def run_inference_process(
# which would stall the switch until the dispatcher idle-timeout.
if _drain_skip_generate(cmd, resp_queue, drain_event):
continue
- _handle_generate(backend, cmd, resp_queue, cancel_event)
+ if _teardown_skip(cmd, resp_queue, pending_teardowns):
+ continue
+ if _stopped_before_it_ran(cmd, resp_queue, stops):
+ continue
+ _dispatch_generate(
+ backend,
+ cmd,
+ resp_queue,
+ _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),
+ )
elif cmd_type == "generate_audio_input":
- # Drain discipline as in "generate" (see that branch).
if _drain_skip_generate(cmd, resp_queue, drain_event):
continue
+ if batch.rows_in_flight or (deferred and not from_deferred):
+ deferred.append(cmd)
+ continue
+ batch.close()
cancel_event.clear()
if _drain_skip_generate(cmd, resp_queue, drain_event):
continue
- _handle_generate_audio_input(backend, cmd, resp_queue, cancel_event)
+ if _teardown_skip(cmd, resp_queue, pending_teardowns):
+ continue
+ if _stopped_before_it_ran(cmd, resp_queue, stops):
+ continue
+ _handle_generate_audio_input(
+ backend,
+ cmd,
+ resp_queue,
+ _StopWhileItRuns(cancel_event, stops, cmd.get("request_id", "")),
+ )
elif cmd_type == "generate_audio":
# No TTS here, but codec checkpoints still reach this loop
# (dispatch is by device). Answer, or the parent waits 120s.
@@ -1317,19 +1923,33 @@ def run_inference_process(
},
)
elif cmd_type == "count_tokens":
- _handle_count_tokens(backend, cmd, resp_queue)
+ if batch.rows_in_flight:
+ _decline_count_tokens(cmd, resp_queue, "A generation is in progress.")
+ else:
+ _handle_count_tokens(backend, cmd, resp_queue)
elif cmd_type == "share_object":
_handle_share_object(backend, cmd, resp_queue)
elif cmd_type == "load":
+ batch.cancel_all()
+ batch.close()
+ _abandon_held_commands(deferred, resp_queue)
if backend.active_model_name:
backend.unload_model(backend.active_model_name)
_handle_load(backend, cmd, resp_queue)
elif cmd_type == "unload":
+ batch.cancel_all()
+ batch.close()
+ _abandon_held_commands(deferred, resp_queue)
_handle_unload(backend, cmd, resp_queue)
elif cmd_type == "cancel":
+ batch.cancel_all()
+ _abandon_held_commands(deferred, resp_queue)
cancel_event.set()
elif cmd_type == "reset":
+ batch.cancel_all()
+ batch.close()
cancel_event.set()
+ _abandon_held_commands(deferred, resp_queue)
backend.reset_generation_state()
_send_response(resp_queue, {"type": "reset_ack"})
elif cmd_type == "gpu_memory":
@@ -1355,6 +1975,7 @@ def run_inference_process(
},
)
elif cmd_type == "shutdown":
+ batch.close()
return
else:
# As in the GPU loop: dropping a command silently costs the
@@ -1560,7 +2181,7 @@ def run_inference_process(
# the switch until the dispatcher idle-timeout tears the subprocess down.
if _drain_skip_generate(cmd, resp_queue, drain_event):
continue
- _handle_generate(backend, cmd, resp_queue, cancel_event)
+ _dispatch_generate(backend, cmd, resp_queue, cancel_event)
elif cmd_type == "count_tokens":
_decline_count_tokens(cmd, resp_queue)
@@ -1586,7 +2207,6 @@ def run_inference_process(
_handle_unload(backend, cmd, resp_queue)
elif cmd_type == "cancel":
- # Redundant with mp.Event but handle gracefully.
cancel_event.set()
logger.info("Cancel command received")
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 364b3b9a5f2..2f37b0369e0 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -186,11 +186,14 @@ class LoadRequest(BaseModel):
ge = PARALLEL_MIN,
le = PARALLEL_MAX,
description = (
- "Parallel decode slots for llama-server (--parallel) for this "
- f"load ({PARALLEL_MIN}..{PARALLEL_MAX}). Omit for the server-wide "
- "default set at launch (the --parallel CLI flag). The VRAM fitter "
- "may launch fewer slots to keep the model fully on GPU. Ignored "
- "for non-GGUF models."
+ f"Replies this load may decode at once ({PARALLEL_MIN}..{PARALLEL_MAX}). "
+ "For GGUF these are llama-server's --parallel slots, and the VRAM "
+ "fitter may launch fewer to keep the model fully on GPU; omit for the "
+ "server-wide default set at launch. Elsewhere it is a ceiling, not a promise: "
+ "a runtime that decodes replies together commits its memory to this width and "
+ "spends it on both the replies to one turn and the concurrent chats joining a "
+ "batch already decoding, while one that does not answers every reply on its "
+ "own whatever this says. The load reply's parallel_slots is what it came to."
),
)
n_batch: Optional[int] = Field(
@@ -1221,9 +1224,26 @@ class _InferenceRuntimeFields(BaseModel):
context_length_enforced: Optional[bool] = Field(
None,
description = (
- "Whether context_length actually bounds the runtime's KV cache. True confirmed, "
- "false confirmed unbounded, null the backend does not answer. MLX builds a cache "
- "to check, since a model with its own make_cache ignores the requested size."
+ "Whether context_length bounds the runtime's KV cache for a reply decoded on "
+ "its own. True confirmed, false confirmed unbounded, null the backend does not "
+ "answer. MLX builds a cache to check, since a model with its own make_cache "
+ "ignores the requested size. Replies decoded together are answered for by "
+ "context_unbounded_when_batched, not by this."
+ ),
+ )
+ context_unbounded_when_batched: bool = Field(
+ False,
+ description = (
+ "Whether replies this load decodes together escape the window this reports. "
+ "mlx-vlm's batch generator takes no window control, so a vision load able to "
+ "run either batch reports true; everything else, llama.cpp included, reports "
+ "false. Which batch a given reply gets is a per-request question this cannot "
+ "settle, so it errs towards not promising a window. Both fields describe the "
+ "load as it was loaded and neither changes afterwards, so a client combines "
+ "them with parallel_slots to decide what to tell a user: a load reporting true "
+ "and more than one slot cannot promise a window. That is the load's answer, not "
+ "the reply's -- a load whose release keeps no batch open decodes a plain chat "
+ "bounded while still reporting true -- so read it as the weaker claim it is."
),
)
mlx_context_budget: Optional[int] = Field(
@@ -1430,18 +1450,23 @@ class _InferenceRuntimeFields(BaseModel):
requested_parallel_slots: Optional[int] = Field(
None,
description = (
- "Parallel decode slots the load was invoked with (per-load "
- "n_parallel, else the server-wide --parallel default). None for "
- "non-GGUF loads and for the diffusion runner, which ignores "
- "--parallel."
+ "Replies the load was invoked with (per-load n_parallel, else the "
+ "default; for GGUF, the server-wide --parallel set at launch). None "
+ "for the diffusion runner, which ignores --parallel."
),
)
parallel_slots: Optional[int] = Field(
None,
description = (
- "Serving slots the active llama-server actually runs (--parallel "
- "after any fit-time slot reduction). None for non-GGUF loads and "
- "for the diffusion runner, which ignores --parallel."
+ "Replies the loaded model is configured to decode at once. Not what any "
+ "one reply is running under: a batch already decoding keeps the width it "
+ "opened at until it drains, since resizing it would drop the replies "
+ "inside it, and a request submitted before a change carries the width it "
+ "was submitted with. For "
+ "GGUF these are llama-server's --parallel slots after any fit-time "
+ "reduction; elsewhere it is 1 when the runtime has no batched "
+ "generation, or its settings rule it out. None for the diffusion "
+ "runner, which ignores --parallel, and when the runtime did not say."
),
)
requested_n_batch: Optional[int] = Field(
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index ee0c82474ac..a9de136bca0 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7595,6 +7595,19 @@ def _is_explicit_tensor_drop(request: LoadRequest) -> bool:
return override is not None and override.strip().lower() != "tensor"
+def _unsloth_serving_fields(model_info: dict) -> dict:
+ """What a non-GGUF load says about serving replies, for the replies that report it."""
+ slots = model_info.get("parallel_slots")
+ batches = model_info.get("can_batch")
+ effective = None if slots is None or batches is None else (slots if batches else 1)
+ return {
+ "requested_parallel_slots": slots,
+ "parallel_slots": effective,
+ "context_length_enforced": model_info.get("context_length_enforced"),
+ "context_unbounded_when_batched": bool(model_info.get("context_unbounded_when_batched")),
+ }
+
+
def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
"""Runtime state shared by load, dedupe, and status; duplicates echo active settings."""
fields = {
@@ -7613,8 +7626,8 @@ def _llama_runtime_fields(llama_backend: LlamaCppBackend) -> dict:
mlx_kv_quant_note = None,
mlx_context_budget = None,
chat_template_override_reason = None,
- # llama.cpp allocates the window it reports: bounded by construction.
context_length_enforced = True,
+ context_unbounded_when_batched = False,
# Older/custom backend doubles predate this additive runtime field.
preserve_thinking_default = bool(getattr(llama_backend, "preserve_thinking_default", False)),
speculative_type = llama_backend.requested_spec_mode,
@@ -16600,6 +16613,7 @@ async def _load_model_impl(
):
api_monitor.discard(_load_event) # nothing loaded, no monitor row
logger.info(f"Model already loaded (Unsloth): {model_log_label}, skipping reload")
+ backend.set_parallel_slots(_n_parallel)
# A no-op Unsloth load of a preview-owned checkpoint still claims it.
_set_preview_resident(None)
inference_config = load_inference_config(backend.active_model_name)
@@ -16636,6 +16650,7 @@ async def _load_model_impl(
has_audio_input = _model_info.get("has_audio_input", False),
has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
+ **_unsloth_serving_fields(_model_info),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),
@@ -16661,7 +16676,6 @@ async def _load_model_impl(
_model_info.get("native_context_length")
),
max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),
- context_length_enforced = _model_info.get("context_length_enforced"),
mlx_context_budget = _model_info.get("mlx_context_budget"),
chat_template = _chat_template,
)
@@ -17323,6 +17337,7 @@ async def _load_model_impl(
on_prior_worker_released = _release_chat_after_teardown,
post_handoff_expected_free_gb = post_chat_handoff_expected_free_gb,
audio_device = request.audio_device,
+ n_parallel = _n_parallel,
cache_environment = cache_environment,
**anonymous_hf_kw,
**speech_codec_kw,
@@ -17449,6 +17464,7 @@ async def _load_model_impl(
has_audio_input = _model_info.get("has_audio_input", config.has_audio_input),
has_video_input = _model_info.get("has_video_input", False),
is_mlx = bool(_model_info.get("is_mlx", False)),
+ **_unsloth_serving_fields(_model_info),
mlx_kv_bits = _model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = _model_info.get("mlx_kv_bits_requested"),
mlx_kv_quant_eligibility = _model_info.get("mlx_kv_quant_eligibility"),
@@ -17470,7 +17486,6 @@ async def _load_model_impl(
context_length = _positive_int_or_none(_model_info.get("context_length")),
native_context_length = _positive_int_or_none(_model_info.get("native_context_length")),
max_context_length = _positive_int_or_none(_model_info.get("max_context_length")),
- context_length_enforced = _model_info.get("context_length_enforced"),
mlx_context_budget = _model_info.get("mlx_context_budget"),
chat_template = _chat_template,
)
@@ -19486,6 +19501,9 @@ async def generate_stream(
break
chunk = await asyncio.to_thread(next, gen, _DONE)
if chunk is _DONE:
+ if cancel_event.is_set():
+ mark_response_failed(_gs_scope)
+ break
completed = True
break
if isinstance(chunk, GenStreamError):
@@ -19890,6 +19908,7 @@ async def get_status(current_subject: str):
has_audio_input = has_audio_input,
has_video_input = has_video_input,
is_mlx = bool(model_info.get("is_mlx", False)),
+ **_unsloth_serving_fields(model_info),
mlx_kv_bits = model_info.get("mlx_kv_bits"),
mlx_kv_bits_requested = model_info.get("mlx_kv_bits_requested"),
mlx_kv_quant_eligibility = model_info.get("mlx_kv_quant_eligibility"),
@@ -19913,7 +19932,6 @@ async def get_status(current_subject: str):
context_length = _positive_int_or_none(model_info.get("context_length")),
native_context_length = _positive_int_or_none(model_info.get("native_context_length")),
max_context_length = _positive_int_or_none(model_info.get("max_context_length")),
- context_length_enforced = model_info.get("context_length_enforced"),
mlx_context_budget = model_info.get("mlx_context_budget"),
# 0 is an answer (size it yourself); None means no request is recorded. Either
# spelling: the route stamps max_seq_length_requested on every non-GGUF load,
@@ -26193,6 +26211,9 @@ async def produce_openai_chat_completions(
return
chunk_text = await asyncio.to_thread(next, gen, _DONE)
if chunk_text is _DONE:
+ if cancel_event.is_set():
+ cancelled = True
+ mark_current_response_failed()
break
if isinstance(chunk_text, GenStreamError):
_msg = _friendly_gen_stream_error(chunk_text)
@@ -29846,20 +29867,77 @@ async def produce_openai_chat_completions(
final = token
return final
+ _produced: dict = {}
+
+ async def _produce_choices(_indices):
+ """These choices' first replies, asked for in one command."""
+ rows = [
+ {} if idx == 0 else {"seed": _choice_seed(payload.seed, idx)}
+ for idx in _indices
+ ]
+ batch_stats: dict = {}
+ texts = [""] * len(rows)
+ _finished = set()
+
+ def _drain_batch():
+ for event in backend.generate_chat_batch(
+ rows = rows,
+ cancel_event = cancel_event,
+ stats_holder = batch_stats,
+ **gen_kwargs,
+ ):
+ if isinstance(event, GenStreamError):
+ return event
+ _row, _text = event
+ if _text is None:
+ _finished.add(_row)
+ else:
+ texts[_row] = _text
+ return None
+
+ failed = await _run_blocking_generation(
+ _drain_batch,
+ cancel_event,
+ name = "openai-chat-nonstream-batch",
+ )
+ _reported = batch_stats.get("stats") or []
+
+ def _keep(_row, idx):
+ _stats = _reported[_row] if _row < len(_reported) else None
+ if texts[_row] or _stats is not None:
+ _produced[idx] = (texts[_row], _stats)
+
+ if isinstance(failed, GenStreamError):
+ backend.reset_generation_state(cancel_event)
+ logger.warning(
+ "Falling back to one command per choice: %s",
+ _friendly_gen_stream_error(failed),
+ )
+ for _row, idx in enumerate(_indices):
+ if _row in _finished:
+ _keep(_row, idx)
+ return False
+ for _row, idx in enumerate(_indices):
+ _keep(_row, idx)
+ return any(idx in _produced for idx in _indices)
+
async def _one_choice(choice_index):
"""One sampled answer, healing and nudge retry included.
- A full generation of its own -- its own seed, its own stats --
- exactly as the llama-server path drains its own ``n``.
+ Its own seed and its own stats, exactly as the llama-server path drains
+ its own ``n``. Everything after the reply runs per choice either way.
"""
# Cleared per choice: one cancelled before it publishes must report
# nothing rather than inherit the previous choice's token count.
stats_holder.pop("stats", None)
- full_text = await _run_blocking_generation(
- lambda: _drain_generate(choice_index = choice_index),
- cancel_event,
- name = "openai-chat-nonstream",
- )
+ if choice_index in _produced:
+ full_text, stats_holder["stats"] = _produced[choice_index]
+ else:
+ full_text = await _run_blocking_generation(
+ lambda: _drain_generate(choice_index = choice_index),
+ cancel_event,
+ name = "openai-chat-nonstream",
+ )
if isinstance(full_text, GenStreamError):
backend.reset_generation_state(cancel_event)
_msg = _friendly_gen_stream_error(full_text)
@@ -29892,10 +29970,12 @@ async def produce_openai_chat_completions(
if _reasoning_text:
_msg["reasoning_content"] = _reasoning_text
# Budget exhaustion unless a heal below promotes a tool call; a cancelled turn
- # stopped on request, not at the cap, so it stays "stop".
+ # stopped on request, not at the cap, so it stays "stop". A reply the
+ # batch produced reports its own end instead: a Stop aimed at some other
+ # choice must not rewrite the reason this one already finished for.
_finish = (
"stop"
- if cancel_event.is_set()
+ if cancel_event.is_set() and choice_index not in _produced
else _stats_finish_reason(stats_holder.get("stats"))
)
if _sf_heal:
@@ -29985,12 +30065,19 @@ async def produce_openai_chat_completions(
_msg["tool_calls"] = _tcs[:1]
return _msg, _finish, stats_holder.get("stats")
+ from core.inference.llama_server_args import PARALLEL_DEFAULT
+
+ _width = getattr(backend, "effective_parallel_slots", PARALLEL_DEFAULT)
+ if _n > 1 and _width > 1 and payload.use_adapter is None and not cancel_event.is_set():
+ for _start in range(0, _n, _width):
+ if cancel_event.is_set():
+ break
+ if not await _produce_choices(range(_start, min(_start + _width, _n))):
+ break
+
for _idx in range(_n):
- # Stop spawning the remaining choices once cancelled. Index 0 runs
- # regardless, so a cancelled turn still answers with one choice
- # rather than the empty `choices` a client indexes into.
- if _idx and cancel_event.is_set():
- break
+ if _idx and cancel_event.is_set() and _idx not in _produced:
+ continue
_msg, _finish, _choice_stats = await _one_choice(_idx)
_choices.append(
CompletionChoice(
@@ -30050,9 +30137,11 @@ async def produce_openai_chat_completions(
if len(_choices) > 1:
# Every choice, labelled, as the llama-server drain reports them:
# showing the last one alone would hide the rest of the turn.
+ # Numbered from the choice's own index, not its place in the list: a
+ # cancelled turn skips the choices the batch never produced.
_monitor_reply = "\n\n".join(
- f"Choice {_i + 1}:\n{_reply_text(_reply, _choice.finish_reason)}"
- for _i, (_reply, _choice) in enumerate(zip(_monitor_replies, _choices))
+ f"Choice {_choice.index + 1}:\n{_reply_text(_reply, _choice.finish_reason)}"
+ for _reply, _choice in zip(_monitor_replies, _choices)
)
else:
_monitor_reply = _reply_text(_msg, _finish)
@@ -30558,8 +30647,13 @@ def _openai_model_objects() -> list[dict]:
_value = _positive_int_or_none(model_info.get(_field))
if _value is not None:
entry[_field] = _value
- if model_info.get("context_length_enforced") is not None:
- entry["context_length_enforced"] = bool(model_info["context_length_enforced"])
+ _serving = _unsloth_serving_fields(model_info)
+ if _serving["context_length_enforced"] is not None:
+ entry["context_length_enforced"] = bool(_serving["context_length_enforced"])
+ if _serving["context_unbounded_when_batched"]:
+ entry["context_unbounded_when_batched"] = True
+ if _serving["parallel_slots"] is not None:
+ entry["parallel_slots"] = _serving["parallel_slots"]
models.append(entry)
return models
diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py
index ba124b0581e..9e1d0d574bc 100644
--- a/studio/backend/tests/test_active_generations.py
+++ b/studio/backend/tests/test_active_generations.py
@@ -7,15 +7,18 @@ A load/unload has to know which streaming chats it would interrupt. Everything
under test is a dict + threading.Lock, so this passes on every platform.
"""
+import multiprocessing as _mp
import os
import sys
import threading
+import uuid
import pytest
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
+from core.inference.worker import _SLOTS, StopLedger
from state import active_generations
@@ -276,16 +279,25 @@ def test_tracked_cancel_shares_its_event_with_the_registry():
tracker.__exit__(None, None, None)
-def _stub_load_route(monkeypatch, *, active_model_name):
- """Point POST /load at an in-memory safetensors backend.
-
- active_model_name == the requested path makes the request idempotent, so
- _load_model_impl takes its already_loaded fast return.
- """
+def _stub_load_route(
+ monkeypatch,
+ *,
+ active_model_name,
+ backend = None,
+):
from types import SimpleNamespace
import routes.inference as inf_mod
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ if backend is None:
+ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ backend._stop_ledger = None
+ backend._pending_teardowns = None
+ backend.active_model_name = active_model_name
+ backend.models = {}
+
monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", lambda: None)
monkeypatch.setattr(inf_mod, "validate_extra_args", lambda args: [])
monkeypatch.setattr(
@@ -307,11 +319,7 @@ def _stub_load_route(monkeypatch, *, active_model_name):
},
)
monkeypatch.setattr(inf_mod, "_resolve_loaded_trust_remote_code", lambda *a, **k: False)
- monkeypatch.setattr(
- inf_mod,
- "get_inference_backend",
- lambda: SimpleNamespace(active_model_name = active_model_name, models = {}),
- )
+ monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend)
monkeypatch.setattr(
inf_mod,
"get_llama_cpp_backend",
@@ -343,6 +351,27 @@ def test_idempotent_load_neither_refuses_nor_cancels_running_chats(monkeypatch):
assert not ev.is_set()
+def test_a_width_change_reaches_the_resident_model_without_replacing_it(monkeypatch):
+ _route_gate()
+ import asyncio
+
+ from models.inference import LoadRequest
+
+ inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A")
+ applied = []
+ inf_mod.get_inference_backend().set_parallel_slots = applied.append
+
+ ev = threading.Event()
+ with active_generations.ActiveGeneration(ev, thread_id = "t1"):
+ response = asyncio.run(
+ inf_mod.load_model(LoadRequest(model_path = "org/A", n_parallel = 2), object(), "tester")
+ )
+
+ assert response.status == "already_loaded"
+ assert applied == [2]
+ assert not ev.is_set(), "a width change ended a reply that was decoding"
+
+
def test_a_real_reload_still_refuses_while_chats_stream(monkeypatch):
# A load that would really replace the model still 409s and names the chats.
_route_gate()
@@ -2550,6 +2579,70 @@ def test_drain_returns_as_soon_as_the_cancelled_requests_unwind(monkeypatch):
assert polls == 3
+def _ids(n):
+ return [str(uuid.uuid4()) for _ in range(n)]
+
+
+def _stopped(ledger):
+ """What the worker would read: the only way the record is read."""
+ return ledger.snapshot()[1]
+
+
+def test_a_request_reads_as_stopped_once_it_is():
+ ledger = StopLedger(_mp.get_context("spawn"))
+ mine, theirs = _ids(2)
+
+ assert _stopped(ledger) == set()
+ assert ledger.stop(mine)
+ assert _stopped(ledger) == {mine}, "a stop names one request and no other"
+ assert theirs not in _stopped(ledger)
+
+
+def test_the_oldest_stop_is_the_one_that_ages_out():
+ ledger = StopLedger(_mp.get_context("spawn"))
+ recorded = _ids(_SLOTS + 1)
+ for request_id in recorded:
+ assert ledger.stop(request_id)
+
+ assert _stopped(ledger) == set(recorded[1:]), "the oldest made way for the newest"
+
+
+def test_stopping_the_same_request_twice_spends_one_slot():
+ ledger = StopLedger(_mp.get_context("spawn"))
+ theirs, mine = _ids(2)
+ assert ledger.stop(theirs)
+ for _ in range(_SLOTS):
+ assert ledger.stop(mine)
+
+ assert _stopped(ledger) == {mine, theirs}, "the one stopped first is still stopped"
+
+
+def test_a_snapshot_answers_for_every_reply_at_once():
+ ledger = StopLedger(_mp.get_context("spawn"))
+ mine, absent = _ids(2)
+ theirs = "short"
+ ledger.stop(mine)
+ ledger.stop(theirs)
+
+ written, stopped = ledger.snapshot()
+ assert written == 2
+ assert stopped == {mine, theirs}, "the ids as recorded, with nothing padded onto them"
+ assert absent not in stopped
+
+
+class _Ledger:
+ def __init__(self, read_by_worker = False):
+ self.stopped: set = set()
+ self._read_by_worker = read_by_worker
+
+ def stop(self, request_id):
+ self.stopped.add(request_id)
+ return True
+
+ def read_by_worker(self) -> bool:
+ return self._read_by_worker
+
+
# ── queued chats must not cancel the running one ──────────────────────
@@ -2560,10 +2653,15 @@ def _orchestrator_for_ownership():
"core.inference.orchestrator", reason = "inference stack not installed"
)
orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
+ orch._stop_ledger = _Ledger()
+ orch._pending_teardowns = None
orch._gen_lock = threading.Lock()
orch._active_cancel_events = []
orch._executing_cancel_events = []
orch._active_cancel_lock = threading.Lock()
+ orch._send_order_lock = threading.Lock()
+ orch._mailbox_lock = threading.Lock()
+ orch._request_cancel_events = {}
orch._cancel_event = threading.Event()
orch._ensure_subprocess_alive = lambda: False # stop before _send_cmd
return orch
@@ -2738,6 +2836,8 @@ def test_claim_order_matches_send_order_under_concurrent_dispatch():
"core.inference.orchestrator", reason = "inference stack not installed"
)
orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator)
+ orch._stop_ledger = None
+ orch._pending_teardowns = None
orch._active_cancel_events = []
orch._executing_cancel_events = []
orch._active_cancel_lock = threading.Lock()
diff --git a/studio/backend/tests/test_audio_tts_cancellation.py b/studio/backend/tests/test_audio_tts_cancellation.py
index 25aa74f3205..177b6715133 100644
--- a/studio/backend/tests/test_audio_tts_cancellation.py
+++ b/studio/backend/tests/test_audio_tts_cancellation.py
@@ -73,6 +73,8 @@ from models.inference import ChatCompletionRequest # noqa: E402
def _bare_orchestrator():
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orchestrator._stop_ledger = None
+ orchestrator._pending_teardowns = None
orchestrator._gen_lock = threading.Lock()
orchestrator._send_order_lock = threading.Lock()
orchestrator._active_cancel_lock = threading.Lock()
@@ -86,13 +88,14 @@ def _bare_orchestrator():
orchestrator._dispatcher_thread = None
orchestrator._dispatcher_stop = threading.Event()
orchestrator._dispatcher_lifecycle_lock = threading.Lock()
+ orchestrator._worker_released = threading.Condition(orchestrator._dispatcher_lifecycle_lock)
orchestrator._subprocess_shutdown_lock = threading.Lock()
orchestrator._mailbox_lock = threading.Lock()
orchestrator._mailboxes = {}
orchestrator._direct_mailboxes = {}
orchestrator._request_cancel_events = {}
orchestrator._unload_pending = False
- orchestrator._exclusive_tts_pending = False
+ orchestrator._worker_reserved_for = None
orchestrator.active_model_name = "model"
orchestrator.models = {"model": {}}
orchestrator.loading_models = set()
@@ -222,7 +225,11 @@ def test_audio_response_cancellation_signals_worker_and_drains_terminal_response
monkeypatch.setattr(
orchestrator,
"_direct_reader",
- lambda _request_id: (read_one, lambda **_kwargs: None, lambda: released.append(True)),
+ lambda _request_id, _cancel_event = None: (
+ read_one,
+ lambda **_kwargs: None,
+ lambda: released.append(True),
+ ),
)
with pytest.raises(RuntimeError, match = "cancel"):
@@ -330,7 +337,7 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):
monkeypatch.setattr(
orchestrator,
"_direct_reader",
- lambda _request_id: (
+ lambda _request_id, _cancel_event = None: (
read_one,
lambda **_kwargs: pytest.fail("the cancellation drain window was already spent"),
lambda: None,
@@ -339,7 +346,7 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):
shutdown_state = []
def shutdown(*, timeout):
- shutdown_state.append((orchestrator._exclusive_tts_pending, timeout))
+ shutdown_state.append((orchestrator._worker_reserved_for, timeout))
return True
monkeypatch.setattr(orchestrator, "_shutdown_subprocess", shutdown)
@@ -350,10 +357,10 @@ def test_audio_response_cancellation_bounds_an_unresponsive_worker(monkeypatch):
assert time.monotonic() - started < 0.5
assert cancel_signals == [True]
- assert shutdown_state == [(True, 0.03)]
+ assert shutdown_state == [("audio generation is in progress", 0.03)]
assert orchestrator.active_model_name is None
assert orchestrator.models == {}
- assert orchestrator._exclusive_tts_pending is False
+ assert orchestrator._worker_reserved_for is None
def test_audio_response_cancellation_before_worker_start_is_still_bounded(monkeypatch):
@@ -382,7 +389,7 @@ def test_audio_response_cancellation_before_worker_start_is_still_bounded(monkey
monkeypatch.setattr(
orchestrator,
"_direct_reader",
- lambda _request_id: (
+ lambda _request_id, _cancel_event = None: (
read_one,
lambda **_kwargs: pytest.fail("the cancellation drain window was already spent"),
lambda: None,
@@ -401,7 +408,7 @@ def test_audio_response_cancellation_before_worker_start_is_still_bounded(monkey
assert time.monotonic() - started < 0.5
assert shutdown_state == [0.03]
- assert orchestrator._exclusive_tts_pending is False
+ assert orchestrator._worker_reserved_for is None
def test_audio_generation_timeout_scales_with_requested_tokens(monkeypatch):
@@ -444,7 +451,7 @@ def test_audio_worker_command_uses_the_model_token_budget(
sent = []
monkeypatch.setattr(orchestrator, "_send_cmd", lambda cmd: sent.append(cmd))
- def direct_reader(request_id):
+ def direct_reader(request_id, cancel_event = None):
responses = queue.Queue()
responses.put(
{
@@ -491,7 +498,7 @@ def test_audio_response_timeout_cancels_and_drains_before_releasing(monkeypatch)
cancel_state = []
def cancel_generation():
- cancel_state.append(orchestrator._exclusive_tts_pending)
+ cancel_state.append(orchestrator._worker_reserved_for)
orchestrator._cancel_event.set()
orchestrator._resp_queue.put(
{
@@ -506,8 +513,10 @@ def test_audio_response_timeout_cancels_and_drains_before_releasing(monkeypatch)
with pytest.raises(RuntimeError, match = "Timeout waiting for audio generation"):
orchestrator.generate_audio_response("hello")
- assert cancel_state == [True], "timeout cancellation must occur under TTS exclusivity"
- assert orchestrator._exclusive_tts_pending is False
+ assert cancel_state == [
+ "audio generation is in progress"
+ ], "timeout cancellation must occur under TTS exclusivity"
+ assert orchestrator._worker_reserved_for is None
assert orchestrator._active_cancel_events == []
assert orchestrator._executing_cancel_events == []
@@ -530,7 +539,7 @@ def test_audio_response_timeout_tears_down_unresponsive_worker_before_release(mo
shutdown_state = []
def shutdown(*, timeout):
- shutdown_state.append((orchestrator._exclusive_tts_pending, timeout))
+ shutdown_state.append((orchestrator._worker_reserved_for, timeout))
return True
monkeypatch.setattr(orchestrator, "_shutdown_subprocess", shutdown)
@@ -538,8 +547,8 @@ def test_audio_response_timeout_tears_down_unresponsive_worker_before_release(mo
with pytest.raises(RuntimeError, match = "Timeout waiting for audio generation"):
orchestrator.generate_audio_response("hello")
- assert shutdown_state == [(True, 0.03)]
- assert orchestrator._exclusive_tts_pending is False
+ assert shutdown_state == [("audio generation is in progress", 0.03)]
+ assert orchestrator._worker_reserved_for is None
assert orchestrator.active_model_name is None
assert orchestrator.models == {}
@@ -598,14 +607,14 @@ class _AliveDispatcher:
def test_dispatcher_refuses_during_exclusive_tts_and_resumes_after():
orchestrator = _bare_orchestrator()
orchestrator._resp_queue = queue.Queue()
- orchestrator._exclusive_tts_pending = True
+ orchestrator._worker_reserved_for = "audio generation is in progress"
- assert orchestrator._start_dispatcher() is False
+ assert orchestrator._start_dispatcher() is None
assert orchestrator._dispatcher_thread is None
- orchestrator._exclusive_tts_pending = False
+ orchestrator._worker_reserved_for = None
try:
- assert orchestrator._start_dispatcher() is True
+ assert orchestrator._start_dispatcher() is orchestrator._dispatcher_thread
assert orchestrator._dispatcher_thread is not None
assert orchestrator._dispatcher_thread.is_alive()
finally:
@@ -628,7 +637,7 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):
responses = queue.Queue()
- def direct_reader(request_id):
+ def direct_reader(request_id, cancel_event = None):
responses.put({"type": "audio_started", "request_id": request_id})
responses.put(
{
@@ -652,9 +661,9 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):
thread.start()
deadline = time.monotonic() + 2
- while not orchestrator._exclusive_tts_pending and time.monotonic() < deadline:
+ while not orchestrator._worker_reserved_for and time.monotonic() < deadline:
time.sleep(0.01)
- assert orchestrator._exclusive_tts_pending is True
+ assert orchestrator._worker_reserved_for == "audio generation is in progress"
assert sent == [], "TTS must not enqueue behind the active compare request"
with orchestrator._mailbox_lock:
@@ -664,7 +673,7 @@ def test_tts_waits_for_existing_compare_before_send(monkeypatch):
assert thread.is_alive() is False
assert result["value"] == (b"RIFFfake", 24000)
assert sent and sent[0]["type"] == "generate_audio"
- assert orchestrator._exclusive_tts_pending is False
+ assert orchestrator._worker_reserved_for is None
def test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):
@@ -691,9 +700,9 @@ def test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):
thread = threading.Thread(target = run)
thread.start()
deadline = time.monotonic() + 2
- while not orchestrator._exclusive_tts_pending and time.monotonic() < deadline:
+ while not orchestrator._worker_reserved_for and time.monotonic() < deadline:
time.sleep(0.01)
- assert orchestrator._exclusive_tts_pending is True
+ assert orchestrator._worker_reserved_for == "audio generation is in progress"
caller_cancel.set()
thread.join(timeout = 2)
@@ -702,30 +711,7 @@ def test_tts_cancel_while_waiting_does_not_signal_active_compare(monkeypatch):
assert "cancel" in str(error["value"]).lower()
assert cancel_calls == []
assert set(orchestrator._mailboxes) == {"compare"}
- assert orchestrator._exclusive_tts_pending is False
-
-
-def test_dispatched_generation_rechecks_tts_reservation_before_registration(monkeypatch):
- orchestrator = _bare_orchestrator()
- orchestrator._dispatcher_thread = _AliveDispatcher()
- monkeypatch.setattr(orchestrator, "_ensure_subprocess_alive", lambda: True)
- monkeypatch.setattr(orchestrator, "_start_dispatcher", lambda: False)
-
- def reserve_tts(*_args, **_kwargs):
- orchestrator._exclusive_tts_pending = True
- return {"type": "generate", "request_id": "compare-1"}
-
- monkeypatch.setattr(orchestrator, "_build_generate_cmd", reserve_tts)
- monkeypatch.setattr(
- orchestrator,
- "_send_cmd",
- lambda _cmd: pytest.fail("compare must not enqueue after TTS reservation"),
- )
-
- output = list(orchestrator._generate_dispatched(messages = [{"role": "user", "content": "x"}]))
-
- assert any("audio generation" in str(chunk).lower() for chunk in output)
- assert orchestrator._mailboxes == {}
+ assert orchestrator._worker_reserved_for is None
def test_worker_audio_forwards_shared_cancel_event():
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index 5dfff397478..a1f9494acbc 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -1062,6 +1062,8 @@ class TestLoadHubDownloadExclusion:
"chat_template_override_reason",
# Constant True: llama.cpp allocates the window it reports.
"context_length_enforced",
+ # Constant False: each slot decodes against its own window.
+ "context_unbounded_when_batched",
# Read from requested_extra_args, which is what the load was invoked
# with rather than the rewritten launch list.
"requested_llama_extra_args",
diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.py
index 00172e6a0de..003f7e72d59 100644
--- a/studio/backend/tests/test_inference_dispatcher_resilience.py
+++ b/studio/backend/tests/test_inference_dispatcher_resilience.py
@@ -14,7 +14,6 @@ import ast
import queue
import sys
import threading
-import time
from pathlib import Path
import pytest
@@ -38,6 +37,8 @@ class _ScriptedQueue:
def _dispatcher():
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._stop_ledger = None
+ o._pending_teardowns = None
o._dispatcher_stop = threading.Event()
o._mailbox_lock = threading.Lock()
o._mailboxes = {}
@@ -128,6 +129,8 @@ def test_route_llama_streaming_async_clients_disable_proxy_env():
def _direct_reader_host():
"""Orchestrator with only what _direct_reader and the ownership helpers touch."""
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._stop_ledger = None
+ o._pending_teardowns = None
o._mailbox_lock = threading.Lock()
o._mailboxes = {}
o._direct_mailboxes = {}
@@ -185,10 +188,14 @@ def test_rerouting_a_foreign_gen_done_retires_that_request():
release()
-def _direct_reader_calls(o, request_id):
+def _direct_reader_calls(
+ o,
+ request_id,
+ cancel_event = None,
+):
"""_direct_reader wired to a scripted _read_resp (o._scripted, popped in order)."""
o._read_resp = lambda timeout = 1.0, observe = True: (o._scripted.pop(0) if o._scripted else None)
- return o._direct_reader(request_id)
+ return o._direct_reader(request_id, cancel_event)
@pytest.mark.parametrize(
diff --git a/studio/backend/tests/test_inference_orchestrator_crash_message.py b/studio/backend/tests/test_inference_orchestrator_crash_message.py
index be1a673e620..ac32c30c0e3 100644
--- a/studio/backend/tests/test_inference_orchestrator_crash_message.py
+++ b/studio/backend/tests/test_inference_orchestrator_crash_message.py
@@ -21,6 +21,7 @@ def test_subprocess_crash_message_includes_signal_and_oom_hint():
spec.loader.exec_module(module)
orchestrator = module.InferenceOrchestrator.__new__(module.InferenceOrchestrator)
+ orchestrator._stop_ledger = None
orchestrator._proc = SimpleNamespace(pid = 1234, exitcode = -9)
msg = orchestrator._subprocess_crash_message("wait")
diff --git a/studio/backend/tests/test_inference_worker_stderr_tail.py b/studio/backend/tests/test_inference_worker_stderr_tail.py
index 2c2e6901290..61ea1dba778 100644
--- a/studio/backend/tests/test_inference_worker_stderr_tail.py
+++ b/studio/backend/tests/test_inference_worker_stderr_tail.py
@@ -1135,7 +1135,7 @@ def test_a_stream_whose_worker_was_swapped_gets_no_tail_at_all():
body = inspect.getsource(orchestrator_module.InferenceOrchestrator._consume_token_stream)
swap = body.index("initial_proc or self._resp_queue is not initial_resp_queue")
- following = body[swap : body.index("resp = read_one(read_timeout)", swap)]
+ following = body[swap : body.index("resp = read_one(", swap)]
assert "_subprocess_crash_message(crash_context)" in following, following
assert "with_worker_output" not in following, following
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index d94fc4090ff..dfd327f88f9 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -394,12 +394,17 @@ def _install_fake_mlx(monkeypatch):
mlx_core.set_wired_limit = _DummyMX.set_wired_limit
mlx_core.device_info = _DummyMX.device_info
mlx_core.synchronize = _DummyMX.synchronize
+ mlx_core.clear_cache = lambda: None
mlx_utils.tree_unflatten = dict
+ mlx_nn = types.ModuleType("mlx.nn")
+ mlx_nn.Module = type("Module", (), {"__init__": lambda self: None})
mlx_pkg.core = mlx_core
mlx_pkg.utils = mlx_utils
+ mlx_pkg.nn = mlx_nn
monkeypatch.setitem(sys.modules, "mlx", mlx_pkg)
monkeypatch.setitem(sys.modules, "mlx.core", mlx_core)
monkeypatch.setitem(sys.modules, "mlx.utils", mlx_utils)
+ monkeypatch.setitem(sys.modules, "mlx.nn", mlx_nn)
def _install_fake_fast_mlx(monkeypatch, calls):
@@ -3323,6 +3328,60 @@ def test_template_probe_renders_through_the_path_generation_uses(monkeypatch):
backend._render_template_probe(False)
+def _vlm_backend(
+ monkeypatch,
+ *,
+ markers,
+ prompt_tail = "",
+):
+ from core.inference import mlx_inference
+
+ monkeypatch.setattr(
+ "core.inference.chat_template_helpers.apply_chat_template_for_generation",
+ lambda *a, **k: "<start_of_turn>user\nhi<end_of_turn>\n<start_of_turn>model\n"
+ + prompt_tail,
+ )
+ monkeypatch.setattr(
+ mlx_inference, "detect_reasoning_channel_markers", lambda target, tools = None: markers
+ )
+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)
+ backend._tokenizer = SimpleNamespace(chat_template = "t", all_special_tokens = [])
+ backend._processor = SimpleNamespace(
+ chat_template = "t", tokenizer = backend._tokenizer, apply_chat_template = lambda *a, **k: ""
+ )
+ backend._model = object()
+ return backend
+
+
+def _vlm_row(
+ backend,
+ *,
+ stop = None,
+ messages = None,
+ continue_final_message = False,
+):
+ """One vision reply resolved by that backend."""
+ return backend._plan_vlm_row(
+ messages or [{"role": "user", "content": "hi"}],
+ None,
+ temperature = 0.7,
+ top_p = 0.9,
+ top_k = 0,
+ min_p = 0.0,
+ max_new_tokens = 16,
+ repetition_penalty = 1.0,
+ continue_final_message = continue_final_message,
+ stop = stop,
+ ).stream
+
+
+def test_a_vision_reply_cuts_at_its_own_stop_sequence(monkeypatch):
+ """The row carries the request's stop sequences, matched on the sampled text."""
+ row = _vlm_row(_vlm_backend(monkeypatch, markers = None), stop = ["HALT"])
+ assert row.feed(text = "some text HALT and more") == ["some text "]
+ assert row.stopped
+
+
def test_the_audio_refusal_puts_the_native_template_back(monkeypatch):
"""The refusal itself, not just the marker check.
@@ -5232,6 +5291,309 @@ def test_the_mlx_mcp_snapshot_is_taken_under_the_same_guard_the_gguf_count_uses(
assert guard < snapshot, "the guard must be held across the snapshot, not after it"
+class _TwoRowStream:
+ """A vision batch reporting two rows, recording whatever is taken back."""
+
+ def __init__(
+ self,
+ events,
+ retired = (),
+ ):
+ self._events, self._retired, self.withdrawn = events, set(retired), []
+
+ def step(self):
+ return iter(self._events)
+
+ def withdraw(self, row):
+ self.withdrawn.append(row)
+ if row in self._retired:
+ return None
+ return SimpleNamespace(prompt_token_count = 3, token_ids = [1, 2])
+
+
+def _open_vision_session(rows):
+ """A session holding the given rows, with nothing else started."""
+ from core.inference import mlx_inference
+
+ session = mlx_inference._VisionBatchSession.__new__(mlx_inference._VisionBatchSession)
+ session._rows = {row.handle: row for row in rows}
+ session._by_row, session._settled = {row.row: row for row in rows}, {}
+ return session
+
+
[email protected]("stopping", (0, 1))
[email protected]("terminal", (False, True))
+def test_a_vision_row_cut_by_its_stop_sequence_leaves_the_batch(monkeypatch, stopping, terminal):
+ """Whichever row stops is closed out where it stopped and its neighbour."""
+ from core.inference import mlx_inference
+
+ backend = _vlm_backend(monkeypatch, markers = None)
+ handles = ("a", "b")
+ session = _open_vision_session(
+ [
+ mlx_inference._VisionBatchRow(
+ handle = handle,
+ plan = SimpleNamespace(stream = _vlm_row(backend, stop = ["HALT"])),
+ row = number,
+ )
+ for number, handle in enumerate(handles)
+ ]
+ )
+ other = 1 - stopping
+ result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1, 2])
+ session.stream = _TwoRowStream(
+ [
+ SimpleNamespace(
+ index = stopping, delta = "one HALT two", result = result if terminal else None
+ ),
+ SimpleNamespace(index = other, delta = "still going", result = None),
+ ],
+ retired = (stopping,) if terminal else (),
+ )
+
+ reported = list(session.step())
+
+ assert (handles[stopping], None) in reported, "the stopped row is reported finished"
+ assert session._settled[handles[stopping]]["finish_reason"] == "stop"
+ assert handles[stopping] not in session._rows, "and it has left the batch"
+ assert handles[other] in session._rows, "its neighbour is still in the batch"
+ assert (handles[other], "still going") in reported
+ assert session.stream.withdrawn == (
+ [] if terminal else [stopping]
+ ), "a row the engine has already retired is not handed back again"
+
+
+def _batch_engine(monkeypatch):
+ """The zoo batch engine, or a stand-in where the backend CI installs no unsloth-zoo."""
+ try:
+ from unsloth_zoo.mlx import generate as engine
+ except ImportError:
+ engine = types.ModuleType("unsloth_zoo.mlx.generate")
+ engine.stream_unavailable_reason = lambda *a, **k: None
+ for name in ("unsloth_zoo", "unsloth_zoo.mlx"):
+ if not isinstance(sys.modules.get(name), types.ModuleType):
+ monkeypatch.setitem(sys.modules, name, types.ModuleType(name))
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.generate", engine)
+ monkeypatch.setattr(sys.modules["unsloth_zoo.mlx"], "generate", engine, raising = False)
+ return engine
+
+
+def test_a_text_load_whose_engine_cannot_batch_says_so_before_it_commits(monkeypatch):
+ """The reply is owed the decode it would have had, not the error building raises."""
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ engine = _batch_engine(monkeypatch)
+
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False
+ for gap in ("BatchGenerator.insert missing", None):
+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: gap)
+ assert backend.resident_unavailable_reason({}) == gap
+ assert backend.batch_unavailable_reason([{}, {}]) == gap
+
+
+ "extra",
+ (
+ {"images": [object()]},
+ {"images_base64": ["AAAA"]},
+ {"video": "AAAAGGZ0eXBtcDQy"},
+ {"video_base64": "AAAAGGZ0eXBtcDQy"},
+ {"tools": [{"type": "function", "function": {"name": "lookup"}}]},
+ {"tool_protocol_active": True},
+ {"response_format": {"type": "json_object"}},
+ ),
+)
+def test_a_reply_the_batch_cannot_serve_decodes_alone(monkeypatch, extra):
+ """Replayed tool images, a clip and the native tool protocol live only on the"""
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ engine = _batch_engine(monkeypatch)
+
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False
+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)
+ plain = {"messages": [], "images": [], "image_ordinal": 0}
+
+ assert backend.resident_unavailable_reason(plain) is None
+ assert backend.batch_unavailable_reason([plain, plain]) is None
+ assert backend.resident_unavailable_reason({**plain, **extra}) is not None
+ assert backend.batch_unavailable_reason([plain, {**plain, **extra}]) is not None
+
+
+def test_a_quantized_or_budgeted_kv_cache_does_not_batch(monkeypatch):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ engine = _batch_engine(monkeypatch)
+
+ backend = MLXInferenceBackend.__new__(MLXInferenceBackend)
+ backend._model, backend._tokenizer, backend._is_vlm = object(), object(), False
+ backend._kv_quant, backend._kv_context_budget = {"kv_bits": None}, None
+ monkeypatch.setattr(engine, "stream_unavailable_reason", lambda *a, **k: None)
+ assert backend.resident_unavailable_reason({}) is None
+
+ for quant, budget in (({"kv_bits": 8}, None), ({"kv_bits": None}, 4096)):
+ backend._kv_quant, backend._kv_context_budget = quant, budget
+ assert backend.resident_unavailable_reason({}) is not None
+ assert backend.batch_unavailable_reason([{}, {}]) is not None
+
+
+try:
+ import mlx.core as _mx
+ _METAL = _mx.metal.is_available()
+except Exception:
+ _METAL = False
+
+metal_only = pytest.mark.skipif(not _METAL, reason = "requires Apple Silicon Metal")
+
+BATCH_MODEL = "mlx-community/SmolLM2-135M-Instruct"
+BATCH_PROMPT = [{"role": "user", "content": "Name a colour and explain why."}]
+BATCH_VLM_MODEL = "mlx-community/Qwen3.5-0.8B-4bit"
+
+
[email protected](scope = "module")
+def batch_backend(allow_outbound_network):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ loaded = MLXInferenceBackend()
+ with allow_outbound_network():
+ loaded.load_model(BATCH_MODEL)
+ return loaded
+
+
[email protected](scope = "module")
+def batch_vlm_backend(allow_outbound_network):
+ from core.inference.mlx_inference import MLXInferenceBackend
+
+ loaded = MLXInferenceBackend()
+ with allow_outbound_network():
+ loaded.load_model(SimpleNamespace(identifier = BATCH_VLM_MODEL, is_vision = True))
+ return loaded
+
+
+def _batch_request(**overrides):
+ base = dict(messages = BATCH_PROMPT, max_new_tokens = 24, temperature = 0.9)
+ base.update(overrides)
+ return base
+
+
+def _drain_batch(backend, requests, **kwargs):
+ """Cumulative snapshots per row, plus the stats latched at the end."""
+ replies: dict[int, list[str]] = {row: [] for row in range(len(requests))}
+ finished = []
+ for row, snapshot in backend.generate_chat_batch(requests, **kwargs):
+ if snapshot is None:
+ finished.append(row)
+ else:
+ replies[row].append(snapshot)
+ assert sorted(finished) == sorted(replies), "every row must report completion once"
+ return replies, list(backend.last_batch_generation_stats)
+
+
+@metal_only
+def test_a_batch_costs_one_forward_per_token_not_one_per_reply(batch_backend):
+ model = batch_backend._model
+ original = type(model).__call__
+ calls = {"n": 0}
+
+ def counted(self, *args, **kwargs):
+ calls["n"] += 1
+ return original(self, *args, **kwargs)
+
+ type(model).__call__ = counted
+ try:
+ _replies, stats = _drain_batch(
+ batch_backend, [_batch_request(seed = 200 + i) for i in range(4)]
+ )
+ finally:
+ type(model).__call__ = original
+
+ generated = sum(entry["usage"]["completion_tokens"] for entry in stats)
+ assert generated >= 4 * 20
+ assert (
+ calls["n"] < generated / 2
+ ), f"{calls['n']} forwards for {generated} tokens across 4 replies"
+
+
+@metal_only
+def test_an_out_of_range_logit_bias_id_cannot_corrupt_a_batch(batch_backend):
+ """MLX does no bounds checking on the gather these processors do."""
+ plain = _batch_request(seed = 21)
+ stray = _batch_request(seed = 21, logit_bias = {10**9: 5.0, -4: 3.0})
+ _first, _ = _drain_batch(batch_backend, [plain, plain])
+ opener = batch_backend._tokenizer.encode(_first[0][-1])[0]
+ real = _batch_request(seed = 21, logit_bias = {int(opener): -100.0})
+
+ replies, stats = _drain_batch(batch_backend, [plain, stray, stray, real])
+ assert all(entry["usage"]["completion_tokens"] > 0 for entry in stats)
+ assert replies[1] == replies[0]
+ assert replies[2] == replies[0]
+ assert replies[3] != replies[0]
+
+
+def test_a_repeated_token_is_charged_the_penalty_it_earned():
+ """The frequency processor counts occurrences in float32 and scales once."""
+ mx = pytest.importorskip("mlx.core")
+
+ from core.inference.mlx_inference import _make_mlx_frequency_penalty_processor
+
+ processor = _make_mlx_frequency_penalty_processor(0.3)
+ logits = mx.zeros((1, 8), dtype = mx.float16)
+ processor(mx.array([0, 1]), logits) # latches the prompt length
+ tokens = mx.array([0, 1] + [5] * 1000)
+ charged = processor(tokens, logits)
+ assert abs(float(charged[0, 5]) + 300.0) < 0.5, float(charged[0, 5])
+ assert float(charged[0, 4]) == 0.0
+
+
+@metal_only
+def test_a_loaded_vision_model_says_a_batch_would_leave_its_window_behind(batch_vlm_backend):
+ stored = batch_vlm_backend.models[batch_vlm_backend.active_model_name]
+
+ assert stored["context_unbounded_when_batched"] is True
+
+
+@metal_only
+def test_a_vision_reply_stopped_partway_reports_the_tokens_it_actually_used(batch_vlm_backend):
+ """A stopped row has no result, so its usage has to come from what it produced."""
+ import threading
+
+ from PIL import Image
+
+ def rows():
+ return [
+ dict(
+ messages = [{"role": "user", "content": "Describe this in detail."}],
+ image = Image.new("RGB", (64, 64), (12, 200, 60)),
+ max_new_tokens = 64,
+ temperature = 0.0,
+ )
+ for _ in range(2)
+ ]
+
+ _replies, whole = _drain_batch(batch_vlm_backend, rows())
+
+ cancel = threading.Event()
+ delivered: dict[int, list[str]] = {0: [], 1: []}
+ for row, snapshot in batch_vlm_backend.generate_chat_batch(rows(), cancel_event = cancel):
+ if snapshot is not None:
+ delivered[row].append(snapshot)
+ if all(len(seen) >= 3 for seen in delivered.values()):
+ cancel.set()
+
+ cut_short = batch_vlm_backend.last_batch_generation_stats
+ assert all(delivered.values()), "nothing was delivered before the batch was cancelled"
+ for row, seen in delivered.items():
+ usage, entire = cut_short[row]["usage"], whole[row]["usage"]
+ assert (
+ usage["prompt_tokens"] == entire["prompt_tokens"]
+ ), "a stopped row read a different prompt than the same row read whole"
+ assert len(seen) <= usage["completion_tokens"] < entire["completion_tokens"], (
+ f"row {row} delivered {len(seen)} snapshots and reported "
+ f"{usage['completion_tokens']} of {entire['completion_tokens']} tokens"
+ )
+
+
import numpy as np # noqa: E402
from core.inference import grammar_constraint as gc
diff --git a/studio/backend/tests/test_mlx_resident_batch.py b/studio/backend/tests/test_mlx_resident_batch.py
new file mode 100644
index 00000000000..2dfa88027a1
--- /dev/null
+++ b/studio/backend/tests/test_mlx_resident_batch.py
@@ -0,0 +1,438 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Replies joining and leaving a batch the MLX worker keeps open."""
+
+import functools
+import importlib
+import inspect
+import multiprocessing as _mp
+import queue as _queue
+import sys
+import threading
+import types
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - stub unusable imports
+ pass
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+
+from core.inference import worker # noqa: E402
+from core.inference.worker import RowRefused # noqa: E402
+from core.inference.worker import StopLedger # noqa: E402
+
+
+class _RespQueue:
+ def __init__(self):
+ self.sent = []
+
+ def put(self, item, *a, **k):
+ self.sent.append(item)
+
+
+class _Session:
+ """A batch that hands back whatever the test scripted for each reply."""
+
+ def __init__(self, *, width):
+ self.opened_at = width
+ self.rows, self.ending, self.refusals = [], [], []
+ self.settled, self.script, self.stats_at_retire = {}, {}, {}
+ self.will_not_take, self.closed, self.admitted = set(), False, []
+
+ @property
+ def rows_in_flight(self):
+ return len(self.rows)
+
+ @property
+ def handles(self):
+ return list(self.rows)
+
+ def take_stats(self, handle):
+ return self.settled.pop(handle, None)
+
+ def admit(self, request, handle):
+ if handle[0] in self.will_not_take or handle in self.will_not_take:
+ self.refusals.append(handle)
+ raise RowRefused("it does not prepare like the replies in the batch")
+ self.admitted.append((handle, request))
+ self.rows.append(handle)
+ return ""
+
+ def _retire(self, handle):
+ self.rows.remove(handle)
+ self.settled[handle] = self.stats_at_retire.get(handle, {})
+
+ def step(self):
+ for handle in list(self.rows):
+ events = self.script.get(handle) or []
+ if not events:
+ continue
+ event = events.pop(0)
+ if event is None:
+ self._retire(handle)
+ yield handle, event
+
+ def withdraw(self, handles):
+ for handle in [h for h in handles if h in self.rows]:
+ self._retire(handle)
+ yield handle, None
+
+ def close(self):
+ self.closed = True
+
+
+class _Backend:
+ def __init__(
+ self,
+ reason = None,
+ script = None,
+ ):
+ self.reason = reason
+ self.script = script or {}
+ self.sessions = []
+ self.on_open = None
+ self.on_reason = None
+ self.apart = []
+
+ def generate_chat_response(self, **kwargs):
+ self.apart.append(kwargs)
+ yield "apart"
+
+ def resident_unavailable_reason(self, request):
+ if self.on_reason is not None:
+ self.on_reason()
+ return self.reason
+
+ def open_resident_batch(
+ self,
+ *,
+ width,
+ adapter_state = None,
+ ):
+ session = _Session(width = width)
+ session.script.update(self.script)
+ if self.on_open is not None:
+ self.on_open(session)
+ self.sessions.append(session)
+ return session
+
+
+def _cmd(request_id = "r1", **extra):
+ return {"type": "generate", "request_id": request_id, "messages": [], **extra}
+
+
+def _batch(backend = None):
+ resp = _RespQueue()
+ return worker._ResidentBatch(backend or _Backend(), resp), resp
+
+
+def test_replies_a_command_asked_for_together_are_addressed_and_counted_by_row():
+ batch, resp = _batch()
+ batch.admit(_cmd(rows = [{"seed": 1}, {"seed": 2}]), None)
+ for row, stats in ((0, {"n": 1}), (1, {"n": 2})):
+ batch.session.script[("r1", row)] = ["ab"[row], None]
+ batch.session.stats_at_retire[("r1", row)] = stats
+ for _ in range(2):
+ batch.step()
+
+ rows = [(m["type"], m.get("row")) for m in resp.sent]
+ assert ("token", 0) in rows and ("token", 1) in rows
+ assert rows.count(("row_done", 0)) == 1 and rows.count(("row_done", 1)) == 1
+ done = [m for m in resp.sent if m["type"] == "row_done"]
+ assert sorted(m["stats"]["n"] for m in done) == [1, 2]
+ assert [m["type"] for m in resp.sent][-1] == "gen_done"
+ assert resp.sent[-1]["stats"] is None
+
+
+IDLE = object()
+
+
+def _run_loop(monkeypatch, backend, cmds):
+ """Drive the real worker loop over a scripted queue."""
+ import queue as _stdqueue
+
+ from utils.hardware import hardware as _hw
+ from loggers.config import LogConfig
+ from utils import worker_stderr
+ import core.inference.mlx_inference as mlx_mod
+
+ monkeypatch.setenv("ENVIRONMENT_TYPE", "development")
+ inert = lambda *a, **k: None # noqa: E731
+ for name, value in (
+ ("is_apple_silicon", lambda: True),
+ ("apply_gpu_ids", inert),
+ ("_recorded_local_base", lambda m: (None, False)),
+ ("_hub_targets_are_local", lambda *a, **k: True),
+ ("_activate_transformers_version", inert),
+ ("_handle_load", inert),
+ ):
+ monkeypatch.setattr(worker, name, value)
+ monkeypatch.setattr(_hw, "detect_hardware", inert)
+ monkeypatch.setattr(_hw, "DEVICE", _hw.DeviceType.MLX)
+ monkeypatch.setattr(LogConfig, "setup_logging", staticmethod(lambda *a, **k: None))
+ monkeypatch.setattr(worker_stderr, "mark_log_record_continuations", inert)
+ monkeypatch.setattr(mlx_mod, "MLXInferenceBackend", lambda *a, **k: backend)
+
+ class _Script:
+ def __init__(self, items):
+ self._items = list(items)
+
+ def get(self, timeout = None):
+ if self._items and (item := self._items.pop(0)) is not IDLE:
+ return item
+ raise _stdqueue.Empty
+
+ resp = _RespQueue()
+ never = SimpleNamespace(is_set = lambda: False, clear = lambda: None, set = lambda: None)
+ worker.run_inference_process(
+ cmd_queue = _Script([*cmds, {"type": "shutdown"}]),
+ resp_queue = resp,
+ cancel_event = never,
+ config = {"model_name": "unsloth/orpheus-3b-0.1-ft"},
+ )
+ return resp
+
+
+def test_a_one_slot_load_runs_its_reply_without_ever_opening_a_batch(monkeypatch):
+ """A batch one reply wide is one nothing can join, so it is never opened."""
+ backend = _Backend()
+
+ resp = _run_loop(monkeypatch, backend, [_cmd("r1", parallel_slots = 1), IDLE])
+
+ assert backend.sessions == [], "a batch was opened for a single reply"
+ assert [m["type"] for m in resp.sent if m.get("request_id") == "r1"] == ["token", "gen_done"]
+
+
+_HOLD_CASES = [
+ pytest.param(
+ dict(
+ cmds = [("r1", 4), ("r2", 2), ("r3", 4), ("r4", 4)],
+ script = {"r1": ["a", None], "r2": ["b", None]},
+ widths = [4, 2, 4],
+ order = ["r1", "r2", "r3", "r4"],
+ ),
+ id = "the head runs on a queue that never falls quiet",
+ ),
+ pytest.param(
+ dict(
+ cmds = [("r1", 4), ("r2", 2), ("r3", 2), ("r4", 2)],
+ script = {
+ "r1": ["a"] * 4 + [None],
+ "r2": ["b", "b", "b", None],
+ "r3": ["c", "c", None],
+ "r4": ["d", None],
+ },
+ idle = 12,
+ widths = [4, 2, 2],
+ order = ["r1", "r2", "r3", "r4"],
+ ),
+ id = "everything held behind one incompatible command still batches, within the width",
+ ),
+ pytest.param(
+ dict(
+ cmds = [("r1", 2), ("r2", 2), ("r3", 2)],
+ script = {"r1": ["a"] * 4 + [None], "r2": ["b", "b", None], "r3": ["c", None]},
+ refuse = ("r2", 0),
+ idle = 12,
+ refusals = (0, "r2"),
+ order = ["r1", "r2", "r3"],
+ ),
+ id = "a command the batch turned away is not offered to it again",
+ ),
+ pytest.param(
+ dict(
+ cmds = [("r1", 4), ("r0", 2), ("r2", 2), ("r3", 2)],
+ script = {
+ "r1": ["a"] * 5 + [None],
+ "r0": ["x", "x", "x", None],
+ "r2": ["b", None],
+ "r3": ["c", None],
+ },
+ refuse = ("r2", 1),
+ idle = 16,
+ refusals = (1, "r2"),
+ order = ["r1", "r0", "r2", "r3"],
+ ),
+ id = "a head the batch turns away keeps its place over what waited behind it",
+ ),
+]
+
+
[email protected]("case", _HOLD_CASES)
+def test_the_hold_releases_its_head_and_keeps_its_order(monkeypatch, case):
+ """The hold's whole contract, over the traffic that breaks each part of it."""
+ backend = _Backend()
+ backend.script = {(name, None): events for name, events in case["script"].items()}
+ if "refuse" in case:
+ who, when = case["refuse"]
+ backend.on_open = lambda session: (
+ session.will_not_take.add(who) if len(backend.sessions) == when else None
+ )
+
+ resp = _run_loop(
+ monkeypatch,
+ backend,
+ [_cmd(name, parallel_slots = width) for name, width in case["cmds"]]
+ + [IDLE] * case.get("idle", 0),
+ )
+
+ admitted = [handle[0] for session in backend.sessions for handle, _r in session.admitted]
+ assert admitted == case["order"], f"the hold lost or reordered a command: {admitted}"
+ if "widths" in case:
+ assert [session.opened_at for session in backend.sessions] == case["widths"]
+ if "refusals" in case:
+ index, name = case["refusals"]
+ assert backend.sessions[index].refusals == [(name, None)]
+ if case.get("idle"):
+ terminal = [
+ m["request_id"] for m in resp.sent if m.get("type") in ("gen_done", "gen_error")
+ ]
+ assert sorted(terminal) == sorted(case["order"]), f"a held reply was lost: {resp.sent}"
+
+
+class _DecliningBackend:
+ def __init__(self, reason):
+ self._reason, self.batched = reason, 0
+ self.last_generation_stats = {"usage": {"completion_tokens": 2}}
+
+ def batch_unavailable_reason(self, requests):
+ return None if len(requests) < 2 else self._reason
+
+ def generate_chat_response(
+ self,
+ seed = None,
+ **kwargs,
+ ):
+ yield f"seed {seed}"
+ yield f"seed {seed} done"
+
+ def generate_chat_batch(self, requests, **kwargs):
+ self.batched += 1
+ raise AssertionError("a declined request set must not reach the batch")
+
+
+def _run(
+ reason,
+ cancelled = False,
+ backend = None,
+):
+ from core.inference import worker
+
+ sent, cancel = [], threading.Event()
+ backend = backend or _DecliningBackend(reason)
+ if cancelled:
+ cancel.set()
+ worker._handle_generate_rows(
+ backend,
+ {
+ "request_id": "r1",
+ "messages": [{"role": "user", "content": "hi"}],
+ "rows": [{"seed": 1}, {"seed": 2}],
+ "parallel_slots": 2,
+ },
+ SimpleNamespace(put = sent.append),
+ cancel,
+ )
+ return backend, sent
+
+
+def test_a_declined_request_set_is_decoded_one_reply_at_a_time():
+ """Reporting the command done without decoding would answer a caller's two."""
+ backend, sent = _run("a reply asks for stop sequences")
+ kinds = [event["type"] for event in sent if event["type"] != "batch_state"]
+ tokens = [event for event in sent if event["type"] == "token"]
+
+ assert backend.batched == 0
+ assert kinds == ["token", "token", "row_done", "token", "token", "row_done", "gen_done"]
+ assert [(event["row"], event["text"]) for event in tokens] == [
+ (0, "seed 1"),
+ (0, "seed 1 done"),
+ (1, "seed 2"),
+ (1, "seed 2 done"),
+ ], "each row decodes with the request it was given, not with row zero's"
+ assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]
+ assert all(event["request_id"] == "r1" for event in sent)
+ assert all(
+ event["stats"] == backend.last_generation_stats
+ for event in sent
+ if event["type"] == "row_done"
+ )
+
+
+def test_a_cancelled_fallback_still_reports_every_row_done():
+ """A row nobody decoded is still a row somebody is waiting on."""
+ _backend, sent = _run("a reply asks for stop sequences", cancelled = True)
+
+ assert [event["type"] for event in sent if event["type"] != "batch_state"] == [
+ "row_done",
+ "row_done",
+ "gen_done",
+ ]
+ assert [event["row"] for event in sent if event["type"] == "row_done"] == [0, 1]
+ assert all(event["stats"] is None for event in sent if event["type"] == "row_done")
+
+
+def test_a_batch_that_will_not_open_still_answers_the_reply(monkeypatch):
+ """Building the batch is what fails, and the reply predates any batch existing."""
+
+ def refuse(**_kwargs):
+ raise RuntimeError("BatchGenerator.insert missing")
+
+ backend = _Backend()
+ backend.open_resident_batch = refuse
+ resp = _run_loop(monkeypatch, backend, [_cmd(parallel_slots = 2), IDLE])
+
+ assert [m["type"] for m in resp.sent if m["type"] != "status"] == ["token", "gen_done"]
+ assert len(backend.apart) == 1, "it was decoded one reply at a time"
+
+
+def test_a_row_override_the_backend_cannot_take_is_dropped_not_raised():
+ """The route varies the seed per row, and the fallback decodes on backends without one."""
+ InferenceBackend = pytest.importorskip("core.inference.inference").InferenceBackend
+
+ class _NoSeedBackend(_DecliningBackend):
+ """Transformers-shaped: wraps() carries the real signature, which declares no seed."""
+
+ @functools.wraps(InferenceBackend.generate_chat_response)
+ def generate_chat_response(self, *args, **kwargs):
+ inspect.signature(InferenceBackend.generate_chat_response).bind(self, *args, **kwargs)
+ yield "reply"
+
+ _backend, sent = _run(None, backend = _NoSeedBackend("a reply asks for stop sequences"))
+ kinds = [event["type"] for event in sent if event["type"] != "batch_state"]
+
+ assert kinds == ["token", "row_done", "token", "row_done", "gen_done"], f"{sent}"
+
+
+def test_a_count_is_refused_while_the_batch_decodes(monkeypatch):
+ """Dispatched generations hold no orchestrator lock, so the loop keeps its promise."""
+ backend = _Backend(script = {("r1", None): ["a", None]})
+ backend.count_chat_tokens, backend.active_model_name = (lambda *a, **k: 7), "m"
+ count = {"type": "count_tokens", "request_id": "c1", "messages": []}
+ cmds = [_cmd("r1", parallel_slots = 2), count, IDLE, IDLE, IDLE, count, IDLE]
+
+ sent = _run_loop(monkeypatch, backend, cmds).sent
+ counts = [m for m in sent if m["type"] == "count_tokens_response"]
+
+ assert [m.get("error") for m in counts] == [
+ "A generation is in progress.",
+ None,
+ ], f"the count was served mid-decode, or never ran once the batch drained: {counts}"
+ assert counts[1]["input_tokens"] == 7
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 17b25dc70c4..4d7aa9386e6 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -7410,7 +7410,7 @@ def test_saved_parallel_slots_reach_an_api_load(monkeypatch):
assert rec.calls[0].n_parallel == 8
-def test_parallel_slots_are_stored_and_gated_on_gguf():
+def test_parallel_slots_are_stored_and_reach_either_backend():
override = settings.normalize_model_override({"n_parallel": 8})
assert override == {"n_parallel": 8}
# Blank, out of range and non-integer all mean "follow the server-wide default".
@@ -7419,9 +7419,13 @@ def test_parallel_slots_are_stored_and_gated_on_gguf():
gguf = settings.model_override_load_kwargs(override, is_gguf = True)
assert gguf["n_parallel"] == 8
- # A safetensors load has no llama-server slots, exactly as the picker gates it.
- assert "n_parallel" not in settings.model_override_load_kwargs(override, is_gguf = False)
+ safetensors = settings.model_override_load_kwargs(override, is_gguf = False)
+ assert safetensors["n_parallel"] == 8
+ for flag in ("n_batch", "n_ubatch"):
+ stored = settings.normalize_model_override({flag: 512})
+ assert flag not in settings.model_override_load_kwargs(stored, is_gguf = False)
LoadRequest(model_path = "unsloth/B-GGUF", **gguf)
+ LoadRequest(model_path = "unsloth/B", **safetensors)
def test_override_route_persists_parallel_slots(override_store):
@@ -10047,11 +10051,13 @@ def test_auto_switch_loads_an_unloaded_mlx_model(monkeypatch):
"unsloth/Qwen3-MLX", "/srv/models/Qwen3-MLX", (), is_gguf = False
)
monkeypatch.setattr(resolver, "_scan", (time.monotonic(), {"unsloth/qwen3-mlx": entry}))
+ settings.set_model_override("unsloth/Qwen3-MLX", n_parallel = 8)
_run_hook("unsloth/Qwen3-MLX")
assert [c.model_path for c in calls] == ["/srv/models/Qwen3-MLX"]
assert calls[0].gguf_variant is None
+ assert calls[0].n_parallel == 8
# The alias lands on the orchestrator, leaving the llama.cpp backend untouched.
assert orchestrator._openai_advertised_id == "unsloth/Qwen3-MLX"
assert getattr(llama, "_openai_advertised_id", None) is None
diff --git a/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py b/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py
index 37d986a416d..a78cf864a45 100644
--- a/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py
+++ b/studio/backend/tests/test_orchestrator_idle_subprocess_teardown.py
@@ -17,8 +17,11 @@ from core.inference.orchestrator import InferenceOrchestrator
def _idle_orchestrator(models, loading = ()):
"""An orchestrator whose unload round-trip is stubbed to succeed."""
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._stop_ledger = None
+ o._pending_teardowns = None
o._gen_lock = threading.Lock()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
o._drain_event = threading.Event()
o._unload_pending = False
o.models = dict(models)
@@ -29,7 +32,7 @@ def _idle_orchestrator(models, loading = ()):
o.cancel_load = lambda _name: False
o._ensure_subprocess_alive = lambda: True
o._cancel_generation = lambda: None
- o._wait_dispatcher_idle = lambda: True
+ o._wait_worker_idle = lambda timeout = None: True
o._drain_queue = lambda: None
o._send_cmd = lambda _cmd: None
o._wait_response = lambda _token: None
diff --git a/studio/backend/tests/test_orchestrator_load_cancel_event.py b/studio/backend/tests/test_orchestrator_load_cancel_event.py
index d6f9766a885..2b14c8a8a28 100644
--- a/studio/backend/tests/test_orchestrator_load_cancel_event.py
+++ b/studio/backend/tests/test_orchestrator_load_cancel_event.py
@@ -10,6 +10,8 @@ from core.inference.orchestrator import InferenceOrchestrator
def test_safetensors_load_event_interrupts_worker_wait(monkeypatch):
orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orchestrator._stop_ledger = None
+ orchestrator._pending_teardowns = None
orchestrator.__dict__.update(
loading_models = set(), active_model_name = None, models = {}, _proc = None, _resp_queue = object()
)
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index 491c998a45d..9b90f1039bb 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -6,6 +6,7 @@ hanging the UI. ``unload_model`` now cancels first (the mp.Event the worker chec
each token) and takes ``_gen_lock`` before the unload round-trip.
"""
+import inspect
import queue
import threading
import time
@@ -27,9 +28,26 @@ class _Unsloth:
return None # no Unsloth load in flight -> Unsloth fast path skipped
-def _bare_orchestrator():
+class _RecordOfStops:
+ """Stands in for the ledger; only whether the worker reads it matters here."""
+
+ def __init__(self, read_by_worker: bool):
+ self.stopped: list = []
+ self._read_by_worker = read_by_worker
+
+ def stop(self, request_id):
+ self.stopped.append(request_id)
+ return True
+
+ def read_by_worker(self) -> bool:
+ return self._read_by_worker
+
+
+def _bare_orchestrator(reads_stops = False):
"""An orchestrator without the real __init__ subprocess/network."""
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._stop_ledger = _RecordOfStops(True) if reads_stops else None
+ o._pending_teardowns = None
o._gen_lock = threading.Lock()
o._send_order_lock = threading.Lock()
o._subprocess_shutdown_lock = threading.Lock()
@@ -44,12 +62,13 @@ def _bare_orchestrator():
o._dispatcher_thread = None
o._dispatcher_stop = threading.Event()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
+ o._unload_pending = False
+ o._worker_reserved_for = None
o._mailbox_lock = threading.Lock()
o._mailboxes = {}
o._direct_mailboxes = {}
o._request_cancel_events = {}
- o._unload_pending = False
- o._exclusive_tts_pending = False
o.active_model_name = "m"
o.models = {"m": {}}
o.loading_models = set()
@@ -117,7 +136,9 @@ def test_worker_closes_cancelled_generator_before_gen_done():
cancel,
)
- assert [item["type"] for item in responses.items] == ["gen_done"]
+ assert [item["type"] for item in responses.items if item["type"] != "batch_state"] == [
+ "gen_done"
+ ]
def test_unload_cancels_inflight_generation_then_unloads(monkeypatch):
@@ -176,7 +197,13 @@ def test_unload_falls_back_to_shutdown_when_generation_wont_yield(monkeypatch):
monkeypatch.setattr(orch_mod, "_UNLOAD_GEN_LOCK_TIMEOUT", 0.2)
shutdown = []
monkeypatch.setattr(o, "_shutdown_subprocess", lambda timeout = 5: shutdown.append(timeout))
- monkeypatch.setattr(o, "_send_cmd", lambda cmd: pytest.fail("must not send unload when wedged"))
+ monkeypatch.setattr(
+ o,
+ "_send_cmd",
+ lambda cmd: pytest.fail("must not send unload when wedged")
+ if cmd.get("type") == "unload"
+ else None,
+ )
# A wedged worker never releases _gen_lock, even after the cancel.
o._gen_lock.acquire()
@@ -210,7 +237,11 @@ def test_unload_tears_down_when_compare_dispatcher_wedged(monkeypatch):
monkeypatch.setattr(o, "_shutdown_subprocess", lambda timeout = 5: shutdown.append(timeout))
monkeypatch.setattr(o, "_drain_queue", lambda: [])
monkeypatch.setattr(
- o, "_send_cmd", lambda cmd: pytest.fail("must not send unload with a wedged dispatcher")
+ o,
+ "_send_cmd",
+ lambda cmd: pytest.fail("must not send unload with a wedged dispatcher")
+ if cmd.get("type") == "unload"
+ else None,
)
monkeypatch.setattr(
o,
@@ -743,7 +774,13 @@ def test_unload_clears_drain_event_even_on_wedged_teardown(monkeypatch):
o = _bare_orchestrator()
monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)
monkeypatch.setattr(orch_mod, "_UNLOAD_GEN_LOCK_TIMEOUT", 0.2)
- monkeypatch.setattr(o, "_send_cmd", lambda cmd: pytest.fail("must not send when wedged"))
+ monkeypatch.setattr(
+ o,
+ "_send_cmd",
+ lambda cmd: pytest.fail("must not send when wedged")
+ if cmd.get("type") == "unload"
+ else None,
+ )
# A wedged worker never releases _gen_lock; unload tears the subprocess down. The
# real teardown nulls _drain_event, so emulate that so the finally exercises its guard.
@@ -761,6 +798,20 @@ def test_unload_clears_drain_event_even_on_wedged_teardown(monkeypatch):
# ----------------------------------------------------------------------------
+def _announcing_lock(lock, reached):
+ """The held lock, saying so when someone starts waiting for it."""
+
+ class _Announcing:
+ def __enter__(self):
+ reached.set()
+ return lock.__enter__()
+
+ def __exit__(self, *exc):
+ return lock.__exit__(*exc)
+
+ return _Announcing()
+
+
def test_generation_rechecks_model_after_lock_wait(monkeypatch):
# A request passes the pre-lock active-model check, then blocks on _gen_lock while
# an unload clears/swaps the model. Even if _unload_pending was already reset (the
@@ -773,13 +824,11 @@ def test_generation_rechecks_model_after_lock_wait(monkeypatch):
)
reached_lock = threading.Event()
- # _wait_dispatcher_idle runs after the pre-lock check and before acquiring the lock;
- # signalling here means the generator captured the model and is about to block.
- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda: (reached_lock.set(), True)[1])
-
o.active_model_name = "m"
o._unload_pending = False
- o._gen_lock.acquire() # stand in for an in-flight unload holding the lock
+ held = o._gen_lock
+ held.acquire() # stand in for an in-flight unload holding the lock
+ o._gen_lock = _announcing_lock(held, reached_lock)
out: list = []
@@ -791,7 +840,7 @@ def test_generation_rechecks_model_after_lock_wait(monkeypatch):
assert reached_lock.wait(timeout = 5)
# Unload finished: model swapped, pending already cleared. Release the lock.
o.active_model_name = "other"
- o._gen_lock.release()
+ held.release()
t.join(timeout = 5)
assert out and any("unloaded" in chunk.lower() for chunk in out)
@@ -805,11 +854,11 @@ def test_generation_rechecks_model_when_unloaded_to_none(monkeypatch):
o, "_send_cmd", lambda cmd: pytest.fail("must not generate after the model was unloaded")
)
reached_lock = threading.Event()
- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda: (reached_lock.set(), True)[1])
-
o.active_model_name = "m"
o._unload_pending = False
- o._gen_lock.acquire()
+ held = o._gen_lock
+ held.acquire()
+ o._gen_lock = _announcing_lock(held, reached_lock)
out: list = []
t = threading.Thread(
@@ -818,7 +867,7 @@ def test_generation_rechecks_model_when_unloaded_to_none(monkeypatch):
t.start()
assert reached_lock.wait(timeout = 5)
o.active_model_name = None
- o._gen_lock.release()
+ held.release()
t.join(timeout = 5)
assert out and any("unloaded" in chunk.lower() for chunk in out)
@@ -1487,13 +1536,11 @@ def test_stale_unload_does_not_hide_an_update_refusal():
# ----------------------------------------------------------------------------
# A dispatched (compare-mode) request that races an unload must not orphan its
-# mailbox after _wait_dispatcher_idle stops the dispatcher.
# ----------------------------------------------------------------------------
def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypatch):
# The request passes the pre-work _unload_pending check, then an unload sets
- # _unload_pending and _wait_dispatcher_idle stops the dispatcher (mailboxes empty)
# before this request registers its mailbox. The recheck under _mailbox_lock must
# make it bail, or the worker's skipped-generate reply has nothing to route it and
# the compare stream hangs on an orphaned mailbox.
@@ -1506,7 +1553,6 @@ def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypa
monkeypatch.setattr(o, "_start_dispatcher", lambda: None)
# Flip the unload flag after the pre-work check (626) but before mailbox
- # registration -- exactly the window _wait_dispatcher_idle exploits.
def flip(*a, **k):
o._unload_pending = True
return {"type": "generate", "request_id": "r1"}
@@ -1568,8 +1614,7 @@ def test_dispatched_bails_when_model_swapped_before_mailbox_registration(monkeyp
def test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(monkeypatch):
# Same window, but the unload was a same-model reload so active_model_name is
- # unchanged; the give-away is that the dispatcher was stopped. Registering a
- # mailbox with no dispatcher to route the reply would hang the compare stream.
+ monkeypatch.setattr(orch_mod, "_DISPATCH_IDLE_TIMEOUT", 0.05)
o = _bare_orchestrator()
o._mailbox_lock = threading.Lock()
o._mailboxes = {}
@@ -1590,7 +1635,7 @@ def test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(mo
out = list(o._generate_dispatched(messages = [{"role": "user", "content": "hi"}]))
- assert any("unloaded" in chunk.lower() for chunk in out)
+ assert any("nothing is routing replies" in chunk.lower() for chunk in out), out
assert o._mailboxes == {}, "must not leave an orphaned mailbox"
@@ -2243,9 +2288,9 @@ def test_dispatched_bail_stops_orphan_dispatcher_it_started(monkeypatch):
def fake_start():
started["v"] = True
o._dispatcher_thread = _AliveDispatcher()
- return True # _start_dispatcher returns True for the caller that spawned it
+ return o._dispatcher_thread # what _start_dispatcher hands the caller that spawned it
- def fake_stop():
+ def fake_stop(thread = None):
stopped["v"] = True
o._dispatcher_thread = None
@@ -2446,6 +2491,7 @@ def test_concurrent_start_dispatcher_spawns_exactly_one():
o._dispatcher_thread = None
o._dispatcher_stop = threading.Event()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
n = 32
# A barrier aligns every thread on the check-then-spawn window: without the lifecycle
@@ -2467,10 +2513,9 @@ def test_concurrent_start_dispatcher_spawns_exactly_one():
t.join(timeout = 5)
try:
- # _start_dispatcher returns True only for the caller that actually spawned a thread.
- # Exactly one caller may win; every other must observe the dispatcher alive and bail.
- assert results.count(True) == 1, f"expected exactly one spawn, got {results.count(True)}"
- assert results.count(False) == n - 1
+ spawned = [result for result in results if result is not None]
+ assert len(spawned) == 1, f"expected exactly one spawn, got {len(spawned)}"
+ assert results.count(None) == n - 1
# And exactly one live dispatcher thread exists -- no orphan racing resp_queue.
live = [
t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()
@@ -2512,11 +2557,12 @@ def test_start_dispatcher_refuses_while_unload_pending():
o._dispatcher_thread = None
o._dispatcher_stop = threading.Event()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
o._unload_pending = True
started = o._start_dispatcher()
- assert started is False, "must not start a dispatcher while an unload is pending"
+ assert started is None, "must not start a dispatcher while an unload is pending"
assert o._dispatcher_thread is None, "no dispatcher thread may be created"
live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()]
assert live == [], "no dispatcher may exist to consume the unloaded reply"
@@ -2534,11 +2580,12 @@ def test_start_dispatcher_resumes_after_unload_clears():
o._dispatcher_thread = None
o._dispatcher_stop = threading.Event()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
o._unload_pending = False
try:
assert (
- o._start_dispatcher() is True
+ o._start_dispatcher() is o._dispatcher_thread is not None
), "a fresh dispatcher must start once no unload is pending"
assert o._dispatcher_thread is not None and o._dispatcher_thread.is_alive()
finally:
@@ -2564,6 +2611,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():
o._request_cancel_events = {}
o._dispatcher_stop = threading.Event()
o._dispatcher_lifecycle_lock = threading.Lock()
+ o._worker_released = threading.Condition(o._dispatcher_lifecycle_lock)
o._unload_pending = False
start_queued = threading.Event() # release the stop's join once the start is queued behind it
@@ -2584,7 +2632,6 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():
def unload_side():
# unload_model's sequence: set _unload_pending under the lifecycle lock, then stop
- # the idle dispatcher (also under the lock, via _wait_dispatcher_idle).
with o._dispatcher_lifecycle_lock:
o._unload_pending = True
o._stop_dispatcher()
@@ -2611,7 +2658,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher():
u.join(timeout = 5)
c.join(timeout = 5)
- assert started_result.get("v") is False, "the queued start must refuse while unloading"
+ assert started_result.get("v") is None, "the queued start must refuse while unloading"
assert o._dispatcher_thread is None, "the stop cleared it and the queued start spawned nothing"
live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()]
assert live == [], "no fresh dispatcher may be left to consume the unloaded reply"
@@ -2889,7 +2936,6 @@ def test_generation_stopped_while_queued_is_never_sent(monkeypatch):
# gen_done without one) still held up its siblings.
o = _bare_orchestrator()
monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True)
- monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda *a, **k: None)
monkeypatch.setattr(
o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped")
)
@@ -2960,6 +3006,74 @@ def test_a_scoped_load_cancel_that_never_reports_back_releases_the_load():
inf._scoped_load_cancel_tombstones.clear()
+def test_a_cancelled_row_stream_shows_no_token_a_single_reply_would_have_dropped():
+ o = _bare_orchestrator()
+ cancel = threading.Event()
+ cancel.set()
+ events = [
+ {"type": "token", "request_id": "r1", "row": 0, "text": "buffered"},
+ {"type": "row_done", "request_id": "r1", "row": 0},
+ {"type": "row_done", "request_id": "r1", "row": 1},
+ {"type": "gen_done", "request_id": "r1"},
+ ]
+
+ seen = list(
+ o._consume_token_stream(
+ lambda timeout: events.pop(0) if events else None,
+ lambda: None,
+ crash_context = "generation",
+ request_id = "r1",
+ cancel_event = cancel,
+ mark_started = False,
+ rows = 2,
+ )
+ )
+
+ assert seen == [(0, None), (1, None)]
+
+
+def _real_send_order_lock():
+ """The send-order lock exactly as InferenceOrchestrator.__init__ makes it."""
+ source = inspect.getsource(InferenceOrchestrator.__init__)
+ line = next(one for one in source.splitlines() if "self._send_order_lock = " in one)
+ return eval(line.split("=", 1)[1].strip(), {"threading": threading})
+
+
+def test_a_recovering_caller_resets_the_worker_without_deadlocking_on_itself():
+ """The reset decides under the send-order lock and then sends under it."""
+ from types import SimpleNamespace
+
+ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ backend._send_order_lock = _real_send_order_lock()
+ backend._mailbox_lock = threading.Lock()
+ backend._request_cancel_events = {}
+ backend._cancel_event = threading.Event()
+ backend._proc = SimpleNamespace(is_alive = lambda: True)
+ sent = []
+ backend._send_cmd = sent.append
+ backend._teardown_going_out = backend._teardown_not_sent = lambda: None
+ backend._owns_worker = lambda event: True
+ done = threading.Event()
+
+ def reset():
+ backend.reset_generation_state(threading.Event())
+ done.set()
+
+ worker = threading.Thread(target = reset, daemon = True)
+ worker.start()
+ worker.join(timeout = 5)
+
+ assert done.is_set(), "the reset deadlocked on the lock it already held"
+ assert sent == [{"type": "reset"}]
+
+
+def test_the_idle_wait_is_bounded_when_no_timeout_is_asked_for():
+ default = (
+ inspect.signature(InferenceOrchestrator._wait_worker_idle).parameters["timeout"].default
+ )
+ assert isinstance(default, float) and default > 0, default
+
+
def test_shutdown_cancels_loads_that_have_not_reached_the_backend():
"""A /load between admission and the backend call holds nothing the backend's
shutdown flag can see: it can sit in the lifecycle gate or preflight for
@@ -3382,6 +3496,9 @@ def test_a_shutdown_that_begins_during_the_spawn_reaps_the_new_worker():
class _Ctx:
Queue = staticmethod(lambda: mock.Mock())
Event = staticmethod(lambda: mock.Mock())
+ Lock = staticmethod(lambda: mock.Mock())
+ Array = staticmethod(lambda *a, **k: mock.Mock())
+ Value = staticmethod(lambda *a, **k: mock.Mock())
@staticmethod
def Process(**kw):
@@ -3859,3 +3976,16 @@ def test_both_generation_paths_forward_the_image_ordinal():
assert "image_ordinal = image_ordinal" in inspect.getsource(method), (
f"{method.__name__} accepts the ordinal but does not put it on the " "worker command"
)
+
+
+def test_a_count_during_a_dispatched_generation_is_refused_at_once():
+ o = _bare_orchestrator(reads_stops = True)
+ o._mailboxes["running"] = queue.Queue()
+ sent = []
+ o._send_cmd = sent.append
+
+ with pytest.raises(RuntimeError, match = "generation is in progress"):
+ o.count_chat_tokens([{"role": "user", "content": "hi"}])
+
+ assert sent == [], "the count reached a worker that reads no command until the reply ends"
+ assert o._gen_lock.acquire(blocking = False), "the refusal kept the generation lock"
diff --git a/studio/backend/tests/test_parallel_slots_per_load.py b/studio/backend/tests/test_parallel_slots_per_load.py
index bd56d419759..e2bae9045e3 100644
--- a/studio/backend/tests/test_parallel_slots_per_load.py
+++ b/studio/backend/tests/test_parallel_slots_per_load.py
@@ -13,6 +13,7 @@ wiring behind the /load, /validate and /status echoes.
from __future__ import annotations
+import importlib.util
import inspect
import re
import struct
@@ -26,10 +27,10 @@ _BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
-# Same external-dep stubs as the other llama_cpp unit tests.
-_loggers_stub = _types.ModuleType("loggers")
-_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
-sys.modules.setdefault("loggers", _loggers_stub)
+if importlib.util.find_spec("loggers") is None:
+ _loggers_stub = _types.ModuleType("loggers")
+ _loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+ sys.modules.setdefault("loggers", _loggers_stub)
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
@@ -346,7 +347,7 @@ def test_route_resolves_slots_once_before_dedupe_guard_and_load():
assert active_intent < resolved_intent < resolved_dedupe
assert resolved_dedupe < guard < load_call
# Both immutable intents and the guard share the value; resolution runs once.
- assert load_impl.count("n_parallel = _n_parallel") == 3
+ assert load_impl.count("n_parallel = _n_parallel") == 4
assert load_impl.count("_resolve_parallel_slots(request, fastapi_request)") == 1
assert "fastapi_request.app.state" not in load_impl
@@ -574,3 +575,44 @@ def test_training_guard_keeps_slots_for_an_unclassified_gguf(monkeypatch, tmp_pa
one = _guard_required_gb(monkeypatch, gguf, n_parallel = 1, diffusion = None)
many = _guard_required_gb(monkeypatch, gguf, n_parallel = 8, diffusion = None)
assert many > one
+
+
+def test_the_unsloth_load_hands_its_requested_slots_to_the_runtime():
+ """The resolved width reaches the non-GGUF load too, not only llama-server's."""
+ load_impl = _load_impl_source()
+ call = load_impl.index("asyncio.to_thread(\n backend.load_model,")
+ kwargs = load_impl[call : load_impl.index("\n )", call)]
+ assert "n_parallel = _n_parallel" in kwargs
+ assert "request.n_parallel" not in load_impl
+ assert "clamp_parallel_slots" not in load_impl
+
+
+def test_a_load_reply_carries_the_two_window_facts_and_the_width_that_selects_them():
+ """The three window facts are read as a set, so a reply carrying some answers nothing."""
+ from models.inference import LoadResponse, _InferenceRuntimeFields
+ from routes.inference import _unsloth_serving_fields
+
+ info = {
+ "context_length_enforced": True,
+ "context_unbounded_when_batched": True,
+ "parallel_slots": 4,
+ "can_batch": True,
+ }
+
+ def reply(**overrides):
+ return LoadResponse(
+ status = "loaded",
+ model = "m",
+ display_name = "m",
+ inference = {},
+ **_unsloth_serving_fields({**info, **overrides}),
+ ).model_dump()
+
+ body = reply()
+ assert (
+ body["context_length_enforced"],
+ body["context_unbounded_when_batched"],
+ body["parallel_slots"],
+ ) == (True, True, 4)
+ assert reply(can_batch = False, context_unbounded_when_batched = False)["parallel_slots"] == 1
+ assert _InferenceRuntimeFields.model_fields["context_unbounded_when_batched"].default is False
diff --git a/studio/backend/tests/test_presence_penalty.py b/studio/backend/tests/test_presence_penalty.py
index d48bb3bf078..fe5cb3a255e 100644
--- a/studio/backend/tests/test_presence_penalty.py
+++ b/studio/backend/tests/test_presence_penalty.py
@@ -206,6 +206,10 @@ def test_orchestrator_cmd_carries_all_sampling_params():
from core.inference.orchestrator import InferenceOrchestrator
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o.models, o.active_model_name = {}, None
+ o._stop_ledger = o._pending_teardowns = None
+ o._stop_ledger = None
+ o._pending_teardowns = None
cmd = o._build_generate_cmd(
"req1",
None,
@@ -273,6 +277,8 @@ def test_orchestrator_cmd_carries_the_tool_protocol_flag():
from core.inference.orchestrator import InferenceOrchestrator
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o.models, o.active_model_name = {}, None
+ o._stop_ledger = o._pending_teardowns = None
base = dict(messages = [{"role": "user", "content": "hi"}], tools = [])
assert (
o._build_generate_cmd("r", None, tool_protocol_active = True, **base)["tool_protocol_active"]
@@ -303,7 +309,7 @@ def test_the_worker_gates_the_tool_protocol_flag_on_the_backend_signature():
from core.inference import worker
- src = inspect.getsource(worker._handle_generate)
+ src = inspect.getsource(worker._generation_kwargs)
gated = src[src.index("for gated in (") : src.index("for gated in (") + 200]
assert '"tool_protocol_active"' in gated, "the flag must be gated on _backend_declares"
@@ -379,6 +385,8 @@ def test_a_video_clip_crosses_the_worker_boundary_only_to_a_backend_that_takes_i
from core.inference.worker import _handle_generate
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o.models, o.active_model_name = {}, None
+ o._stop_ledger = o._pending_teardowns = None
assert "video_base64" not in o._build_generate_cmd("r", None, messages = [])
cmd = o._build_generate_cmd(
"r", None, messages = [{"role": "user", "content": "hi"}], video_b64 = "AAAA"
@@ -438,12 +446,14 @@ def test_both_orchestrator_entry_points_forward_the_clip_into_the_command():
built = []
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o.models, o.active_model_name = {}, None
+ o._stop_ledger = o._pending_teardowns = None
o._gen_lock = threading.Lock()
o._unload_pending = False
- o._exclusive_tts_pending = False
+ o._worker_reserved_for = None
o.active_model_name = "m"
o._ensure_subprocess_alive = lambda: True
- o._wait_dispatcher_idle = lambda: None
+ o._wait_worker_idle = lambda *a, **k: True
o._start_dispatcher = lambda: False
def _build(*_args, **kwargs):
diff --git a/studio/backend/tests/test_safetensors_capability_advertise.py b/studio/backend/tests/test_safetensors_capability_advertise.py
index f4ee2590387..2902c8ac46c 100644
--- a/studio/backend/tests/test_safetensors_capability_advertise.py
+++ b/studio/backend/tests/test_safetensors_capability_advertise.py
@@ -683,6 +683,7 @@ def test_orchestrator_mirrors_chat_template_info_into_models_dict():
from core.inference.orchestrator import InferenceOrchestrator
orch = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orch._stop_ledger = None
orch.models = {}
orch.active_model_name = None
orch.loading_models = set()
@@ -738,6 +739,7 @@ def test_orchestrator_missing_chat_template_info_falls_back_to_all_false():
from routes.inference import _detect_safetensors_features
orch = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ orch._stop_ledger = None
orch.models = {}
orch.active_model_name = "unsloth/Qwen3-0.6B"
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 61302f63714..324cac6f9c7 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -82,6 +82,7 @@ class _ScriptedBackend:
self._responder = responder
self._stats = stats
self.calls: list = []
+ self.batch_calls: list = []
self.reset_count = 0
def generate_chat_response(
@@ -104,6 +105,25 @@ class _ScriptedBackend:
for snap in snapshots:
yield snap
+ def generate_chat_batch(
+ self,
+ rows,
+ *,
+ stats_holder = None,
+ **kwargs,
+ ):
+ """Every choice's first reply in one command, as the real bridge does."""
+ self.batch_calls.append({"rows": rows, "shared": kwargs})
+ reported: list = []
+ for index, row in enumerate(rows):
+ holder: dict = {}
+ for snapshot in self.generate_chat_response(stats_holder = holder, **{**kwargs, **row}):
+ yield index, snapshot
+ reported.append(holder.get("stats"))
+ yield index, None
+ if stats_holder is not None:
+ stats_holder["stats"] = reported
+
def reset_generation_state(self, caller_cancel_event = None):
self.reset_count += 1
@@ -631,24 +651,6 @@ def test_what_this_backend_can_serve_reaches_it_rather_than_being_refused(monkey
assert body["choices"][0]["message"]["content"] == "hi"
-def test_n_serves_one_full_generation_per_choice(monkeypatch):
- """Each choice is its own sampling run, as on the llama-server path: the
- backend is asked once per choice rather than one reply being copied, and the
- prompt they share is not re-counted. Two runs may sample the same text; what
- is guaranteed is that each was generated."""
- turns = iter(["first", "second"])
- backend = _ScriptedBackend(
- lambda messages, tools: [next(turns)],
- stats = {"usage": {"prompt_tokens": 7, "completion_tokens": 3}},
- )
- body = _json_body(_call(_request(n = 2), monkeypatch, backend, supports_tools = False))
- assert [c["index"] for c in body["choices"]] == [0, 1]
- assert [c["message"]["content"] for c in body["choices"]] == ["first", "second"]
- assert len(backend.calls) == 2 # a generation per choice, not one reused
- # The shared prompt is counted once; only generated tokens accumulate.
- assert _totals(body) == {"prompt_tokens": 7, "completion_tokens": 6, "total_tokens": 13}
-
-
def _totals(body):
return {k: body["usage"][k] for k in ("prompt_tokens", "completion_tokens", "total_tokens")}
@@ -1432,6 +1434,49 @@ def test_legacy_image_field_keeps_the_client_tool_catalog(monkeypatch):
assert backend.calls[0]["image"] is not None
+def test_a_turn_asking_for_several_replies_sends_them_as_one_batch(monkeypatch):
+ """One command carries every choice, each with its own seed."""
+ from routes.inference import _choice_seed
+
+ backend = _ScriptedBackend(_fixed("hi"))
+ body = _json_body(_call(_request(stream = False, n = 3, seed = 11), monkeypatch, backend))
+
+ assert len(body["choices"]) == 3
+ assert len(backend.batch_calls) == 1, "the choices did not go out together"
+ call = backend.batch_calls[0]
+ effective = [row.get("seed", call["shared"].get("seed")) for row in call["rows"]]
+ assert effective == [11, _choice_seed(11, 1), _choice_seed(11, 2)], effective
+
+
+class _StoppedAfterFirstRowBackend(_ScriptedBackend):
+ """A backend that cannot batch: rows run apart and a Stop skips the rest."""
+
+ def generate_chat_batch(
+ self,
+ rows,
+ *,
+ stats_holder = None,
+ cancel_event = None,
+ **kwargs,
+ ):
+ self.batch_calls.append({"rows": rows, "shared": kwargs})
+ yield 0, "partial"
+ cancel_event.set()
+ yield 0, None
+ for row in range(1, len(rows)):
+ yield row, None
+ if stats_holder is not None:
+ stats_holder["stats"] = [{"completion_tokens": 1}] + [None] * (len(rows) - 1)
+
+
+def test_a_stop_during_the_first_choice_returns_no_empty_choices(monkeypatch):
+ backend = _StoppedAfterFirstRowBackend(_fixed("unused"))
+ body = _json_body(_call(_request(stream = False, n = 3), monkeypatch, backend))
+
+ assert len(backend.batch_calls) == 1
+ assert [c["message"]["content"] for c in body["choices"]] == ["partial"], body["choices"]
+
+
_RF_SCHEMA = {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}
_RF_FORMAT = {"type": "json_schema", "json_schema": {"name": "c", "schema": _RF_SCHEMA}}
_MARKUP_DOC = '{"city":"<think>Paris</think>"}'
diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.py
index 8e0182793c4..29acb1072e4 100644
--- a/studio/backend/tests/test_shutdown_preserves_live_worker.py
+++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py
@@ -43,8 +43,10 @@ class _FakeProc:
def _bare_inference():
o = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ o._stop_ledger = None
+ o._pending_teardowns = None
o._subprocess_shutdown_lock = threading.Lock()
- o._stop_dispatcher = lambda: None
+ o._stop_dispatcher = lambda thread = None: None
o._cancel_generation = lambda: None
o._drain_queue = lambda: []
diff --git a/studio/backend/tests/test_stream_errors.py b/studio/backend/tests/test_stream_errors.py
index d62c635c8ba..8d272a41d58 100644
--- a/studio/backend/tests/test_stream_errors.py
+++ b/studio/backend/tests/test_stream_errors.py
@@ -219,7 +219,12 @@ class TestSurvivesTheRouteLayer:
consume = orch.InferenceOrchestrator._consume_token_stream
stream = consume(
- SimpleNamespace(_proc = None, _resp_queue = None, _mark_worker_started = lambda _event: None),
+ SimpleNamespace(
+ _proc = None,
+ _resp_queue = None,
+ _mark_worker_started = lambda _event: None,
+ _forget_request = lambda *_a: None,
+ ),
lambda _timeout: payload,
lambda: None,
crash_context = "test",
diff --git a/studio/backend/tests/test_warm_window_review_fixes.py b/studio/backend/tests/test_warm_window_review_fixes.py
index c94276ef150..b2246772caf 100644
--- a/studio/backend/tests/test_warm_window_review_fixes.py
+++ b/studio/backend/tests/test_warm_window_review_fixes.py
@@ -2227,6 +2227,7 @@ def test_an_offline_read_leaves_the_fetch_available():
started = []
backend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)
+ backend._stop_ledger = None
backend._top_models_started = False
with mock.patch.object(orch, "hf_env_offline", lambda: True):
backend._start_top_models_fetch()
@@ -2250,6 +2251,7 @@ def test_a_slow_refresh_cannot_overwrite_a_newer_default_list():
import utils.hardware.hardware as hw_mod
backend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)
+ backend._stop_ledger = None
backend._static_models_lock = threading.Lock()
backend._static_models = ["gen0"]
backend._static_models_generation = 0
@@ -2282,6 +2284,7 @@ def test_a_refresh_whose_generation_moved_mid_build_does_not_commit():
import utils.hardware.hardware as hw_mod
backend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)
+ backend._stop_ledger = None
backend._static_models_lock = threading.Lock()
backend._static_models = ["old"]
backend._static_models_generation = 1
@@ -2310,6 +2313,7 @@ def test_an_ordinary_refresh_still_happens():
import utils.hardware.hardware as hw_mod
backend = orch.InferenceOrchestrator.__new__(orch.InferenceOrchestrator)
+ backend._stop_ledger = None
backend._static_models_lock = threading.Lock()
backend._static_models = ["old"]
backend._static_models_generation = 1
diff --git a/studio/backend/utils/openai_auto_switch_settings.py b/studio/backend/utils/openai_auto_switch_settings.py
index fc1238469fc..7aabfdd22af 100644
--- a/studio/backend/utils/openai_auto_switch_settings.py
+++ b/studio/backend/utils/openai_auto_switch_settings.py
@@ -582,6 +582,7 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
("llama_extra_args", "llama_extra_args"),
("kv_cache_dtype", "cache_type_kv"),
("mlx_kv_bits", "mlx_kv_bits"),
+ ("n_parallel", "n_parallel"),
("speculative_type", "speculative_type"),
("spec_draft_n_max", "spec_draft_n_max"),
("reasoning_budget", "reasoning_budget"),
@@ -594,8 +595,6 @@ def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> di
kwargs[target] = override[source]
if is_gguf:
- if override.get("n_parallel") is not None:
- kwargs["n_parallel"] = override["n_parallel"]
if override.get("n_batch") is not None:
kwargs["n_batch"] = override["n_batch"]
if override.get("n_ubatch") is not None:
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 412ed360759..4b572ef98df 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2340,6 +2340,9 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"activeGgufVariant",
"activeModelIsLocal",
"loadedContextLength",
+ "loadedContextEnforced",
+ "loadedContextUnboundedWhenBatched",
+ "loadedParallelSlots",
"maxContextLength",
"nativeContextLength",
"loadedIsGguf",
@@ -3173,6 +3176,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
// A DSpark sidecar is ~11 GB, and Auto reaches it.
speculative_type?: string | null;
spec_draft_n_max?: number | null;
+ n_parallel?: number | null;
}): Promise<boolean> {
// Before the POST, so an abort costs nothing: no request was sent.
options?.abortSignal?.throwIfAborted();
@@ -3387,6 +3391,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
cache_type_kv: config.kvCacheDtype,
tensor_parallel: effectiveTensorParallel,
disable_vision: effectiveDisableVision,
+ n_parallel: config.nParallel ?? null,
speculative_type: effectiveSpeculativeType,
spec_draft_n_max: effectiveSpecDraftNMax,
...(candidate.kind === "gguf"
@@ -3395,7 +3400,6 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
gpu_memory_mode: effectiveGpuMemoryMode,
// A remembered manual DiffusionGemma split (0 especially) must not be refused as a full-GGUF occupant.
gpu_layers: effectiveGpuLayers,
- n_parallel: config.nParallel ?? null,
reasoning_budget: isDiffusion ? -1 : config.reasoningBudget,
reasoning_budget_message: isDiffusion
? ""
@@ -3456,6 +3460,7 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
: "",
tensor_parallel: effectiveTensorParallel,
disable_vision: effectiveDisableVision,
+ n_parallel: config.nParallel ?? null,
// GGUF-only; the split ratio is never remembered (it is bound to an exact GPU set), so
// llama.cpp's free-VRAM default stays in charge.
...(candidate.kind === "gguf"
@@ -3464,8 +3469,6 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
gpu_layers: effectiveGpuLayers,
n_cpu_moe: effectiveNCpuMoe,
gpu_ids: effectiveGpuIds ?? undefined,
- // Per-model too, or the auto-load reverts a remembered override.
- n_parallel: config.nParallel ?? null,
...(config.nBatch != null ? { n_batch: config.nBatch } : {}),
...(config.nUbatch != null ? { n_ubatch: config.nUbatch } : {}),
// Remembered like the rest of this block, or the auto-load reverts the override.
@@ -3545,13 +3548,12 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
display_name: loadResp.display_name ?? candidate.id,
is_gguf: loadResp.is_gguf ?? candidate.kind === "gguf",
});
+ const committedSlots =
+ (loadResp.is_diffusion ?? false) ? null : (config.nParallel ?? null);
if (candidate.kind === "gguf") {
// The saved Context Length, not fitMaxSeqLength: the wire value is Auto-resolved on a
// same-model reload, so pinning it turns Auto into a number the user never set.
const keepCustomCtx = resolveExplicitCtxPin(config.customContextLength);
- // Diffusion ignores --parallel, so counting slots there would mint a phantom override.
- const committedSlots =
- (loadResp.is_diffusion ?? false) ? null : (config.nParallel ?? null);
// same rule for the batch sizes
const committedNBatch =
(loadResp.is_diffusion ?? false) ? null : (config.nBatch ?? null);
@@ -3643,9 +3645,8 @@ async function autoLoadSmallestModel(options?: AutoLoadOptions): Promise<{
kvCacheDtype: loadResp.cache_type_kv ?? null,
loadedKvCacheDtype: loadResp.cache_type_kv ?? null,
...mlxRuntimeStateFrom(loadResp),
- // GGUF-only and never sent here: a staged override would be saved for a model that cannot use it.
- nParallel: null,
- loadedNParallel: null,
+ nParallel: committedSlots,
+ loadedNParallel: committedSlots,
reasoningBudget: -1,
loadedReasoningBudget: -1,
loadedReasoningBudgetRequested: -1,
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index 70d49b7afc3..f2d2d9ec8e5 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -2303,6 +2303,12 @@ export function ChatPage({
);
const contextUsage = useChatRuntimeStore((state) => state.contextUsage);
const loadedIsGguf = useChatRuntimeStore((state) => state.loadedIsGguf);
+ const loadedContextUnboundedWhenBatched = useChatRuntimeStore(
+ (state) => state.loadedContextUnboundedWhenBatched,
+ );
+ const loadedParallelSlots = useChatRuntimeStore(
+ (state) => state.loadedParallelSlots,
+ );
const loadedContextBudget = useChatRuntimeStore(
(state) => state.loadedContextBudget,
);
@@ -4259,6 +4265,8 @@ export function ChatPage({
platformChatOnlyReason,
)}
contextEnforced={loadedContextEnforced}
+ contextUnboundedWhenBatched={loadedContextUnboundedWhenBatched}
+ parallelSlots={loadedParallelSlots}
contextBudget={loadedContextBudget}
className="h-[var(--studio-chat-control-height,34px)]"
/>
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index 8fd491d0fa4..2dfe699584e 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -1439,8 +1439,6 @@ export function useChatModelRuntime() {
useChatRuntimeStore.getState().params.checkpoint;
const pendingConfig =
typeof selection !== "string" ? selection.config : undefined;
- // nativePathToken is excluded: a leased file is named by a label two files can share, and only a
- // completed load writes the lease, so adopting would keep a stale token.
if (!forceReload && !nativePathToken) {
const residentStatus = await getInferenceStatus().catch(() => null);
// Warm before reconciling the remembered GPU pick below: load-on-selection can run before any
@@ -1504,8 +1502,6 @@ export function useChatModelRuntime() {
// no saved record the caller has already run applyModelLoadConfigToRuntime(null), which resets the
// store to DEFAULT_PER_MODEL_CONFIG, and performLoad reads the store for the rest.
residentRuntimeMatchesConfig(status, comparedConfig, {
- // What the applier fills an unset field with, so the comparison is against what /load would send
- // rather than against silence.
speculativeType: readPersistedSpeculativeType(),
gpuMemoryMode: readPersistedGpuMemoryMode(),
gpuLayers: GPU_LAYERS_AUTO,
@@ -2476,8 +2472,7 @@ export function useChatModelRuntime() {
mlx_kv_bits: loadMlxKvBits ?? null,
speculative_type: loadSpeculativeType,
spec_draft_n_max: loadSpecDraftNMax,
- // GGUF-only: slots mean nothing for a transformers load.
- n_parallel: isGguf ? loadNParallel : null,
+ n_parallel: loadNParallel,
reasoning_budget:
isGguf && !targetIsDiffusion ? loadReasoningBudget : -1,
reasoning_budget_message:
@@ -2589,11 +2584,9 @@ export function useChatModelRuntime() {
const loadedSpec = normalizeSpeculativeType(
loadResponse.speculative_type,
);
- // Slots the load actually committed: non-GGUF never sends them and diffusion ignores --parallel,
- // so a click-time count would mint a phantom override.
const committedSlots =
- (loadResponse.is_gguf ?? false) &&
- !(loadResponse.is_diffusion ?? false)
+ ((loadResponse.is_gguf ?? false) && !(loadResponse.is_diffusion ?? false)) ||
+ (loadResponse.is_mlx ?? false)
? (loadNParallel ?? null)
: null;
// same rule for the batch sizes: gguf-only llama-server flags
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index dbfaf636daf..c528c2a6ca9 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -253,10 +253,9 @@ export function applyActiveModelStatusToStore(
: hydratingExistingModel)
? resolveResidentInitialConfig(checkpointId, status.gguf_variant ?? null)
: null;
- const rememberedNParallel =
- status.is_gguf && remembered?.remembered
- ? (remembered.config.nParallel ?? null)
- : null;
+ const rememberedNParallel = remembered?.remembered
+ ? (remembered.config.nParallel ?? null)
+ : null;
const rememberedNBatch =
status.is_gguf && remembered?.remembered
? (remembered.config.nBatch ?? null)
@@ -568,18 +567,14 @@ export function applyActiveModelStatusToStore(
mlxKvQuantNote: status.mlx_kv_quant_note ?? null,
}),
// Baseline only, never the control: the echo is the RESOLVED count and would pin a blank
- // "server default" control. The rollback re-sends the baseline, so without this a rollback
- // after a tab reload loses the override. Refresh on every echo: another client
- // can reload the same model with a different count.
...(seedLoadParams &&
status.requested_parallel_slots != null && {
loadedNParallel: status.requested_parallel_slots,
}),
- // A slotless model must not keep the previous GGUF's baseline, since the rollback re-sends
- // it. /status omits the echo for non-GGUF and nulls it for diffusion, so an absent field
- // on a GGUF means an older backend.
+ // A slotless load must not keep the previous model's baseline, since the rollback
+ // re-sends it. /status nulls the echo for a load that decodes one reply at a time.
...(seedLoadParams &&
- (status.is_gguf === false || status.requested_parallel_slots === null) && {
+ status.requested_parallel_slots === null && {
loadedNParallel: null,
}),
// Per-model: a change underneath this tab blanks the control like performLoad's cross-model
diff --git a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
index 08bc92a68e8..c4368950e64 100644
--- a/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
+++ b/studio/frontend/src/features/chat/lib/context-usage-bar-state.ts
@@ -25,6 +25,8 @@ export type ContextUsageBarInput = {
// MLX keeps generating past the window instead of stopping there, so it needs the
// opposite advice from llama.cpp once a conversation outgrows the limit.
isMlx?: boolean;
+ contextUnboundedWhenBatched?: boolean;
+ parallelSlots?: number | null;
/** context_length_enforced as the load reported it; null where it does not answer. */
contextEnforced?: boolean | null;
contextBudget?: number | null;
@@ -52,6 +54,8 @@ function contextLimitAdvice(
total: number,
isMlx: boolean | undefined,
enforced: boolean | null | undefined,
+ unboundedWhenBatched: boolean | undefined,
+ slots: number | null | undefined,
budget: number | null | undefined,
): ContextLimitAdvice {
if ((used / total) * 100 <= 85) return "none";
@@ -61,7 +65,13 @@ function contextLimitAdvice(
// nothing rotates and nothing stops, so neither of the other two is true of it. An
// unjudged MLX window says the same thing operationally: the probe could not build a
// cache, so none was bounded and it grows exactly as a confirmed false one does.
- if (enforced === false || (isMlx && enforced == null)) return "unenforced-limit";
+ if (
+ enforced === false ||
+ (isMlx && enforced == null) ||
+ (unboundedWhenBatched && (slots ?? 1) > 1)
+ ) {
+ return "unenforced-limit";
+ }
if (!isMlx) return "stops-at-limit";
return used > total ? "mlx-past-limit" : "mlx-near-limit";
}
@@ -88,6 +98,8 @@ export function deriveContextUsageBar({
completionTokens,
isMlx,
contextEnforced,
+ contextUnboundedWhenBatched,
+ parallelSlots,
contextBudget,
}: ContextUsageBarInput): ContextUsageBarState | null {
const limit = typeof total === "number" && total > 0 ? total : null;
@@ -138,6 +150,8 @@ export function deriveContextUsageBar({
limit,
isMlx,
contextEnforced,
+ contextUnboundedWhenBatched,
+ parallelSlots,
contextBudget,
),
};
diff --git a/studio/frontend/src/features/chat/lib/resident-config-match.ts b/studio/frontend/src/features/chat/lib/resident-config-match.ts
index 81898ea1cce..eecb3bc01b2 100644
--- a/studio/frontend/src/features/chat/lib/resident-config-match.ts
+++ b/studio/frontend/src/features/chat/lib/resident-config-match.ts
@@ -112,9 +112,6 @@ export type StandingConfigDefaults = {
type SettingCheck = {
/** Placement, which the backend rewrites wholesale on a preserved CPU fallback. */
placement?: true;
- /** One of the two fields `_mlx_runtime_settings_match` compares. The non-GGUF branch of /load
- * checks identity and those, then answers already_loaded, so nothing else here may decide
- * against a safetensors or MLX resident. */
mlxComparable?: true;
/** Placement the diffusion branch of `_runtime_matches_intent` replaces wholesale with one
* `_diffusion_manual_ngl` comparison. */
@@ -244,10 +241,6 @@ const requestedGpuMemoryMode = (
const cleanTemplate = (value: string | null | undefined): string | null =>
value?.trim() ? value : null;
-/** Mirrors the fields `_runtime_matches_intent` reloads for, plus the MLX pair
- * `_mlx_runtime_settings_match` compares. Nearly every check is unconditionally pinned: a
- * config reaches here only after `applyModelLoadConfigToRuntime` resolved each field with `??
- * null`, so an unset field asks for the default. Only `llamaExtraArgs` is optional. */
const SETTING_CHECKS: SettingCheck[] = [
{
// Resolved, not compared raw: an unset length is Auto, which the load sends as 0 for a cross-model
@@ -293,8 +286,7 @@ const SETTING_CHECKS: SettingCheck[] = [
},
{
chatOnly: true,
- // Unknown default: null against the status's resolved count is a reload, the safe direction.
- // defaultParallelSlots is the EFFECTIVE count, so a build that clamps to one slot reloads.
+ mlxComparable: true,
pinned: () => true,
agrees: (c, s, standing) =>
(c.nParallel ?? standing.parallelSlots) ===
@@ -591,8 +583,6 @@ export function residentRuntimeMatchesConfig(
const diffusion = status.is_diffusion === true;
return SETTING_CHECKS.every(
(check) =>
- // The non-GGUF branch of /load checks identity and the MLX pair, then answers already_loaded, so
- // no llama.cpp invocation field may decide against one.
(status.is_gguf === false && !check.mlxComparable) ||
(diffusion && check.chatOnly) ||
(diffusion && check.ggufPlacement) ||
@@ -602,3 +592,4 @@ export function residentRuntimeMatchesConfig(
check.agrees(config, status, standing),
);
}
+
diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx
index 8f124f4787f..d672d22ee02 100644
--- a/studio/frontend/src/features/chat/shared-composer.tsx
+++ b/studio/frontend/src/features/chat/shared-composer.tsx
@@ -1675,6 +1675,7 @@ export function SharedComposer({
: "",
tensor_parallel: effectiveTensorParallel,
disable_vision: effectiveDisableVision,
+ n_parallel: ownConfig.nParallel ?? null,
force_cancel_active:
compareStopDecision?.forceCancelActive ?? false,
...(targetIsGguf
@@ -1684,9 +1685,6 @@ export function SharedComposer({
n_cpu_moe: effectiveNCpuMoe,
tensor_split: compareLoadKnobs.splitRatio ?? undefined,
gpu_ids: effectiveSelectedGpuIds ?? undefined,
- n_parallel: ownConfig.nParallel ?? null,
- // Only when this panel has read the stored value: omitted, the load inherits it, which keeps
- // CLI-set flags working.
...(ownConfig.llamaExtraArgs !== undefined
? // biome-ignore lint/style/useNamingConvention: API schema
{ llama_extra_args: ownConfig.llamaExtraArgs ?? [] }
@@ -1747,7 +1745,7 @@ export function SharedComposer({
// Slots this compare load committed. Diffusion ignores --parallel, so a
// count there would mint a phantom override a preset carries onto a GGUF.
const committedSlots =
- targetIsGguf && !(resp.is_diffusion ?? false)
+ ((resp.is_gguf ?? false) && !(resp.is_diffusion ?? false)) || (resp.is_mlx ?? false)
? (ownConfig.nParallel ?? null)
: null;
// same rule for the batch sizes
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 4848a8c4248..9dbaaf27440 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2292,6 +2292,8 @@ type ChatRuntimeStore = {
/** Whether loadedContextLength actually bounds the cache. Null where the backend does
* not answer, which is not the same as a confirmed false. */
loadedContextEnforced: boolean | null;
+ loadedContextUnboundedWhenBatched: boolean;
+ loadedParallelSlots: number | null;
loadedContextBudget: number | null;
modelRequiresTrustRemoteCode: boolean;
supportsReasoning: boolean;
@@ -2412,8 +2414,9 @@ type ChatRuntimeStore = {
/** User --spec-draft-n-max override (null = platform default). */
specDraftNMax: number | null;
loadedSpecDraftNMax: number | null;
- /** --parallel slots override for GGUF loads (null = server default). Never re-seeded from an
- * echo: the resolved count would pin a blank control. */
+ /** User reply-width override, for either backend (null = server default). GGUF spends it
+ * on llama-server's --parallel slots; MLX on how many replies decode together.
+ * Never re-seeded from an echo: the resolved count would pin a blank control. */
nParallel: number | null;
/** Slots the last successful load sent (null = default); a rollback re-sends them so a failed
* switch cannot lose the override. */
@@ -4103,6 +4106,8 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedIsGguf: null,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextUnboundedWhenBatched: false,
+ loadedParallelSlots: null,
loadedContextBudget: null,
modelRequiresTrustRemoteCode: false,
supportsReasoning: false,
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index 54886c5a4e8..6927e7a5c3f 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -252,6 +252,7 @@ export interface LoadModelResponse {
max_context_length?: number | null;
native_context_length?: number | null;
context_length_enforced?: boolean | null;
+ context_unbounded_when_batched?: boolean;
supports_reasoning?: boolean;
reasoning_style?:
| "enable_thinking"
@@ -296,7 +297,6 @@ export interface LoadModelResponse {
gpu_ids?: number[] | null;
/** User-requested GPU placement pool before fit-time narrowing. */
requested_gpu_ids?: number[] | null;
- /** Slots the load was invoked with (else the --parallel default). Null for non-GGUF loads. */
requested_parallel_slots?: number | null;
reasoning_budget?: number;
reasoning_budget_message?: string;
@@ -305,7 +305,6 @@ export interface LoadModelResponse {
requested_reasoning_budget?: number;
// biome-ignore lint/style/useNamingConvention: API schema
requested_reasoning_budget_message?: string;
- /** Slots llama-server actually runs, after any fit-time reduction. Null for non-GGUF loads. */
parallel_slots?: number | null;
/** batch size (--batch-size) the load was invoked with; null = default */
requested_n_batch?: number | null;
@@ -376,6 +375,7 @@ export interface InferenceStatusResponse {
max_context_length?: number | null;
native_context_length?: number | null;
context_length_enforced?: boolean | null;
+ context_unbounded_when_batched?: boolean;
cache_type_kv?: string | null;
mlx_kv_bits?: number | null;
mlx_kv_bits_requested?: number | null;
@@ -409,7 +409,6 @@ export interface InferenceStatusResponse {
gpu_ids?: number[] | null;
/** User-requested GPU placement pool before fit-time narrowing. */
requested_gpu_ids?: number[] | null;
- /** Slots the active load was invoked with (else the --parallel default). Null when no GGUF model is loaded. */
requested_parallel_slots?: number | null;
reasoning_budget?: number;
reasoning_budget_message?: string;
@@ -418,7 +417,6 @@ export interface InferenceStatusResponse {
requested_reasoning_budget?: number;
// biome-ignore lint/style/useNamingConvention: API schema
requested_reasoning_budget_message?: string;
- /** Slots llama-server actually runs, after any fit-time reduction. Null when no GGUF model is loaded. */
parallel_slots?: number | null;
/** batch size (--batch-size) the active load was invoked with; null = default */
requested_n_batch?: number | null;
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 8bec2615fee..f14861a1b54 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -1078,6 +1078,61 @@ function AdvancedSettingsToggle({
);
}
+const GGUF_PARALLEL_HINT =
+ "Decode slots (--parallel) for concurrent requests. Leave blank for the server " +
+ "default. More slots share the context pool and use more VRAM.";
+
+const MLX_PARALLEL_HINT =
+ "Chat replies this model decodes at once (--parallel). Leave blank for the " +
+ "default. Replies sharing a decode finish sooner together, but each one holds " +
+ "its own context in memory for as long as it runs, and nothing reduces the " +
+ "number to fit — raise it only if the memory is there.";
+
+function ParallelSlotsRow({
+ config,
+ update,
+ hint,
+}: {
+ config: PerModelConfig;
+ update: (patch: Partial<PerModelConfig>) => void;
+ hint: string;
+}) {
+ return (
+ <div className={ROW_CLASS}>
+ <div className="flex min-w-0 items-center gap-1.5">
+ <span className={LABEL_CLASS}>Parallel Slots</span>
+ <InfoHint>{hint}</InfoHint>
+ </div>
+ <input
+ type="number"
+ min={N_PARALLEL_MIN}
+ max={N_PARALLEL_MAX}
+ step={1}
+ value={config.nParallel ?? ""}
+ placeholder="auto"
+ onChange={(event) => {
+ const raw = event.target.value;
+ if (raw === "") {
+ update({ nParallel: null });
+ return;
+ }
+ const parsed = Number.parseInt(raw, 10);
+ if (Number.isFinite(parsed)) {
+ update({
+ nParallel: Math.max(
+ N_PARALLEL_MIN,
+ Math.min(N_PARALLEL_MAX, parsed),
+ ),
+ });
+ }
+ }}
+ aria-label="Parallel decode slots"
+ className={NUMBER_INPUT_CLASS}
+ />
+ </div>
+ );
+}
+
function MlxAdvancedSettings({
config,
update,
@@ -1138,6 +1193,9 @@ function MlxAdvancedSettings({
) : null}
</div>
)}
+ {servedByMlx && (
+ <ParallelSlotsRow config={config} update={update} hint={MLX_PARALLEL_HINT} />
+ )}
<div className="space-y-1">
<ChatTemplateSetting
config={config}
@@ -1433,42 +1491,7 @@ function GgufAdvancedSettings({
</div>
)}
- <div className={ROW_CLASS}>
- <div className="flex min-w-0 items-center gap-1.5">
- <span className={LABEL_CLASS}>Parallel Slots</span>
- <InfoHint>
- Decode slots (--parallel) for concurrent requests. Leave blank for
- the server default. More slots share the context pool and use more
- VRAM.
- </InfoHint>
- </div>
- <input
- type="number"
- min={N_PARALLEL_MIN}
- max={N_PARALLEL_MAX}
- step={1}
- value={config.nParallel ?? ""}
- placeholder="auto"
- onChange={(event) => {
- const raw = event.target.value;
- if (raw === "") {
- update({ nParallel: null });
- return;
- }
- const parsed = Number.parseInt(raw, 10);
- if (Number.isFinite(parsed)) {
- update({
- nParallel: Math.max(
- N_PARALLEL_MIN,
- Math.min(N_PARALLEL_MAX, parsed),
- ),
- });
- }
- }}
- aria-label="Parallel decode slots"
- className={NUMBER_INPUT_CLASS}
- />
- </div>
+ <ParallelSlotsRow config={config} update={update} hint={GGUF_PARALLEL_HINT} />
{!isDiffusion && (
<div className="space-y-1">
diff --git a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
index 0141ea6f327..8b2bbdbb9af 100644
--- a/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
+++ b/studio/frontend/src/features/model-picker/model-config/per-model-config.ts
@@ -205,6 +205,8 @@ export function loadedContextFields(resp: {
native_context_length?: number | null;
max_context_length?: number | null;
context_length_enforced?: boolean | null;
+ context_unbounded_when_batched?: boolean;
+ parallel_slots?: number | null;
mlx_context_budget?: number | null;
} | null): {
loadedContextLength: number | null;
@@ -213,6 +215,8 @@ export function loadedContextFields(resp: {
loadedIsGguf: boolean | null;
loadedIsMlx: boolean | null;
loadedContextEnforced: boolean | null;
+ loadedContextUnboundedWhenBatched: boolean;
+ loadedParallelSlots: number | null;
loadedContextBudget: number | null;
} {
if (!resp) {
@@ -223,6 +227,8 @@ export function loadedContextFields(resp: {
loadedIsGguf: null,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextUnboundedWhenBatched: false,
+ loadedParallelSlots: null,
loadedContextBudget: null,
};
}
@@ -237,6 +243,8 @@ export function loadedContextFields(resp: {
loadedIsGguf: false,
loadedIsMlx: resp.is_mlx ?? null,
loadedContextEnforced: null,
+ loadedContextUnboundedWhenBatched: false,
+ loadedParallelSlots: null,
loadedContextBudget: null,
};
}
@@ -251,6 +259,11 @@ export function loadedContextFields(resp: {
// llama.cpp allocates what it reports, so GGUF is enforced by construction.
// Everything else answers for itself, or says nothing.
loadedContextEnforced: isGguf ? true : (resp.context_length_enforced ?? null),
+ // Read from the same response as the other two so the three never mix across loads.
+ loadedContextUnboundedWhenBatched: isGguf
+ ? false
+ : (resp.context_unbounded_when_batched ?? false),
+ loadedParallelSlots: resp.parallel_slots ?? null,
loadedContextBudget: isGguf ? null : (resp.mlx_context_budget ?? null),
};
}
diff --git a/studio/frontend/tests/helpers/store-stubs/auth.ts b/studio/frontend/tests/helpers/store-stubs/auth.ts
index c9dc028703f..f7f484b6035 100644
--- a/studio/frontend/tests/helpers/store-stubs/auth.ts
+++ b/studio/frontend/tests/helpers/store-stubs/auth.ts
@@ -19,3 +19,7 @@ export function authFetch(input: string, init?: RequestInit): Promise<Response>
if (!handler) throw new Error("authFetch: no network in tests");
return Promise.resolve(handler(input, init));
}
+
+export async function prepareHfTokenForUse(): Promise<null> {
+ return null;
+}
diff --git a/studio/frontend/tests/helpers/store-stubs/model-picker.ts b/studio/frontend/tests/helpers/store-stubs/model-picker.ts
index a53900604b4..722a737e1b3 100644
--- a/studio/frontend/tests/helpers/store-stubs/model-picker.ts
+++ b/studio/frontend/tests/helpers/store-stubs/model-picker.ts
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
-// The picker barrel reaches the whole chat UI; the store only needs these three.
export function applyPerModelConfigToRuntime(): void {}
@@ -12,3 +11,25 @@ export function currentRuntimePerModelConfig(): Record<string, unknown> {
export function perModelConfigsEqual(): boolean {
return true;
}
+
+let residentConfig: unknown = null;
+
+export function setResidentInitialConfig(value: unknown): void {
+ residentConfig = value;
+}
+
+export function resolveResidentInitialConfig(): unknown {
+ return residentConfig;
+}
+
+export function loadedContextFields(): Record<string, unknown> {
+ return {};
+}
+
+export function savedContextPin(): null {
+ return null;
+}
+
+export function adoptCachedRepoConfig(): null {
+ return null;
+}
diff --git a/studio/frontend/tests/helpers/store-stubs/settings-http.ts b/studio/frontend/tests/helpers/store-stubs/settings-http.ts
index 3f248741a4b..f43a377d86f 100644
--- a/studio/frontend/tests/helpers/store-stubs/settings-http.ts
+++ b/studio/frontend/tests/helpers/store-stubs/settings-http.ts
@@ -157,3 +157,7 @@ export async function authFetch(
}),
} as Response;
}
+
+export function getAuthSessionEpoch(): number {
+ return 0;
+}
diff --git a/studio/frontend/tests/mlx-context-helpers.test.ts b/studio/frontend/tests/mlx-context-helpers.test.ts
index d0cd8bfd426..a4bdd7a59b5 100644
--- a/studio/frontend/tests/mlx-context-helpers.test.ts
+++ b/studio/frontend/tests/mlx-context-helpers.test.ts
@@ -5,6 +5,7 @@
// coverage regexes the source, which passes with the body deleted; these call them.
import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
import test from "node:test";
import {
@@ -61,6 +62,8 @@ test("an MLX response carries a window without a native one", () => {
loadedIsGguf: false,
loadedIsMlx: true,
loadedContextEnforced: null,
+ loadedContextUnboundedWhenBatched: false,
+ loadedParallelSlots: null,
loadedContextBudget: null,
});
// Transformers, which sizes nothing, still contributes no window.
@@ -71,6 +74,8 @@ test("an MLX response carries a window without a native one", () => {
loadedIsGguf: false,
loadedIsMlx: null,
loadedContextEnforced: null,
+ loadedContextUnboundedWhenBatched: false,
+ loadedParallelSlots: null,
loadedContextBudget: null,
});
assert.equal(loadedContextFields(null).loadedIsGguf, null);
@@ -220,6 +225,12 @@ test("the usage bar names the three ways a window can end", () => {
assert.equal(at(30000, { contextEnforced: undefined }), "unenforced-limit");
// Only for MLX: nothing else installs a window whose bound could go unjudged.
assert.equal(at(30000, { isMlx: false, contextEnforced: null }), "stops-at-limit");
+ const batched = { contextUnboundedWhenBatched: true };
+ assert.equal(at(30000, { ...batched, parallelSlots: 4 }), "unenforced-limit");
+ assert.equal(at(40000, { ...batched, parallelSlots: 4 }), "unenforced-limit");
+ assert.equal(at(30000, { ...batched, parallelSlots: 1 }), "mlx-near-limit");
+ assert.equal(at(30000, { ...batched, parallelSlots: null }), "mlx-near-limit");
+ assert.equal(at(30000, { parallelSlots: 4 }), "mlx-near-limit");
});
test("an outgoing self-sizing window does not become the next model's request", () => {
@@ -313,3 +324,55 @@ test("a load keeps the pin it was built from, wherever the record held it", () =
}),
);
});
+
+test("a load response's two window facts and width reach the bar together", () => {
+ const wide = loadedContextFields({
+ is_mlx: true,
+ context_length: 32768,
+ native_context_length: 32768,
+ context_length_enforced: true,
+ context_unbounded_when_batched: true,
+ parallel_slots: 4,
+ });
+ assert.equal(wide.loadedContextUnboundedWhenBatched, true);
+ assert.equal(wide.loadedParallelSlots, 4);
+
+ assert.equal(
+ deriveContextUsageBar({
+ used: 30000,
+ total: 32768,
+ isMlx: true,
+ contextEnforced: wide.loadedContextEnforced,
+ contextUnboundedWhenBatched: wide.loadedContextUnboundedWhenBatched,
+ parallelSlots: wide.loadedParallelSlots,
+ })?.advice,
+ "unenforced-limit",
+ );
+
+ const gguf = loadedContextFields({
+ is_gguf: true,
+ context_length: 32768,
+ native_context_length: 32768,
+ context_unbounded_when_batched: true,
+ parallel_slots: 4,
+ });
+ assert.equal(gguf.loadedContextUnboundedWhenBatched, false);
+});
+
+test("a background load cannot leave the visible model reading another model's window", () => {
+ const source = readFileSync(
+ new URL("../src/features/chat/api/chat-adapter.ts", import.meta.url),
+ "utf8",
+ );
+ const list = source.slice(
+ source.indexOf("const VISIBLE_MODEL_RUNTIME_KEYS = ["),
+ source.indexOf("] as const satisfies"),
+ );
+ for (const key of [
+ "loadedContextEnforced",
+ "loadedContextUnboundedWhenBatched",
+ "loadedParallelSlots",
+ ]) {
+ assert.match(list, new RegExp(`"${key}"`), `${key} is not preserved`);
+ }
+});
diff --git a/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.ts b/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.ts
new file mode 100644
index 00000000000..7109cc017a3
--- /dev/null
+++ b/studio/frontend/tests/non-gguf-parallel-slots-baseline.test.ts
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+
+import assert from "node:assert/strict";
+import { register } from "node:module";
+import test from "node:test";
+
+import { installLocalStorageFake } from "./helpers/kit.ts";
+
+installLocalStorageFake();
+register("./store-settings-resolver.mjs", import.meta.url);
+
+const { useChatRuntimeStore } = await import(
+ "../src/features/chat/stores/chat-runtime-store.ts"
+);
+const { applyActiveModelStatusToStore } = await import(
+ "../src/features/chat/lib/apply-inference-status-to-store.ts"
+);
+
+const MODEL = "unsloth/Qwen3-8B";
+
+function statusFor(requested: number | null, effective = requested) {
+ return {
+ active_model: MODEL,
+ model_identifier: MODEL,
+ is_gguf: false,
+ requested_parallel_slots: requested,
+ parallel_slots: effective,
+ } as never;
+}
+
+test("a non-GGUF status carries the width it reports into the rollback baseline", () => {
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ loadedNParallel: null,
+ nParallel: null,
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },
+ });
+
+ applyActiveModelStatusToStore(statusFor(8, 1), { previousCheckpoint: MODEL });
+
+ assert.equal(
+ useChatRuntimeStore.getState().loadedNParallel,
+ 8,
+ "the width the load was invoked with is what a rollback must re-send",
+ );
+});
+
+test("a status reporting no width clears the baseline it would otherwise re-send", () => {
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ loadedNParallel: 8,
+ nParallel: null,
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },
+ });
+
+ applyActiveModelStatusToStore(statusFor(null, null), { previousCheckpoint: MODEL });
+
+ assert.equal(useChatRuntimeStore.getState().loadedNParallel, null);
+});
+
+test("a remembered non-GGUF width is adopted into the control, not just the baseline", async () => {
+ const { setResidentInitialConfig } = await import(
+ "./helpers/store-stubs/model-picker.ts"
+ );
+ setResidentInitialConfig({ remembered: true, config: { nParallel: 2 } });
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ loadedNParallel: null,
+ nParallel: null,
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: "unsloth/Other" },
+ });
+
+ applyActiveModelStatusToStore(statusFor(2), { previousCheckpoint: "unsloth/Other" });
+ setResidentInitialConfig(null);
+
+ assert.equal(useChatRuntimeStore.getState().nParallel, 2);
+});
+
+test("the baseline follows a width the server changed underneath this tab", () => {
+ useChatRuntimeStore.setState({
+ modelLoading: false,
+ loadedNParallel: 2,
+ nParallel: 2,
+ params: { ...useChatRuntimeStore.getState().params, checkpoint: MODEL },
+ });
+
+ applyActiveModelStatusToStore(statusFor(4), { previousCheckpoint: MODEL });
+
+ assert.equal(useChatRuntimeStore.getState().loadedNParallel, 4);
+});
diff --git a/studio/frontend/tests/resident-config-match.test.ts b/studio/frontend/tests/resident-config-match.test.ts
index ba4f3f1d64e..93d50341747 100644
--- a/studio/frontend/tests/resident-config-match.test.ts
+++ b/studio/frontend/tests/resident-config-match.test.ts
@@ -733,9 +733,7 @@ test("a non-GGUF resident is not judged on a GGUF invocation field", () => {
}),
false,
);
- // And a non-GGUF resident answers for the two fields the backend actually compares,
- // which is all _mlx_runtime_settings_match looks at. cache_type_kv is deliberately not
- // among them: it is a llama.cpp flag, and the non-GGUF branch never reads it.
+ // Non-GGUF matches only what the backend acts on; cache_type_kv is a llama.cpp flag it never reads.
assert.equal(
matches({ ...DEFAULTS, is_gguf: false, cache_type_kv: "q8_0" }, BLANK),
true,
@@ -1138,9 +1136,9 @@ test("a diffusion pick is reduced to its lowest GPU, as the backend reduces it",
test("no llama.cpp invocation field decides against a non-GGUF resident", () => {
// The non-GGUF branch of /load checks identity and _mlx_runtime_settings_match, then
- // answers already_loaded. Every other field here is a llama.cpp flag it never reads, so
- // a persisted Manual mode, tensor split, slot count or batch size raised the prompt for
- // a load that could not have changed anything.
+ // answers already_loaded. Every field here is a llama.cpp flag it never reads, so a
+ // persisted Manual mode, tensor split or batch size raised the prompt for a load that
+ // could not have changed anything.
const resident = { ...DEFAULTS, is_gguf: false };
assert.equal(
matches(resident, {
@@ -1149,7 +1147,6 @@ test("no llama.cpp invocation field decides against a non-GGUF resident", () =>
gpuLayers: 20,
nCpuMoe: 8,
tensorParallel: true,
- nParallel: 4,
nBatch: 2048,
nUbatch: 512,
selectedGpuIds: [1],
@@ -1165,6 +1162,19 @@ test("no llama.cpp invocation field decides against a non-GGUF resident", () =>
);
});
+test("a resident decoding at another width is not adopted, whichever backend", () => {
+ for (const is_gguf of [true, false]) {
+ assert.equal(
+ matches({ ...DEFAULTS, is_gguf, requested_parallel_slots: 4 }, { ...BLANK, nParallel: 2 }),
+ false,
+ );
+ assert.equal(
+ matches({ ...DEFAULTS, is_gguf, requested_parallel_slots: 2 }, { ...BLANK, nParallel: 2 }),
+ true,
+ );
+ }
+});
+
test("a diffusion resident is judged on its NGL, not on the placement fields", () => {
// The diffusion branch of _runtime_matches_intent replaces the placement comparison with
// one _diffusion_manual_ngl check, and an older shim that dropped a manual NGL leaves
diff --git a/studio/frontend/tests/resident-status-baselines.test.ts b/studio/frontend/tests/resident-status-baselines.test.ts
index 9c29f54509b..f9e36c242c1 100644
--- a/studio/frontend/tests/resident-status-baselines.test.ts
+++ b/studio/frontend/tests/resident-status-baselines.test.ts
@@ -207,7 +207,10 @@ test("slot baselines ignore in-flight loads and absent GGUF echoes", () => {
test("slotless statuses still clear the baseline without editing the control", () => {
const previous = { ...loaded, nParallel: 8 };
- for (const echo of [{ is_gguf: false }, { requested_parallel_slots: null }]) {
+ for (const echo of [
+ { is_gguf: false, requested_parallel_slots: null },
+ { requested_parallel_slots: null },
+ ]) {
const next = applySlots(previous, echo, true, false);
assert.equal(next.loadedNParallel, null);
assert.equal(next.nParallel, 8);
diff --git a/studio/frontend/tests/store-settings-resolver.mjs b/studio/frontend/tests/store-settings-resolver.mjs
index d5387c5b639..08450cf3397 100644
--- a/studio/frontend/tests/store-settings-resolver.mjs
+++ b/studio/frontend/tests/store-settings-resolver.mjs
@@ -7,6 +7,7 @@ import { resolve as resolveBundler } from "./bundler-resolver.mjs";
const STUBS = new Map([
["@/features/auth", "./helpers/store-stubs/settings-http.ts"],
+ ["@/features/hf-auth", "./helpers/store-stubs/auth.ts"],
["@/features/hub", "./helpers/store-stubs/hub.ts"],
["@/features/model-picker", "./helpers/store-stubs/model-picker.ts"],
["@/config/env", "./helpers/store-stubs/env.ts"],
diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py
index 71cc2bd9b12..d3bb35fc03b 100644
--- a/tests/studio/test_model_picker_contracts.py
+++ b/tests/studio/test_model_picker_contracts.py
@@ -2297,8 +2297,8 @@ def test_parallel_slots_setting_wired_end_to_end():
# Click-time snapshot, /load body, validate preflight, cross-model reset and
# failed-switch rollback all carry the value.
assert "pendingLoadConfig?.nParallel" in runtime
- # GGUF-gated, like the compare pane: a transformers load has no slots.
- assert "n_parallel: isGguf ? loadNParallel : null," in runtime
+ # Both backends take a width: llama-server slots, or the MLX batch width.
+ assert "n_parallel: loadNParallel," in runtime
assert "n_parallel: validateNParallel," in runtime
assert "loadNParallel = pendingLoadConfig?.nParallel ?? null;" in runtime
assert "n_parallel: rollbackState.loadedNParallel," in runtime
@@ -2342,9 +2342,10 @@ def test_parallel_slots_reach_an_api_load_through_the_server_mirror():
assert "n_parallel = payload.n_parallel," in route
store = _read_backend("utils/openai_auto_switch_settings.py")
assert 'entry["n_parallel"] = n_parallel' in store
- # GGUF-only, like the picker: a safetensors load has no llama-server slots.
- gguf_block = store.split(" if is_gguf:", 1)[1]
- assert 'kwargs["n_parallel"] = override["n_parallel"]' in gguf_block
+ # Ungated: MLX sizes its batch by the same width llama-server sizes its slots by.
+ shared, gguf_block = store.split(" if is_gguf:", 1)
+ assert '("n_parallel", "n_parallel"),' in shared
+ assert "n_parallel" not in gguf_block.split("\n\n", 1)[0]
def test_parallel_slots_control_cleared_when_the_load_never_sent_them():
@@ -2367,9 +2368,9 @@ def test_parallel_slots_control_cleared_when_the_load_never_sent_them():
# The cached-GGUF branch keeps the remembered override via the gated local...
assert "nParallel: committedSlots," in gguf_branch
assert "nParallel: null," not in gguf_branch
- # ...
- assert "nParallel: null," in non_gguf_branch
- assert "loadedNParallel: null," in non_gguf_branch
+ # ...and so does the non-GGUF branch, now that an MLX load takes a width too.
+ assert "nParallel: committedSlots," in non_gguf_branch
+ assert "loadedNParallel: committedSlots," in non_gguf_branch
fresh_default = adapter.split(
" return { loaded: false, blockedByTrustRemoteCode: false };", 1
@@ -2385,10 +2386,11 @@ def test_hydration_clears_the_slot_baseline_for_a_slotless_model():
"""The baseline is what a rollback re-sends and what preset capture reads, so a model
that cannot have slots must not inherit the previous GGUF's count."""
src = _read("features/chat/lib/apply-inference-status-to-store.ts")
+ # A non-GGUF load reports its width too, so only the explicit null echo means slotless.
assert (
- "(status.is_gguf === false || status.requested_parallel_slots === null) && {" in src
- ), "the slotless clear must key on is_gguf or an explicit null echo"
- clear = src.index("status.is_gguf === false || status.requested_parallel_slots === null")
+ "status.requested_parallel_slots === null && {" in src
+ ), "the slotless clear must key on an explicit null echo"
+ clear = src.index("status.requested_parallel_slots === null && {")
assert "loadedNParallel: null," in src[clear : clear + 200]
# Never `!= null`: that also matches the absent field an older backend sends.
assert "status.requested_parallel_slots !== null && {" not in src
@@ -2488,8 +2490,8 @@ def test_hydration_restores_a_remembered_slot_override():
": hydratingExistingModel)" in status
), "storage is read on a fresh store or a model change, never on a steady poll"
assert (
- "const rememberedNParallel = status.is_gguf && remembered?.remembered" in status
- ), "slots are a llama.cpp knob; reading MLX's record must not seed one"
+ "const rememberedNParallel = remembered?.remembered" in status
+ ), "a remembered width seeds the control on either backend"
assert (
"...(seedLoadParams && (slotsUnseeded || slotsModelChanged) &&" in status
), "the seed fires in both cases the clear leaves the control blank"