Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make Thinking and repetition penalty work on messages with an image (#11048)
Nilay03ae5cbfa7db
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 50ed476ab4a..15e088a0ec7 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -1330,6 +1330,9 @@ class InferenceBackend:tools = tools,tool_protocol_active = tool_protocol_active,stop = stop,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,)returnelse:@@ -1482,6 +1485,9 @@ class InferenceBackend:tools: Optional[list] = None,tool_protocol_active: Optional[bool] = None,stop: Optional[list] = None,+ enable_thinking: Optional[bool] = None,+ reasoning_effort: Optional[str] = None,+ preserve_thinking: Optional[bool] = None,) -> Generator[str, None, None]:"""Handle vision model generation with true token-by-token streaming."""# Reset so a failed or uncountable run cannot surface stale stats.@@ -1501,7 +1507,6 @@ class InferenceBackend:messages_have_tool_history,messages_with_attached_image,render_advertising_tools,- render_prompt_with_boundary,trailing_assistant_text,vlm_prompt_issue,)@@ -1538,6 +1543,9 @@ class InferenceBackend:processor,vision_messages,tools = catalog,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,continue_final_message = bool(continue_partial),)except Exception as e: # noqa: F841 -- read by the fallback below@@ -1561,6 +1569,9 @@ class InferenceBackend:processor,without_system,tools = catalog,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,continue_final_message = bool(continue_partial),)rendered_with["messages"] = without_system@@ -1594,17 +1605,15 @@ class InferenceBackend:self.active_model_name,)else:- # Processor's own template skips the choke point (#7066).- from core.inference.chat_template_helpers import markup_for_tokenizer-- vision_messages = neutralize_control_markup_in_messages(- vision_messages, None, markup_for_tokenizer(processor)- )def _render_plain_vision(msgs):- # Partial taken from the swept msgs, not the raw pre-sweep capture.- return render_prompt_with_boundary(- processor, msgs, continue_final_message = bool(continue_partial)+ return self._apply_chat_template_for_generation(+ processor,+ msgs,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ continue_final_message = bool(continue_partial),)try:@@ -1698,6 +1707,23 @@ class InferenceBackend:if _vision_input_ids is not Noneelse None)+ if repetition_penalty != 1.0 and _vision_input_ids is not None:+ from transformers import LogitsProcessorList, RepetitionPenaltyLogitsProcessor++ # Prompt ids skipped: mllama's <|image|> id lies past the LM head.+ try:+ _rp = RepetitionPenaltyLogitsProcessor(+ repetition_penalty, prompt_ignore_length = prompt_len+ )+ except TypeError:+ # prompt_ignore_length landed in transformers 4.52; the declared+ # floor is 4.51.3, where the same slice belongs here instead.+ class _PromptSkippingRepetitionPenalty(RepetitionPenaltyLogitsProcessor):+ def __call__(self, input_ids, scores):+ return super().__call__(input_ids[:, prompt_len:], scores)++ _rp = _PromptSkippingRepetitionPenalty(repetition_penalty)+ _pp = LogitsProcessorList([_rp, *(_pp or [])])timer = GenerationTimer()generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)stop_streamer = _StopSequenceStreamer(streamer, stop)diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 9ceac27c872..c1e2c2b548f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -25975,11 +25975,6 @@ async def produce_openai_chat_completions(except Exception:_sf_probe_messages = None- # Transformers vision generation drops both reasoning fields; MLX forwards them.- _sf_vision_drops_reasoning = image is not None and not _sf_model_info.get("is_mlx", False)- _sf_gate_enable_thinking = None if _sf_vision_drops_reasoning else payload.enable_thinking- _sf_gate_reasoning_effort = None if _sf_vision_drops_reasoning else payload.reasoning_effort-def _sf_response_protocol(tools = None,template = None,@@ -26006,8 +26001,8 @@ async def produce_openai_chat_completions(logger.debug("safetensors_prefill_template_selection_failed", exc_info = True)parse_think = _sf_parse_think_markers(features,- _sf_gate_enable_thinking,- _sf_gate_reasoning_effort,+ payload.enable_thinking,+ payload.reasoning_effort,)reasoning_prefilled = _sf_reasoning_prefill_mode(features,diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.pyindex 694bf72d096..a819dd7858b 100644--- a/studio/backend/tests/test_safetensors_reasoning_stream.py+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py@@ -711,18 +711,18 @@ def test_route_still_splits_real_thinking_when_the_request_leaves_it_on(monkeypaassert message["reasoning_content"] == "plan"-def test_route_still_splits_a_transformers_image_turn_the_request_asked_to_disable(monkeypatch):- """Transformers vision generation ignores both reasoning fields."""+def test_route_keeps_literal_think_text_on_a_transformers_image_turn(monkeypatch):+ answer = "Use <think>hi</think> in your prompt."message = _sf_route_message(monkeypatch,_TEMPLATE_DEFAULT_OFF_TPL,- ["<think>plan</think>answer"],+ [answer],is_vision = True,enable_thinking = False,image_base64 = _TINY_PNG_B64,)- assert message["content"] == "answer"- assert message["reasoning_content"] == "plan"+ assert message["content"] == answer+ assert not message["reasoning_content"]def test_route_keeps_literal_think_text_on_an_mlx_image_turn(monkeypatch):diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.pyindex 47fee40c46c..56a4dea194b 100644--- a/studio/backend/tests/test_vision_client_tools.py+++ b/studio/backend/tests/test_vision_client_tools.py@@ -1099,6 +1099,101 @@ def test_an_image_continuation_resumes_the_replayed_partial_after_the_no_system_assert kwargs["continue_final_message"] is True+ "system_prompt, messages, tools",+ [+ ("SYSTEM_RULE", [{"role": "user", "content": "what is in this picture"}], None),+ (+ "",+ [+ {"role": "system", "content": "SYSTEM_RULE"},+ {"role": "user", "content": "what is in this picture"},+ ],+ [_LOOKUP],+ ),+ ],+)+def test_every_image_render_carries_the_reasoning_controls(system_prompt, messages, tools):+ backend, _ = _vision_probe()+ renders = []++ def apply_chat_template(messages, **kwargs):+ renders.append(kwargs)+ if any(m["role"] == "system" for m in messages):+ raise ValueError("system role not supported")+ return "PROMPT"++ backend.models["vision-tools"]["processor"].apply_chat_template = apply_chat_template+ _drain(+ backend,+ system_prompt = system_prompt,+ messages = messages,+ tools = tools,+ enable_thinking = False,+ reasoning_effort = "low",+ preserve_thinking = True,+ )+ assert len(renders) >= 2+ for kwargs in renders:+ assert kwargs["enable_thinking"] is False+ assert kwargs["reasoning_effort"] == "low"+ assert kwargs["preserve_thinking"] is True+++def test_an_image_turn_penalizes_repeats_in_the_generated_tokens_only():+ torch = pytest.importorskip("torch")+ from transformers import RepetitionPenaltyLogitsProcessor++ backend, _ = _vision_probe()+ calls = []+ backend.models["vision-tools"]["model"].generate = lambda **kwargs: calls.append(kwargs)+ messages = [{"role": "user", "content": "what is in this picture"}]+ _drain(backend, messages = messages, repetition_penalty = 1.0)+ _drain(backend, messages = messages, repetition_penalty = 2.0)++ def penalties(call):+ return [+ p for p in call["logits_processor"] if isinstance(p, RepetitionPenaltyLogitsProcessor)+ ]++ assert penalties(calls[0]) == []+ (penalty,) = penalties(calls[1])+ scores = penalty(torch.tensor([[9, 2]]), torch.ones((1, 4)))+ assert scores.tolist() == [[1.0, 1.0, 0.5, 1.0]]+++def test_an_image_turn_still_penalizes_on_a_transformers_without_prompt_ignore_length(monkeypatch):+ """``prompt_ignore_length`` arrived in transformers 4.52; pyproject still allows+ 4.51.3, whose processor takes ``penalty`` alone and raised TypeError here."""+ torch = pytest.importorskip("torch")+ import transformers++ live = sys.modules["transformers"]+ real = live.RepetitionPenaltyLogitsProcessor++ class Floor(real):+ def __init__(self, penalty: float):+ super().__init__(penalty)++ monkeypatch.setattr(live, "RepetitionPenaltyLogitsProcessor", Floor, raising = False)+ monkeypatch.setattr(transformers, "RepetitionPenaltyLogitsProcessor", Floor, raising = False)++ backend, _ = _vision_probe()+ calls = []+ backend.models["vision-tools"]["model"].generate = lambda **kwargs: calls.append(kwargs)+ _drain(+ backend,+ messages = [{"role": "user", "content": "what is in this picture"}],+ repetition_penalty = 2.0,+ )++ (penalty,) = [p for p in calls[0]["logits_processor"] if isinstance(p, Floor)]+ # Same slice the 4.52+ processor does internally: the one prompt id is skipped,+ # so id 9 never indexes past this 4-wide vocabulary and only id 2 is penalized.+ scores = penalty(torch.tensor([[9, 2]]), torch.ones((1, 4)))+ assert scores.tolist() == [[1.0, 1.0, 0.5, 1.0]]++@pytest.mark.parametrize("content, structured, expected",[