Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make Thinking and repetition penalty work on messages with an image (#11048)

Nilay03ae5cbfa7db
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 50ed476ab4a..15e088a0ec7 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -1330,6 +1330,9 @@ class InferenceBackend:
tools = tools,
tool_protocol_active = tool_protocol_active,
stop = stop,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
)
return
else:
@@ -1482,6 +1485,9 @@ class InferenceBackend:
tools: Optional[list] = None,
tool_protocol_active: Optional[bool] = None,
stop: Optional[list] = None,
+ enable_thinking: Optional[bool] = None,
+ reasoning_effort: Optional[str] = None,
+ preserve_thinking: Optional[bool] = None,
) -> Generator[str, None, None]:
"""Handle vision model generation with true token-by-token streaming."""
# Reset so a failed or uncountable run cannot surface stale stats.
@@ -1501,7 +1507,6 @@ class InferenceBackend:
messages_have_tool_history,
messages_with_attached_image,
render_advertising_tools,
- render_prompt_with_boundary,
trailing_assistant_text,
vlm_prompt_issue,
)
@@ -1538,6 +1543,9 @@ class InferenceBackend:
processor,
vision_messages,
tools = catalog,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
continue_final_message = bool(continue_partial),
)
except Exception as e: # noqa: F841 -- read by the fallback below
@@ -1561,6 +1569,9 @@ class InferenceBackend:
processor,
without_system,
tools = catalog,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
continue_final_message = bool(continue_partial),
)
rendered_with["messages"] = without_system
@@ -1594,17 +1605,15 @@ class InferenceBackend:
self.active_model_name,
)
else:
- # Processor's own template skips the choke point (#7066).
- from core.inference.chat_template_helpers import markup_for_tokenizer
-
- vision_messages = neutralize_control_markup_in_messages(
- vision_messages, None, markup_for_tokenizer(processor)
- )
def _render_plain_vision(msgs):
- # Partial taken from the swept msgs, not the raw pre-sweep capture.
- return render_prompt_with_boundary(
- processor, msgs, continue_final_message = bool(continue_partial)
+ return self._apply_chat_template_for_generation(
+ processor,
+ msgs,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ continue_final_message = bool(continue_partial),
)
try:
@@ -1698,6 +1707,23 @@ class InferenceBackend:
if _vision_input_ids is not None
else None
)
+ if repetition_penalty != 1.0 and _vision_input_ids is not None:
+ from transformers import LogitsProcessorList, RepetitionPenaltyLogitsProcessor
+
+ # Prompt ids skipped: mllama's <|image|> id lies past the LM head.
+ try:
+ _rp = RepetitionPenaltyLogitsProcessor(
+ repetition_penalty, prompt_ignore_length = prompt_len
+ )
+ except TypeError:
+ # prompt_ignore_length landed in transformers 4.52; the declared
+ # floor is 4.51.3, where the same slice belongs here instead.
+ class _PromptSkippingRepetitionPenalty(RepetitionPenaltyLogitsProcessor):
+ def __call__(self, input_ids, scores):
+ return super().__call__(input_ids[:, prompt_len:], scores)
+
+ _rp = _PromptSkippingRepetitionPenalty(repetition_penalty)
+ _pp = LogitsProcessorList([_rp, *(_pp or [])])
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
stop_streamer = _StopSequenceStreamer(streamer, stop)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 9ceac27c872..c1e2c2b548f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -25975,11 +25975,6 @@ async def produce_openai_chat_completions(
except Exception:
_sf_probe_messages = None
- # Transformers vision generation drops both reasoning fields; MLX forwards them.
- _sf_vision_drops_reasoning = image is not None and not _sf_model_info.get("is_mlx", False)
- _sf_gate_enable_thinking = None if _sf_vision_drops_reasoning else payload.enable_thinking
- _sf_gate_reasoning_effort = None if _sf_vision_drops_reasoning else payload.reasoning_effort
-
def _sf_response_protocol(
tools = None,
template = None,
@@ -26006,8 +26001,8 @@ async def produce_openai_chat_completions(
logger.debug("safetensors_prefill_template_selection_failed", exc_info = True)
parse_think = _sf_parse_think_markers(
features,
- _sf_gate_enable_thinking,
- _sf_gate_reasoning_effort,
+ payload.enable_thinking,
+ payload.reasoning_effort,
)
reasoning_prefilled = _sf_reasoning_prefill_mode(
features,
diff --git a/studio/backend/tests/test_safetensors_reasoning_stream.py b/studio/backend/tests/test_safetensors_reasoning_stream.py
index 694bf72d096..a819dd7858b 100644
--- a/studio/backend/tests/test_safetensors_reasoning_stream.py
+++ b/studio/backend/tests/test_safetensors_reasoning_stream.py
@@ -711,18 +711,18 @@ def test_route_still_splits_real_thinking_when_the_request_leaves_it_on(monkeypa
assert message["reasoning_content"] == "plan"
-def test_route_still_splits_a_transformers_image_turn_the_request_asked_to_disable(monkeypatch):
- """Transformers vision generation ignores both reasoning fields."""
+def test_route_keeps_literal_think_text_on_a_transformers_image_turn(monkeypatch):
+ answer = "Use <think>hi</think> in your prompt."
message = _sf_route_message(
monkeypatch,
_TEMPLATE_DEFAULT_OFF_TPL,
- ["<think>plan</think>answer"],
+ [answer],
is_vision = True,
enable_thinking = False,
image_base64 = _TINY_PNG_B64,
)
- assert message["content"] == "answer"
- assert message["reasoning_content"] == "plan"
+ assert message["content"] == answer
+ assert not message["reasoning_content"]
def test_route_keeps_literal_think_text_on_an_mlx_image_turn(monkeypatch):
diff --git a/studio/backend/tests/test_vision_client_tools.py b/studio/backend/tests/test_vision_client_tools.py
index 47fee40c46c..56a4dea194b 100644
--- a/studio/backend/tests/test_vision_client_tools.py
+++ b/studio/backend/tests/test_vision_client_tools.py
@@ -1099,6 +1099,101 @@ def test_an_image_continuation_resumes_the_replayed_partial_after_the_no_system_
assert kwargs["continue_final_message"] is True
+ "system_prompt, messages, tools",
+ [
+ ("SYSTEM_RULE", [{"role": "user", "content": "what is in this picture"}], None),
+ (
+ "",
+ [
+ {"role": "system", "content": "SYSTEM_RULE"},
+ {"role": "user", "content": "what is in this picture"},
+ ],
+ [_LOOKUP],
+ ),
+ ],
+)
+def test_every_image_render_carries_the_reasoning_controls(system_prompt, messages, tools):
+ backend, _ = _vision_probe()
+ renders = []
+
+ def apply_chat_template(messages, **kwargs):
+ renders.append(kwargs)
+ if any(m["role"] == "system" for m in messages):
+ raise ValueError("system role not supported")
+ return "PROMPT"
+
+ backend.models["vision-tools"]["processor"].apply_chat_template = apply_chat_template
+ _drain(
+ backend,
+ system_prompt = system_prompt,
+ messages = messages,
+ tools = tools,
+ enable_thinking = False,
+ reasoning_effort = "low",
+ preserve_thinking = True,
+ )
+ assert len(renders) >= 2
+ for kwargs in renders:
+ assert kwargs["enable_thinking"] is False
+ assert kwargs["reasoning_effort"] == "low"
+ assert kwargs["preserve_thinking"] is True
+
+
+def test_an_image_turn_penalizes_repeats_in_the_generated_tokens_only():
+ torch = pytest.importorskip("torch")
+ from transformers import RepetitionPenaltyLogitsProcessor
+
+ backend, _ = _vision_probe()
+ calls = []
+ backend.models["vision-tools"]["model"].generate = lambda **kwargs: calls.append(kwargs)
+ messages = [{"role": "user", "content": "what is in this picture"}]
+ _drain(backend, messages = messages, repetition_penalty = 1.0)
+ _drain(backend, messages = messages, repetition_penalty = 2.0)
+
+ def penalties(call):
+ return [
+ p for p in call["logits_processor"] if isinstance(p, RepetitionPenaltyLogitsProcessor)
+ ]
+
+ assert penalties(calls[0]) == []
+ (penalty,) = penalties(calls[1])
+ scores = penalty(torch.tensor([[9, 2]]), torch.ones((1, 4)))
+ assert scores.tolist() == [[1.0, 1.0, 0.5, 1.0]]
+
+
+def test_an_image_turn_still_penalizes_on_a_transformers_without_prompt_ignore_length(monkeypatch):
+ """``prompt_ignore_length`` arrived in transformers 4.52; pyproject still allows
+ 4.51.3, whose processor takes ``penalty`` alone and raised TypeError here."""
+ torch = pytest.importorskip("torch")
+ import transformers
+
+ live = sys.modules["transformers"]
+ real = live.RepetitionPenaltyLogitsProcessor
+
+ class Floor(real):
+ def __init__(self, penalty: float):
+ super().__init__(penalty)
+
+ monkeypatch.setattr(live, "RepetitionPenaltyLogitsProcessor", Floor, raising = False)
+ monkeypatch.setattr(transformers, "RepetitionPenaltyLogitsProcessor", Floor, raising = False)
+
+ backend, _ = _vision_probe()
+ calls = []
+ backend.models["vision-tools"]["model"].generate = lambda **kwargs: calls.append(kwargs)
+ _drain(
+ backend,
+ messages = [{"role": "user", "content": "what is in this picture"}],
+ repetition_penalty = 2.0,
+ )
+
+ (penalty,) = [p for p in calls[0]["logits_processor"] if isinstance(p, Floor)]
+ # Same slice the 4.52+ processor does internally: the one prompt id is skipped,
+ # so id 9 never indexes past this 4-wide vocabulary and only id 2 is penalized.
+ scores = penalty(torch.tensor([[9, 2]]), torch.ones((1, 4)))
+ assert scores.tolist() == [[1.0, 1.0, 0.5, 1.0]]
+
+
@pytest.mark.parametrize(
"content, structured, expected",
[