Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Stop at the requested stop text on transformers models (#10812)

Nilay9360dcce488f
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index f9ea15e672b..7f566dc3b66 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -13,6 +13,7 @@ from peft import PeftModel, PeftModelForCausalLM
import contextlib
import json
import sys
+import threading
import torch
from pathlib import Path
from typing import Optional, Union, Generator, Tuple
@@ -61,6 +62,7 @@ from core.inference.native_tool_tokens import (
reasoning_control_tokens,
stop_token_text,
)
+from core.inference.mlx_inference import _mlx_stop_cut, _mlx_stop_sequences
from io import StringIO
import structlog
from loggers import get_logger
@@ -258,6 +260,198 @@ class _GenerationThreadError(RuntimeError):
"""Generation worker failures that should propagate through stream routes."""
+# What clean_up_tokenization_spaces deletes a space before. Every rule is a space plus
+# one of these, and the longest is three characters, so a rule spans at most four.
+_CLEANUP_TAILS = (".", "?", "!", ",", "' ", "n't", "'m", "'s", "'ve", "'re")
+_CLEANUP_SPAN_CHARS = 4
+# Tokens, not characters: those four characters can arrive as four one-character tokens,
+# and eight leaves room for tokens that decode to "".
+_CLEANUP_SPAN_TOKENS = 8
+
+
+def _cleanup_join_pending(text: str) -> bool:
+ """Could a cleanup rule still be completed at the end of ``text``?
+
+ Only the last few characters can matter, and only after a space: a rule whose space is
+ further back than that is already settled either way. A trailing space counts, since the
+ next token can start any of the rules.
+ """
+ window = text[-_CLEANUP_SPAN_CHARS:]
+ for i, char in enumerate(window):
+ if char != " ":
+ continue
+ rest = window[i + 1 :]
+ if not rest or any(tail.startswith(rest) for tail in _CLEANUP_TAILS):
+ return True
+ return False
+
+
+def _decoder_cleans_up(streamer) -> bool:
+ """Does this streamer's decode apply clean_up_tokenization_spaces?
+
+ Read once, because the widened re-decode below is pure cost for the tokenizers that do
+ not (Qwen, Gemma, Mistral, gpt-oss ship it off, and transformers 5.17 ignores it for
+ every BPE tokenizer). The streamer's own kwargs win, then the tokenizer's default,
+ unwrapping NativeToolTokenDecoder to reach it.
+ """
+ kwargs = getattr(streamer, "decode_kwargs", None) or {}
+ if "clean_up_tokenization_spaces" in kwargs:
+ return bool(kwargs["clean_up_tokenization_spaces"])
+ tokenizer = getattr(streamer, "tokenizer", None)
+ for _ in range(4):
+ if tokenizer is None:
+ break
+ flag = getattr(tokenizer, "clean_up_tokenization_spaces", None)
+ if flag is not None:
+ return bool(flag)
+ tokenizer = getattr(tokenizer, "tokenizer", None)
+ return False
+
+
+class _StopSequenceStreamer:
+ def __init__(self, streamer, stop):
+ self.streamer = streamer
+ self.sequences = _mlx_stop_sequences(stop)
+ self.matched = threading.Event()
+ self.token_ids = []
+ self.text = ""
+ self.settled = ""
+ self.prefix_offset = 0
+ self.read_offset = 0
+ self.scan_from = 0
+ self.longest = max((len(s) for s in self.sequences), default = 0)
+ self.released = 0
+ self.cut = 0
+ self.finished = False
+ self.next_tokens_are_prompt = bool(getattr(streamer, "skip_prompt", False))
+ self.is_harmony = isinstance(streamer, HarmonyTextStreamer)
+ self.cleans_up = _decoder_cleans_up(streamer)
+
+ def __next__(self):
+ return next(self.streamer)
+
+ def __iter__(self):
+ return self
+
+ def put(self, value):
+ if not self.sequences:
+ return self.streamer.put(value)
+ if self.finished or self.matched.is_set():
+ return
+ # Matches the supported batch-one TextIteratorStreamer protocol.
+ if len(value.shape) > 1:
+ if value.shape[0] > 1:
+ raise ValueError("TextStreamer only supports batch size 1")
+ value = value[0]
+ if self.next_tokens_are_prompt:
+ self.next_tokens_are_prompt = False
+ return
+ self.token_ids.extend(value.tolist())
+ self._decode_new_tokens()
+ # Earlier text was already scanned; only a stop overlapping new text can match.
+ start = self.scan_from
+ cut, matched = _mlx_stop_cut(self.text[start:], self.sequences)
+ self.cut = start + cut
+ if matched:
+ # This runs in the producer, before model.generate checks criteria.
+ self.matched.set()
+ else:
+ self.scan_from = max(start, len(self.settled) - self.longest + 1)
+ cut = self.cut
+ if not matched and not self.is_harmony and cut > self.released:
+ # Keep the word-buffering stability guarantee for displayed text.
+ # Stop detection above must still inspect every decoded token.
+ if not self.streamer._is_chinese_char(ord(self.text[cut - 1])):
+ cut = (
+ max(
+ self.text.rfind(" ", self.released, cut),
+ self.text.rfind("\n", self.released, cut),
+ )
+ + 1
+ )
+ self._publish(cut)
+
+ def _decode(self, token_ids):
+ decode_kwargs = (
+ {"skip_special_tokens": False} if self.is_harmony else self.streamer.decode_kwargs
+ )
+ return self.streamer.tokenizer.decode(token_ids, **decode_kwargs)
+
+ def _decode_new_tokens(self):
+ # Re-decoding the whole reply each token is quadratic; decode a short window
+ # and settle its text once it no longer ends in unresolved bytes.
+ prefix = self._decode(self.token_ids[self.prefix_offset : self.read_offset])
+ window = self._decode(self.token_ids[self.prefix_offset :])
+ if (
+ self.cleans_up
+ and window.startswith(prefix)
+ and _cleanup_join_pending(self.settled + window[len(prefix) :])
+ ):
+ # A cleanup rewrite that spans the join is invisible from this window: split
+ # across tokens (" ", "'", "v", "e") each half decodes unchanged, so
+ # ``startswith`` holds and the concatenation keeps a space the full decode
+ # drops. Settled text then diverges permanently and a stop written across the
+ # join never matches. Re-read the few tokens behind the join while a rule could
+ # still complete there, so ordinary text keeps the short window.
+ wider = max(0, self.read_offset - _CLEANUP_SPAN_TOKENS)
+ if wider < self.prefix_offset:
+ wide_prefix = self._decode(self.token_ids[wider : self.read_offset])
+ # Splicing onto text this prefix does not end is worse than the divergence
+ # it repairs, so fall back to the short window rather than guess.
+ if self.settled.endswith(wide_prefix):
+ prefix = wide_prefix
+ window = self._decode(self.token_ids[wider:])
+ if window.endswith("\ufffd"):
+ # Bytes still arriving can rewrite the whole window (byte-fallback tokenizers
+ # show an unfinished emoji as replacement characters), so wait for them.
+ tail = window[len(prefix) :] if window.startswith(prefix) else ""
+ self.text = self.settled + tail
+ return
+ if not window.startswith(prefix):
+ # The new tokens rewrote settled text (e.g. space cleanup turning " ." into
+ # "."), so rebuild it once from every token and rescan it for stops.
+ self.settled = self._decode(self.token_ids)
+ self.prefix_offset = max(0, len(self.token_ids) - 4)
+ self.read_offset = len(self.token_ids)
+ self.scan_from = 0
+ elif len(window) > len(prefix):
+ self.settled += window[len(prefix) :]
+ self.prefix_offset = self.read_offset
+ self.read_offset = len(self.token_ids)
+ self.text = self.settled
+
+ def _publish(self, cut):
+ if cut <= self.released:
+ return
+ if self.is_harmony:
+ # Harmony consumes cumulative raw text and synthesizes tags itself.
+ self.streamer._process_incremental(self.text[:cut])
+ else:
+ # The reasoning subclass normalizes here; plain text just queues it.
+ self.streamer.on_finalized_text(self.text[self.released : cut])
+ self.released = cut
+
+ def end(self):
+ if not self.sequences:
+ return self.streamer.end()
+ if self.finished:
+ return
+ self.finished = True
+ if not self.matched.is_set() and self.read_offset < len(self.token_ids):
+ # Bytes that never resolved still belong to the reply, as a full decode shows.
+ self.text = self._decode(self.token_ids)
+ # A natural end releases a partial unmatched stop and unresolved bytes.
+ self._publish(self.cut if self.matched.is_set() else len(self.text))
+ # Their token caches are empty: put() above feeds raw decoded text, so
+ # end() only finishes reasoning framing and sends the queue sentinel.
+ self.streamer.end()
+
+ def abort(self):
+ abort = getattr(self.streamer, "abort", None)
+ if abort is not None:
+ abort()
+
+
def _prompt_already_has_bos(tokenizer, prompt):
"""Did the rendered chat template emit BOS itself?
@@ -1039,6 +1233,7 @@ class InferenceBackend:
continue_final_message: bool = False,
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
+ stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Generate response for text or vision models (lock held by background thread).
@@ -1065,6 +1260,7 @@ class InferenceBackend:
continue_final_message = continue_final_message,
tool_protocol_active = tool_protocol_active,
presence_penalty = presence_penalty,
+ stop = stop,
)
def _generate_chat_response_inner(
@@ -1087,6 +1283,7 @@ class InferenceBackend:
continue_final_message: bool = False,
presence_penalty: float = 0.0,
tool_protocol_active: Optional[bool] = None,
+ stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Inner generation logic.
@@ -1131,6 +1328,7 @@ class InferenceBackend:
continue_final_message = continue_final_message,
tools = tools,
tool_protocol_active = tool_protocol_active,
+ stop = stop,
)
return
else:
@@ -1262,6 +1460,7 @@ class InferenceBackend:
if tool_protocol_active is None
else tool_protocol_active,
add_special_tokens = add_special_tokens,
+ stop = stop,
)
def _generate_vision_response(
@@ -1280,6 +1479,7 @@ class InferenceBackend:
continue_final_message: bool = False,
tools: Optional[list] = None,
tool_protocol_active: Optional[bool] = None,
+ stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Handle vision model generation with true token-by-token streaming."""
# Reset so a failed or uncountable run cannot surface stale stats.
@@ -1529,7 +1729,10 @@ class InferenceBackend:
)
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
- stopping_criteria = self._cancel_stopping_criteria(cancel_event)
+ stop_streamer = _StopSequenceStreamer(streamer, stop)
+ if stop_streamer.sequences:
+ generation_kwargs["streamer"] = stop_streamer
+ stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
if stopping_criteria is not None:
generation_kwargs["stopping_criteria"] = stopping_criteria
active_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)
@@ -1547,13 +1750,12 @@ class InferenceBackend:
gen_outputs["sequences"] = model.generate(**generation_kwargs)
except Exception as e:
err["msg"] = str(e)
- if hasattr(streamer, "abort"):
- streamer.abort()
+ stop_streamer.abort()
logger.error(f"Vision generation error in thread: {e}")
finally:
timer.finish()
try:
- streamer.end()
+ stop_streamer.end()
except Exception:
pass
@@ -1578,7 +1780,7 @@ class InferenceBackend:
elif time.monotonic() >= cancel_deadline:
break
try:
- new_token = next(streamer)
+ new_token = next(stop_streamer)
except StopIteration:
generation_complete = True
break
@@ -1586,7 +1788,7 @@ class InferenceBackend:
if not thread.is_alive():
generation_complete = True
output = yield from self._drain_streamer_tail(
- streamer, output, active_stop_token_ids
+ stop_streamer, output, active_stop_token_ids
)
break
if cancel_deadline is not None:
@@ -1598,7 +1800,7 @@ class InferenceBackend:
break
generation_complete = True
output = yield from self._drain_streamer_tail(
- streamer, output, active_stop_token_ids
+ stop_streamer, output, active_stop_token_ids
)
break
continue
@@ -1626,9 +1828,8 @@ class InferenceBackend:
model, gen_outputs["sequences"], prompt_len
),
max_new_tokens = max_new_tokens,
- ended_on_stop_token = self._ended_on_stop_token(
- gen_outputs["sequences"], active_stop_token_ids
- ),
+ ended_on_stop_token = stop_streamer.matched.is_set()
+ or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
cancelled = not generation_complete
or (cancel_event is not None and cancel_event.is_set()),
timer = timer,
@@ -1957,6 +2158,7 @@ class InferenceBackend:
continued: bool = False,
preserve_tool_tokens: bool = False,
add_special_tokens: bool = True,
+ stop: Optional[list] = None,
) -> Generator[str, None, None]:
"""Generate a streaming text response (text models only).
@@ -2044,7 +2246,10 @@ class InferenceBackend:
_pp = _make_presence_penalty_processor(presence_penalty, prompt_len)
timer = GenerationTimer()
generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)
- stopping_criteria = self._cancel_stopping_criteria(cancel_event)
+ stop_streamer = _StopSequenceStreamer(streamer, stop)
+ if stop_streamer.sequences:
+ generation_kwargs["streamer"] = stop_streamer
+ stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)
if stopping_criteria is not None:
generation_kwargs["stopping_criteria"] = stopping_criteria
@@ -2060,13 +2265,12 @@ class InferenceBackend:
gen_outputs["sequences"] = model.generate(**generation_kwargs)
except Exception as e:
err["msg"] = str(e)
- if hasattr(streamer, "abort"):
- streamer.abort()
+ stop_streamer.abort()
logger.error(f"Generation error: {e}")
finally:
timer.finish()
try:
- streamer.end()
+ stop_streamer.end()
except Exception:
pass
@@ -2093,7 +2297,7 @@ class InferenceBackend:
elif time.monotonic() >= cancel_deadline:
break
try:
- new_token = next(streamer)
+ new_token = next(stop_streamer)
except StopIteration:
generation_complete = True
break
@@ -2101,7 +2305,7 @@ class InferenceBackend:
if not thread.is_alive():
generation_complete = True
output = yield from self._drain_streamer_tail(
- streamer, output, active_stop_token_ids
+ stop_streamer, output, active_stop_token_ids
)
break
if cancel_deadline is not None:
@@ -2113,7 +2317,7 @@ class InferenceBackend:
break
generation_complete = True
output = yield from self._drain_streamer_tail(
- streamer, output, active_stop_token_ids
+ stop_streamer, output, active_stop_token_ids
)
break
continue
@@ -2143,9 +2347,8 @@ class InferenceBackend:
model, gen_outputs["sequences"], prompt_len
),
max_new_tokens = max_new_tokens,
- ended_on_stop_token = self._ended_on_stop_token(
- gen_outputs["sequences"], active_stop_token_ids
- ),
+ ended_on_stop_token = stop_streamer.matched.is_set()
+ or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),
cancelled = not generation_complete
or (cancel_event is not None and cancel_event.is_set()),
timer = timer,
@@ -2855,9 +3058,10 @@ class InferenceBackend:
stats["timings"] = timings
self.last_generation_stats = stats
- def _cancel_stopping_criteria(self, cancel_event):
+ def _cancel_stopping_criteria(self, *events):
"""Build a Transformers stopping criteria list for user cancellation."""
- if cancel_event is None:
+ events = [ev for ev in events if ev is not None]
+ if not events:
return None
from transformers.generation.stopping_criteria import (
StoppingCriteria,
@@ -2871,7 +3075,7 @@ class InferenceBackend:
def __call__(self, input_ids, scores, **kwargs):
return self.ev.is_set()
- return StoppingCriteriaList([_CancelCriteria(cancel_event)])
+ return StoppingCriteriaList([_CancelCriteria(ev) for ev in events])
def _clean_generated_text(
self,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 3c1f42a635a..acc84d7ca03 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -702,9 +702,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:
if opt_key in cmd:
gen_kwargs[opt_key] = cmd[opt_key]
- # These options are MLX-only. The transformers backend declares none of
- # them and takes no **kwargs, so forwarding unconditionally would turn
- # its documented "ignores them" behavior into a TypeError.
+ # Not every backend declares these (transformers declares only ``stop``)
+ # and none takes **kwargs, so forwarding unconditionally would turn a
+ # backend's documented "ignores them" behavior into a TypeError.
# ``tool_protocol_active`` rides here rather than above: MLX declares no such
# parameter and takes no **kwargs, so an unconditional forward would raise.
for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):
diff --git a/studio/backend/tests/test_transformers_stop_sequences.py b/studio/backend/tests/test_transformers_stop_sequences.py
new file mode 100644
index 00000000000..63083c1f8bf
--- /dev/null
+++ b/studio/backend/tests/test_transformers_stop_sequences.py
@@ -0,0 +1,405 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import importlib
+import sys
+import threading
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+try:
+ import core.inference.inference # noqa: E402,F401
+except ImportError:
+ pass
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+class _Batch(dict):
+ def to(self, *_args, **_kwargs):
+ return self
+
+
+class _Tokenizer:
+ all_special_tokens: list = []
+ eos_token_id = 1
+ pad_token_id = None
+ chat_template = "{{ messages }}"
+
+ def __call__(self, *_args, **_kwargs):
+ torch = pytest.importorskip("torch")
+ return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})
+
+ def decode(self, ids, **kwargs):
+ return "".join(self.pieces.get(int(i), "") for i in ids)
+
+
+class _Processor:
+ chat_template = None
+ tokenizer = _Tokenizer()
+
+
+class _Model:
+ device = "cpu"
+ generation_config = type("Cfg", (), {"eos_token_id": 1})()
+ config = generation_config
+
+ def __init__(self, pieces):
+ self.pieces = pieces
+ self.sent = []
+
+ def generate(self, streamer, stopping_criteria, max_new_tokens, **_kwargs):
+ torch = pytest.importorskip("torch")
+ ids = torch.zeros((1, 1), dtype = torch.long)
+ streamer.put(ids)
+ for index, piece in enumerate(self.pieces[:max_new_tokens], 2):
+ ids = torch.cat([ids, torch.tensor([[index]])], dim = 1)
+ streamer.put(torch.tensor([index]))
+ self.sent.append(piece)
+ if stopping_criteria is not None and bool(stopping_criteria(ids, None).all()):
+ break
+ streamer.end()
+ return ids
+
+
+def _backend(pieces, tokenizer):
+ inf = pytest.importorskip("core.inference.inference")
+ model = _Model(pieces)
+ raw_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ raw_tokenizer.pieces = dict(enumerate(pieces, 2))
+ streamer = inf.TextIteratorStreamer(raw_tokenizer, skip_prompt = True, timeout = 0.2)
+ backend = inf.InferenceBackend.__new__(inf.InferenceBackend)
+ backend.active_model_name = "stop-test"
+ backend._generation_lock = threading.Lock()
+ backend.models = {"stop-test": {"model": model, "tokenizer": tokenizer, "processor": tokenizer}}
+ backend._make_text_streamer = lambda *_args, **_kwargs: streamer
+ return backend, model
+
+
+class _Responses(list):
+ def put(self, response):
+ self.append(response)
+
+
+def test_worker_forwards_stop_and_the_reply_ends_before_it(monkeypatch):
+ from core.inference import worker
+
+ backend, model = _backend(["Hello ", "ST", "OP", " world"], _Tokenizer())
+ monkeypatch.setattr(
+ backend, "_apply_chat_template_for_generation", lambda *a, **k: "PROMPT", raising = False
+ )
+ responses = _Responses()
+ cmd = {
+ "request_id": "r",
+ "messages": [{"role": "user", "content": "hi"}],
+ "max_new_tokens": 3,
+ "stop": ["STOP"],
+ }
+
+ worker._handle_generate(backend, cmd, responses, threading.Event())
+
+ assert [r["text"] for r in responses if r["type"] == "token"] == ["Hello"]
+ assert model.sent == ["Hello ", "ST", "OP"]
+ done = next(r for r in responses if r["type"] == "gen_done")
+ assert done["stats"]["truncated"] is False
+
+
+def test_vision_reply_ends_before_a_stop_sequence():
+ backend, model = _backend(["Hello ", "ST", "OP", " world"], _Processor())
+ backend.format_chat_prompt = lambda *_args, **_kwargs: "PROMPT"
+
+ snapshots = list(
+ backend._generate_vision_response(
+ messages = [{"role": "user", "content": "hi"}],
+ system_prompt = "",
+ image = None,
+ temperature = 0.0,
+ top_p = 1.0,
+ top_k = 0,
+ min_p = 0.0,
+ max_new_tokens = 3,
+ repetition_penalty = 1.0,
+ stop = ["STOP"],
+ )
+ )
+
+ assert snapshots == ["Hello"]
+ assert model.sent == ["Hello ", "ST", "OP"]
+ assert backend.last_generation_stats["truncated"] is False
+
+
+def test_a_partial_stop_sequence_is_released_when_the_reply_ends():
+ backend, _ = _backend(["Hello ", "ST"], _Tokenizer())
+
+ snapshots = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = ["STOP"]))
+
+ assert snapshots == ["Hello", "Hello ST"]
+
+
[email protected]("vision", [False, True])
+def test_compact_json_stops_in_producer_before_word_flush(vision):
+ pieces = ['{"id":1}', ',{"id":2}'] * 100
+ backend, model = _backend(pieces, _Processor() if vision else _Tokenizer())
+ if vision:
+ backend.format_chat_prompt = lambda *a, **k: "PROMPT"
+ output = list(
+ backend._generate_vision_response(
+ messages = [{"role": "user", "content": "hi"}],
+ system_prompt = "",
+ image = None,
+ temperature = 0.0,
+ top_p = 1.0,
+ top_k = 0,
+ min_p = 0.0,
+ max_new_tokens = 200,
+ repetition_penalty = 1.0,
+ stop = ["}"],
+ )
+ )
+ else:
+ output = list(backend.generate_stream("PROMPT", max_new_tokens = 200, stop = ["}"]))
+ assert model.sent == [pieces[0]]
+ assert output == ['{"id":1']
+ assert backend.last_generation_stats["truncated"] is False
+
+
+ "kind,raw,stop",
+ [
+ (
+ "harmony",
+ "<|channel|>analysis<|message|>Reasoning<|channel|>final<|message|>Answer",
+ "<think>",
+ ),
+ ("reasoning", "<|channel>thoughtReasoning <channel|>Answer ", "</think>"),
+ ],
+)
+def test_synthetic_reasoning_tags_do_not_match_stops(kind, raw, stop):
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+ tokenizer = _Tokenizer()
+ tokenizer.pieces = {2: raw}
+ if kind == "harmony":
+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)
+ else:
+ streamer = inf.ReasoningTextIteratorStreamer(
+ tokenizer,
+ markers = ("<|channel>thought", "<channel|>"),
+ skip_prompt = False,
+ )
+ wrapped = inf._StopSequenceStreamer(streamer, [stop])
+ wrapped.put(torch.tensor([2]))
+ wrapped.end()
+ output = []
+ while True:
+ try:
+ output.append(next(wrapped))
+ except StopIteration:
+ break
+ assert not wrapped.matched.is_set()
+ assert "Answer" in "".join(output)
+
+
[email protected]("stop", [None, [], ["missing"]])
+def test_requests_without_a_matching_stop_keep_all_text(stop):
+ backend, model = _backend(["Hello ", "world"], _Tokenizer())
+ output = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = stop))
+ assert output[-1] == "Hello world"
+ assert model.sent == ["Hello ", "world"]
+
+
[email protected]("abort", [False, True])
+def test_stop_wrapper_finishes_reasoning_once(abort):
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+ tokenizer = _Tokenizer()
+ tokenizer.pieces = {2: "<|channel>thoughtReasoning ST"}
+ streamer = inf.ReasoningTextIteratorStreamer(
+ tokenizer,
+ markers = ("<|channel>thought", "<channel|>"),
+ skip_prompt = False,
+ )
+ wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])
+ wrapped.put(torch.tensor([2]))
+ if abort:
+ wrapped.abort()
+ wrapped.end()
+ wrapped.end()
+ output = "".join(wrapped)
+ assert "Reasoning ST" in output
+ assert output.count("</think>") == (0 if abort else 1)
+
+
+def test_stop_matching_decodes_split_unicode_without_leaking_partial_bytes():
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+
+ class Tokenizer(_Tokenizer):
+ pieces = {2: b"Hello \xc3", 3: b"\xa9END"}
+
+ def decode(self, ids, **kwargs):
+ return b"".join(self.pieces[int(i)] for i in ids).decode("utf-8", errors = "replace")
+
+ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
+ wrapped = inf._StopSequenceStreamer(streamer, ["éEND"])
+ wrapped.put(torch.tensor([2]))
+ assert not wrapped.matched.is_set()
+ wrapped.put(torch.tensor([3]))
+ assert wrapped.matched.is_set()
+ wrapped.end()
+ assert "".join(wrapped) == "Hello "
+
+
+def test_a_stop_created_by_decode_cleanup_of_settled_text_still_matches():
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+
+ class Tokenizer(_Tokenizer):
+ pieces = {2: "Hello world", 3: " ", 4: ".", 5: " More"}
+
+ def decode(self, ids, **kwargs):
+ # Mirrors clean_up_tokenization_spaces, which rewrites " ." as ".".
+ return super().decode(ids, **kwargs).replace(" .", ".")
+
+ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
+ wrapped = inf._StopSequenceStreamer(streamer, ["."])
+ for token in (2, 3, 4):
+ wrapped.put(torch.tensor([token]))
+ assert wrapped.matched.is_set()
+ wrapped.end()
+ # The space was already streamed before "." rewrote it, as with a full decode.
+ assert "".join(wrapped) == "Hello world "
+
+
+ "pieces, rule, stop",
+ [
+ # " n't" completes one character after the apostrophe, " 've" three characters
+ # after the space, so a guard that only watches the character right after the
+ # space catches the first and misses the second.
+ ({2: "I", 3: " ca", 4: " ", 5: "n", 6: "'t"}, (" n't", "n't"), "can't"),
+ ({2: "we", 3: " ", 4: "'", 5: "v", 6: "e"}, (" 've", "'ve"), "we've"),
+ ({2: "it", 3: " ", 4: "'", 5: "s"}, (" 's", "'s"), "it's"),
+ ],
+)
+def test_a_cleanup_rewrite_split_across_tokens_still_matches_a_stop(pieces, rule, stop):
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+
+ class Tokenizer(_Tokenizer):
+ # Reported as well as applied: the wider re-decode is gated on this, since it is
+ # pure cost for the tokenizers that ship cleanup off.
+ clean_up_tokenization_spaces = True
+
+ def decode(self, ids, **kwargs):
+ return super().decode(ids, **kwargs).replace(*rule)
+
+ # Split across tokens the rewrite is invisible from a short window: each half decodes
+ # unchanged, so the concatenation keeps the space the full decode drops and the stop
+ # is never found.
+ tokenizer = Tokenizer()
+ tokenizer.pieces = pieces
+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)
+ wrapped = inf._StopSequenceStreamer(streamer, [stop])
+ for token in sorted(pieces):
+ wrapped.put(torch.tensor([token]))
+ assert wrapped.matched.is_set()
+ wrapped.end()
+ assert stop not in "".join(wrapped)
+
+
+def test_an_unfinished_byte_run_is_not_settled_before_it_completes():
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+
+ class Tokenizer(_Tokenizer):
+ pieces = {2: "Hi ", 3: b"\xf0", 4: b"\x9f", 5: b"\x99", 6: b"\x82"}
+ pieces.update({7: b"\xf0", 8: b"\x9f", 9: b"\x99", 10: b"\x83", 11: " done"})
+
+ def decode(self, ids, **kwargs):
+ # Like SentencePiece byte fallback: an unfinished byte run is all replacements.
+ out, run = [], b""
+ for token in ids:
+ piece = self.pieces[int(token)]
+ if isinstance(piece, bytes):
+ run += piece
+ continue
+ out += [self._flush(run), piece]
+ run = b""
+ return "".join(out + [self._flush(run)])
+
+ @staticmethod
+ def _flush(run):
+ try:
+ return run.decode("utf-8")
+ except UnicodeDecodeError:
+ return "\ufffd" * len(run)
+
+ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)
+ wrapped = inf._StopSequenceStreamer(streamer, ["never"])
+ for token in range(2, 12):
+ wrapped.put(torch.tensor([token]))
+ wrapped.end()
+ assert "".join(wrapped) == "Hi \U0001f642\U0001f643 done"
+
+
+def test_stop_matching_decodes_a_bounded_window_per_token():
+ inf = pytest.importorskip("core.inference.inference")
+ torch = pytest.importorskip("torch")
+
+ class Tokenizer(_Tokenizer):
+ decoded = 0
+
+ def decode(self, ids, **kwargs):
+ self.decoded += len(ids)
+ return super().decode(ids, **kwargs)
+
+ tokenizer = Tokenizer()
+ words = 2000
+ tokenizer.pieces = {i: "word " for i in range(2, words + 2)}
+ tokenizer.pieces.update({words + 2 + i: ch for i, ch in enumerate("STOP")})
+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)
+ wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])
+ for token in range(2, words + 2):
+ wrapped.put(torch.tensor([token]))
+ assert not wrapped.matched.is_set()
+ assert tokenizer.decoded < 10 * words
+ for token in range(words + 2, words + 6):
+ wrapped.put(torch.tensor([token]))
+ assert wrapped.matched.is_set()
+ wrapped.end()
+ assert "".join(wrapped) == "word " * words