Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Stop at the requested stop text on transformers models (#10812)
Nilay9360dcce488f
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex f9ea15e672b..7f566dc3b66 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -13,6 +13,7 @@ from peft import PeftModel, PeftModelForCausalLMimport contextlibimport jsonimport sys+import threadingimport torchfrom pathlib import Pathfrom typing import Optional, Union, Generator, Tuple@@ -61,6 +62,7 @@ from core.inference.native_tool_tokens import (reasoning_control_tokens,stop_token_text,)+from core.inference.mlx_inference import _mlx_stop_cut, _mlx_stop_sequencesfrom io import StringIOimport structlogfrom loggers import get_logger@@ -258,6 +260,198 @@ class _GenerationThreadError(RuntimeError):"""Generation worker failures that should propagate through stream routes."""+# What clean_up_tokenization_spaces deletes a space before. Every rule is a space plus+# one of these, and the longest is three characters, so a rule spans at most four.+_CLEANUP_TAILS = (".", "?", "!", ",", "' ", "n't", "'m", "'s", "'ve", "'re")+_CLEANUP_SPAN_CHARS = 4+# Tokens, not characters: those four characters can arrive as four one-character tokens,+# and eight leaves room for tokens that decode to "".+_CLEANUP_SPAN_TOKENS = 8+++def _cleanup_join_pending(text: str) -> bool:+ """Could a cleanup rule still be completed at the end of ``text``?++ Only the last few characters can matter, and only after a space: a rule whose space is+ further back than that is already settled either way. A trailing space counts, since the+ next token can start any of the rules.+ """+ window = text[-_CLEANUP_SPAN_CHARS:]+ for i, char in enumerate(window):+ if char != " ":+ continue+ rest = window[i + 1 :]+ if not rest or any(tail.startswith(rest) for tail in _CLEANUP_TAILS):+ return True+ return False+++def _decoder_cleans_up(streamer) -> bool:+ """Does this streamer's decode apply clean_up_tokenization_spaces?++ Read once, because the widened re-decode below is pure cost for the tokenizers that do+ not (Qwen, Gemma, Mistral, gpt-oss ship it off, and transformers 5.17 ignores it for+ every BPE tokenizer). The streamer's own kwargs win, then the tokenizer's default,+ unwrapping NativeToolTokenDecoder to reach it.+ """+ kwargs = getattr(streamer, "decode_kwargs", None) or {}+ if "clean_up_tokenization_spaces" in kwargs:+ return bool(kwargs["clean_up_tokenization_spaces"])+ tokenizer = getattr(streamer, "tokenizer", None)+ for _ in range(4):+ if tokenizer is None:+ break+ flag = getattr(tokenizer, "clean_up_tokenization_spaces", None)+ if flag is not None:+ return bool(flag)+ tokenizer = getattr(tokenizer, "tokenizer", None)+ return False+++class _StopSequenceStreamer:+ def __init__(self, streamer, stop):+ self.streamer = streamer+ self.sequences = _mlx_stop_sequences(stop)+ self.matched = threading.Event()+ self.token_ids = []+ self.text = ""+ self.settled = ""+ self.prefix_offset = 0+ self.read_offset = 0+ self.scan_from = 0+ self.longest = max((len(s) for s in self.sequences), default = 0)+ self.released = 0+ self.cut = 0+ self.finished = False+ self.next_tokens_are_prompt = bool(getattr(streamer, "skip_prompt", False))+ self.is_harmony = isinstance(streamer, HarmonyTextStreamer)+ self.cleans_up = _decoder_cleans_up(streamer)++ def __next__(self):+ return next(self.streamer)++ def __iter__(self):+ return self++ def put(self, value):+ if not self.sequences:+ return self.streamer.put(value)+ if self.finished or self.matched.is_set():+ return+ # Matches the supported batch-one TextIteratorStreamer protocol.+ if len(value.shape) > 1:+ if value.shape[0] > 1:+ raise ValueError("TextStreamer only supports batch size 1")+ value = value[0]+ if self.next_tokens_are_prompt:+ self.next_tokens_are_prompt = False+ return+ self.token_ids.extend(value.tolist())+ self._decode_new_tokens()+ # Earlier text was already scanned; only a stop overlapping new text can match.+ start = self.scan_from+ cut, matched = _mlx_stop_cut(self.text[start:], self.sequences)+ self.cut = start + cut+ if matched:+ # This runs in the producer, before model.generate checks criteria.+ self.matched.set()+ else:+ self.scan_from = max(start, len(self.settled) - self.longest + 1)+ cut = self.cut+ if not matched and not self.is_harmony and cut > self.released:+ # Keep the word-buffering stability guarantee for displayed text.+ # Stop detection above must still inspect every decoded token.+ if not self.streamer._is_chinese_char(ord(self.text[cut - 1])):+ cut = (+ max(+ self.text.rfind(" ", self.released, cut),+ self.text.rfind("\n", self.released, cut),+ )+ + 1+ )+ self._publish(cut)++ def _decode(self, token_ids):+ decode_kwargs = (+ {"skip_special_tokens": False} if self.is_harmony else self.streamer.decode_kwargs+ )+ return self.streamer.tokenizer.decode(token_ids, **decode_kwargs)++ def _decode_new_tokens(self):+ # Re-decoding the whole reply each token is quadratic; decode a short window+ # and settle its text once it no longer ends in unresolved bytes.+ prefix = self._decode(self.token_ids[self.prefix_offset : self.read_offset])+ window = self._decode(self.token_ids[self.prefix_offset :])+ if (+ self.cleans_up+ and window.startswith(prefix)+ and _cleanup_join_pending(self.settled + window[len(prefix) :])+ ):+ # A cleanup rewrite that spans the join is invisible from this window: split+ # across tokens (" ", "'", "v", "e") each half decodes unchanged, so+ # ``startswith`` holds and the concatenation keeps a space the full decode+ # drops. Settled text then diverges permanently and a stop written across the+ # join never matches. Re-read the few tokens behind the join while a rule could+ # still complete there, so ordinary text keeps the short window.+ wider = max(0, self.read_offset - _CLEANUP_SPAN_TOKENS)+ if wider < self.prefix_offset:+ wide_prefix = self._decode(self.token_ids[wider : self.read_offset])+ # Splicing onto text this prefix does not end is worse than the divergence+ # it repairs, so fall back to the short window rather than guess.+ if self.settled.endswith(wide_prefix):+ prefix = wide_prefix+ window = self._decode(self.token_ids[wider:])+ if window.endswith("\ufffd"):+ # Bytes still arriving can rewrite the whole window (byte-fallback tokenizers+ # show an unfinished emoji as replacement characters), so wait for them.+ tail = window[len(prefix) :] if window.startswith(prefix) else ""+ self.text = self.settled + tail+ return+ if not window.startswith(prefix):+ # The new tokens rewrote settled text (e.g. space cleanup turning " ." into+ # "."), so rebuild it once from every token and rescan it for stops.+ self.settled = self._decode(self.token_ids)+ self.prefix_offset = max(0, len(self.token_ids) - 4)+ self.read_offset = len(self.token_ids)+ self.scan_from = 0+ elif len(window) > len(prefix):+ self.settled += window[len(prefix) :]+ self.prefix_offset = self.read_offset+ self.read_offset = len(self.token_ids)+ self.text = self.settled++ def _publish(self, cut):+ if cut <= self.released:+ return+ if self.is_harmony:+ # Harmony consumes cumulative raw text and synthesizes tags itself.+ self.streamer._process_incremental(self.text[:cut])+ else:+ # The reasoning subclass normalizes here; plain text just queues it.+ self.streamer.on_finalized_text(self.text[self.released : cut])+ self.released = cut++ def end(self):+ if not self.sequences:+ return self.streamer.end()+ if self.finished:+ return+ self.finished = True+ if not self.matched.is_set() and self.read_offset < len(self.token_ids):+ # Bytes that never resolved still belong to the reply, as a full decode shows.+ self.text = self._decode(self.token_ids)+ # A natural end releases a partial unmatched stop and unresolved bytes.+ self._publish(self.cut if self.matched.is_set() else len(self.text))+ # Their token caches are empty: put() above feeds raw decoded text, so+ # end() only finishes reasoning framing and sends the queue sentinel.+ self.streamer.end()++ def abort(self):+ abort = getattr(self.streamer, "abort", None)+ if abort is not None:+ abort()++def _prompt_already_has_bos(tokenizer, prompt):"""Did the rendered chat template emit BOS itself?@@ -1039,6 +1233,7 @@ class InferenceBackend:continue_final_message: bool = False,presence_penalty: float = 0.0,tool_protocol_active: Optional[bool] = None,+ stop: Optional[list] = None,) -> Generator[str, None, None]:"""Generate response for text or vision models (lock held by background thread).@@ -1065,6 +1260,7 @@ class InferenceBackend:continue_final_message = continue_final_message,tool_protocol_active = tool_protocol_active,presence_penalty = presence_penalty,+ stop = stop,)def _generate_chat_response_inner(@@ -1087,6 +1283,7 @@ class InferenceBackend:continue_final_message: bool = False,presence_penalty: float = 0.0,tool_protocol_active: Optional[bool] = None,+ stop: Optional[list] = None,) -> Generator[str, None, None]:"""Inner generation logic.@@ -1131,6 +1328,7 @@ class InferenceBackend:continue_final_message = continue_final_message,tools = tools,tool_protocol_active = tool_protocol_active,+ stop = stop,)returnelse:@@ -1262,6 +1460,7 @@ class InferenceBackend:if tool_protocol_active is Noneelse tool_protocol_active,add_special_tokens = add_special_tokens,+ stop = stop,)def _generate_vision_response(@@ -1280,6 +1479,7 @@ class InferenceBackend:continue_final_message: bool = False,tools: Optional[list] = None,tool_protocol_active: Optional[bool] = None,+ stop: Optional[list] = None,) -> Generator[str, None, None]:"""Handle vision model generation with true token-by-token streaming."""# Reset so a failed or uncountable run cannot surface stale stats.@@ -1529,7 +1729,10 @@ class InferenceBackend:)timer = GenerationTimer()generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)- stopping_criteria = self._cancel_stopping_criteria(cancel_event)+ stop_streamer = _StopSequenceStreamer(streamer, stop)+ if stop_streamer.sequences:+ generation_kwargs["streamer"] = stop_streamer+ stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)if stopping_criteria is not None:generation_kwargs["stopping_criteria"] = stopping_criteriaactive_stop_token_ids = self._generation_stop_token_ids(model, generation_kwargs)@@ -1547,13 +1750,12 @@ class InferenceBackend:gen_outputs["sequences"] = model.generate(**generation_kwargs)except Exception as e:err["msg"] = str(e)- if hasattr(streamer, "abort"):- streamer.abort()+ stop_streamer.abort()logger.error(f"Vision generation error in thread: {e}")finally:timer.finish()try:- streamer.end()+ stop_streamer.end()except Exception:pass@@ -1578,7 +1780,7 @@ class InferenceBackend:elif time.monotonic() >= cancel_deadline:breaktry:- new_token = next(streamer)+ new_token = next(stop_streamer)except StopIteration:generation_complete = Truebreak@@ -1586,7 +1788,7 @@ class InferenceBackend:if not thread.is_alive():generation_complete = Trueoutput = yield from self._drain_streamer_tail(- streamer, output, active_stop_token_ids+ stop_streamer, output, active_stop_token_ids)breakif cancel_deadline is not None:@@ -1598,7 +1800,7 @@ class InferenceBackend:breakgeneration_complete = Trueoutput = yield from self._drain_streamer_tail(- streamer, output, active_stop_token_ids+ stop_streamer, output, active_stop_token_ids)breakcontinue@@ -1626,9 +1828,8 @@ class InferenceBackend:model, gen_outputs["sequences"], prompt_len),max_new_tokens = max_new_tokens,- ended_on_stop_token = self._ended_on_stop_token(- gen_outputs["sequences"], active_stop_token_ids- ),+ ended_on_stop_token = stop_streamer.matched.is_set()+ or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),cancelled = not generation_completeor (cancel_event is not None and cancel_event.is_set()),timer = timer,@@ -1957,6 +2158,7 @@ class InferenceBackend:continued: bool = False,preserve_tool_tokens: bool = False,add_special_tokens: bool = True,+ stop: Optional[list] = None,) -> Generator[str, None, None]:"""Generate a streaming text response (text models only).@@ -2044,7 +2246,10 @@ class InferenceBackend:_pp = _make_presence_penalty_processor(presence_penalty, prompt_len)timer = GenerationTimer()generation_kwargs["logits_processor"] = with_prefill_boundary_processor(_pp, timer)- stopping_criteria = self._cancel_stopping_criteria(cancel_event)+ stop_streamer = _StopSequenceStreamer(streamer, stop)+ if stop_streamer.sequences:+ generation_kwargs["streamer"] = stop_streamer+ stopping_criteria = self._cancel_stopping_criteria(cancel_event, stop_streamer.matched)if stopping_criteria is not None:generation_kwargs["stopping_criteria"] = stopping_criteria@@ -2060,13 +2265,12 @@ class InferenceBackend:gen_outputs["sequences"] = model.generate(**generation_kwargs)except Exception as e:err["msg"] = str(e)- if hasattr(streamer, "abort"):- streamer.abort()+ stop_streamer.abort()logger.error(f"Generation error: {e}")finally:timer.finish()try:- streamer.end()+ stop_streamer.end()except Exception:pass@@ -2093,7 +2297,7 @@ class InferenceBackend:elif time.monotonic() >= cancel_deadline:breaktry:- new_token = next(streamer)+ new_token = next(stop_streamer)except StopIteration:generation_complete = Truebreak@@ -2101,7 +2305,7 @@ class InferenceBackend:if not thread.is_alive():generation_complete = Trueoutput = yield from self._drain_streamer_tail(- streamer, output, active_stop_token_ids+ stop_streamer, output, active_stop_token_ids)breakif cancel_deadline is not None:@@ -2113,7 +2317,7 @@ class InferenceBackend:breakgeneration_complete = Trueoutput = yield from self._drain_streamer_tail(- streamer, output, active_stop_token_ids+ stop_streamer, output, active_stop_token_ids)breakcontinue@@ -2143,9 +2347,8 @@ class InferenceBackend:model, gen_outputs["sequences"], prompt_len),max_new_tokens = max_new_tokens,- ended_on_stop_token = self._ended_on_stop_token(- gen_outputs["sequences"], active_stop_token_ids- ),+ ended_on_stop_token = stop_streamer.matched.is_set()+ or self._ended_on_stop_token(gen_outputs["sequences"], active_stop_token_ids),cancelled = not generation_completeor (cancel_event is not None and cancel_event.is_set()),timer = timer,@@ -2855,9 +3058,10 @@ class InferenceBackend:stats["timings"] = timingsself.last_generation_stats = stats- def _cancel_stopping_criteria(self, cancel_event):+ def _cancel_stopping_criteria(self, *events):"""Build a Transformers stopping criteria list for user cancellation."""- if cancel_event is None:+ events = [ev for ev in events if ev is not None]+ if not events:return Nonefrom transformers.generation.stopping_criteria import (StoppingCriteria,@@ -2871,7 +3075,7 @@ class InferenceBackend:def __call__(self, input_ids, scores, **kwargs):return self.ev.is_set()- return StoppingCriteriaList([_CancelCriteria(cancel_event)])+ return StoppingCriteriaList([_CancelCriteria(ev) for ev in events])def _clean_generated_text(self,diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.pyindex 3c1f42a635a..acc84d7ca03 100644--- a/studio/backend/core/inference/worker.py+++ b/studio/backend/core/inference/worker.py@@ -702,9 +702,9 @@ def _handle_generate(backend, cmd: dict, resp_queue: Any, cancel_event) -> None:if opt_key in cmd:gen_kwargs[opt_key] = cmd[opt_key]- # These options are MLX-only. The transformers backend declares none of- # them and takes no **kwargs, so forwarding unconditionally would turn- # its documented "ignores them" behavior into a TypeError.+ # Not every backend declares these (transformers declares only ``stop``)+ # and none takes **kwargs, so forwarding unconditionally would turn a+ # backend's documented "ignores them" behavior into a TypeError.# ``tool_protocol_active`` rides here rather than above: MLX declares no such# parameter and takes no **kwargs, so an unconditional forward would raise.for gated in ("seed", "frequency_penalty", "logit_bias", "stop", "tool_protocol_active"):diff --git a/studio/backend/tests/test_transformers_stop_sequences.py b/studio/backend/tests/test_transformers_stop_sequences.pynew file mode 100644index 00000000000..63083c1f8bf--- /dev/null+++ b/studio/backend/tests/test_transformers_stop_sequences.py@@ -0,0 +1,405 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import importlib+import sys+import threading+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++try:+ import core.inference.inference # noqa: E402,F401+except ImportError:+ pass++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++class _Batch(dict):+ def to(self, *_args, **_kwargs):+ return self+++class _Tokenizer:+ all_special_tokens: list = []+ eos_token_id = 1+ pad_token_id = None+ chat_template = "{{ messages }}"++ def __call__(self, *_args, **_kwargs):+ torch = pytest.importorskip("torch")+ return _Batch({"input_ids": torch.zeros((1, 1), dtype = torch.long)})++ def decode(self, ids, **kwargs):+ return "".join(self.pieces.get(int(i), "") for i in ids)+++class _Processor:+ chat_template = None+ tokenizer = _Tokenizer()+++class _Model:+ device = "cpu"+ generation_config = type("Cfg", (), {"eos_token_id": 1})()+ config = generation_config++ def __init__(self, pieces):+ self.pieces = pieces+ self.sent = []++ def generate(self, streamer, stopping_criteria, max_new_tokens, **_kwargs):+ torch = pytest.importorskip("torch")+ ids = torch.zeros((1, 1), dtype = torch.long)+ streamer.put(ids)+ for index, piece in enumerate(self.pieces[:max_new_tokens], 2):+ ids = torch.cat([ids, torch.tensor([[index]])], dim = 1)+ streamer.put(torch.tensor([index]))+ self.sent.append(piece)+ if stopping_criteria is not None and bool(stopping_criteria(ids, None).all()):+ break+ streamer.end()+ return ids+++def _backend(pieces, tokenizer):+ inf = pytest.importorskip("core.inference.inference")+ model = _Model(pieces)+ raw_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)+ raw_tokenizer.pieces = dict(enumerate(pieces, 2))+ streamer = inf.TextIteratorStreamer(raw_tokenizer, skip_prompt = True, timeout = 0.2)+ backend = inf.InferenceBackend.__new__(inf.InferenceBackend)+ backend.active_model_name = "stop-test"+ backend._generation_lock = threading.Lock()+ backend.models = {"stop-test": {"model": model, "tokenizer": tokenizer, "processor": tokenizer}}+ backend._make_text_streamer = lambda *_args, **_kwargs: streamer+ return backend, model+++class _Responses(list):+ def put(self, response):+ self.append(response)+++def test_worker_forwards_stop_and_the_reply_ends_before_it(monkeypatch):+ from core.inference import worker++ backend, model = _backend(["Hello ", "ST", "OP", " world"], _Tokenizer())+ monkeypatch.setattr(+ backend, "_apply_chat_template_for_generation", lambda *a, **k: "PROMPT", raising = False+ )+ responses = _Responses()+ cmd = {+ "request_id": "r",+ "messages": [{"role": "user", "content": "hi"}],+ "max_new_tokens": 3,+ "stop": ["STOP"],+ }++ worker._handle_generate(backend, cmd, responses, threading.Event())++ assert [r["text"] for r in responses if r["type"] == "token"] == ["Hello"]+ assert model.sent == ["Hello ", "ST", "OP"]+ done = next(r for r in responses if r["type"] == "gen_done")+ assert done["stats"]["truncated"] is False+++def test_vision_reply_ends_before_a_stop_sequence():+ backend, model = _backend(["Hello ", "ST", "OP", " world"], _Processor())+ backend.format_chat_prompt = lambda *_args, **_kwargs: "PROMPT"++ snapshots = list(+ backend._generate_vision_response(+ messages = [{"role": "user", "content": "hi"}],+ system_prompt = "",+ image = None,+ temperature = 0.0,+ top_p = 1.0,+ top_k = 0,+ min_p = 0.0,+ max_new_tokens = 3,+ repetition_penalty = 1.0,+ stop = ["STOP"],+ )+ )++ assert snapshots == ["Hello"]+ assert model.sent == ["Hello ", "ST", "OP"]+ assert backend.last_generation_stats["truncated"] is False+++def test_a_partial_stop_sequence_is_released_when_the_reply_ends():+ backend, _ = _backend(["Hello ", "ST"], _Tokenizer())++ snapshots = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = ["STOP"]))++ assert snapshots == ["Hello", "Hello ST"]++[email protected]("vision", [False, True])+def test_compact_json_stops_in_producer_before_word_flush(vision):+ pieces = ['{"id":1}', ',{"id":2}'] * 100+ backend, model = _backend(pieces, _Processor() if vision else _Tokenizer())+ if vision:+ backend.format_chat_prompt = lambda *a, **k: "PROMPT"+ output = list(+ backend._generate_vision_response(+ messages = [{"role": "user", "content": "hi"}],+ system_prompt = "",+ image = None,+ temperature = 0.0,+ top_p = 1.0,+ top_k = 0,+ min_p = 0.0,+ max_new_tokens = 200,+ repetition_penalty = 1.0,+ stop = ["}"],+ )+ )+ else:+ output = list(backend.generate_stream("PROMPT", max_new_tokens = 200, stop = ["}"]))+ assert model.sent == [pieces[0]]+ assert output == ['{"id":1']+ assert backend.last_generation_stats["truncated"] is False+++ "kind,raw,stop",+ [+ (+ "harmony",+ "<|channel|>analysis<|message|>Reasoning<|channel|>final<|message|>Answer",+ "<think>",+ ),+ ("reasoning", "<|channel>thoughtReasoning <channel|>Answer ", "</think>"),+ ],+)+def test_synthetic_reasoning_tags_do_not_match_stops(kind, raw, stop):+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")+ tokenizer = _Tokenizer()+ tokenizer.pieces = {2: raw}+ if kind == "harmony":+ streamer = inf.HarmonyTextStreamer(tokenizer, skip_prompt = False)+ else:+ streamer = inf.ReasoningTextIteratorStreamer(+ tokenizer,+ markers = ("<|channel>thought", "<channel|>"),+ skip_prompt = False,+ )+ wrapped = inf._StopSequenceStreamer(streamer, [stop])+ wrapped.put(torch.tensor([2]))+ wrapped.end()+ output = []+ while True:+ try:+ output.append(next(wrapped))+ except StopIteration:+ break+ assert not wrapped.matched.is_set()+ assert "Answer" in "".join(output)++[email protected]("stop", [None, [], ["missing"]])+def test_requests_without_a_matching_stop_keep_all_text(stop):+ backend, model = _backend(["Hello ", "world"], _Tokenizer())+ output = list(backend.generate_stream("PROMPT", max_new_tokens = 8, stop = stop))+ assert output[-1] == "Hello world"+ assert model.sent == ["Hello ", "world"]++[email protected]("abort", [False, True])+def test_stop_wrapper_finishes_reasoning_once(abort):+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")+ tokenizer = _Tokenizer()+ tokenizer.pieces = {2: "<|channel>thoughtReasoning ST"}+ streamer = inf.ReasoningTextIteratorStreamer(+ tokenizer,+ markers = ("<|channel>thought", "<channel|>"),+ skip_prompt = False,+ )+ wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])+ wrapped.put(torch.tensor([2]))+ if abort:+ wrapped.abort()+ wrapped.end()+ wrapped.end()+ output = "".join(wrapped)+ assert "Reasoning ST" in output+ assert output.count("</think>") == (0 if abort else 1)+++def test_stop_matching_decodes_split_unicode_without_leaking_partial_bytes():+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")++ class Tokenizer(_Tokenizer):+ pieces = {2: b"Hello \xc3", 3: b"\xa9END"}++ def decode(self, ids, **kwargs):+ return b"".join(self.pieces[int(i)] for i in ids).decode("utf-8", errors = "replace")++ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)+ wrapped = inf._StopSequenceStreamer(streamer, ["éEND"])+ wrapped.put(torch.tensor([2]))+ assert not wrapped.matched.is_set()+ wrapped.put(torch.tensor([3]))+ assert wrapped.matched.is_set()+ wrapped.end()+ assert "".join(wrapped) == "Hello "+++def test_a_stop_created_by_decode_cleanup_of_settled_text_still_matches():+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")++ class Tokenizer(_Tokenizer):+ pieces = {2: "Hello world", 3: " ", 4: ".", 5: " More"}++ def decode(self, ids, **kwargs):+ # Mirrors clean_up_tokenization_spaces, which rewrites " ." as ".".+ return super().decode(ids, **kwargs).replace(" .", ".")++ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)+ wrapped = inf._StopSequenceStreamer(streamer, ["."])+ for token in (2, 3, 4):+ wrapped.put(torch.tensor([token]))+ assert wrapped.matched.is_set()+ wrapped.end()+ # The space was already streamed before "." rewrote it, as with a full decode.+ assert "".join(wrapped) == "Hello world "+++ "pieces, rule, stop",+ [+ # " n't" completes one character after the apostrophe, " 've" three characters+ # after the space, so a guard that only watches the character right after the+ # space catches the first and misses the second.+ ({2: "I", 3: " ca", 4: " ", 5: "n", 6: "'t"}, (" n't", "n't"), "can't"),+ ({2: "we", 3: " ", 4: "'", 5: "v", 6: "e"}, (" 've", "'ve"), "we've"),+ ({2: "it", 3: " ", 4: "'", 5: "s"}, (" 's", "'s"), "it's"),+ ],+)+def test_a_cleanup_rewrite_split_across_tokens_still_matches_a_stop(pieces, rule, stop):+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")++ class Tokenizer(_Tokenizer):+ # Reported as well as applied: the wider re-decode is gated on this, since it is+ # pure cost for the tokenizers that ship cleanup off.+ clean_up_tokenization_spaces = True++ def decode(self, ids, **kwargs):+ return super().decode(ids, **kwargs).replace(*rule)++ # Split across tokens the rewrite is invisible from a short window: each half decodes+ # unchanged, so the concatenation keeps the space the full decode drops and the stop+ # is never found.+ tokenizer = Tokenizer()+ tokenizer.pieces = pieces+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)+ wrapped = inf._StopSequenceStreamer(streamer, [stop])+ for token in sorted(pieces):+ wrapped.put(torch.tensor([token]))+ assert wrapped.matched.is_set()+ wrapped.end()+ assert stop not in "".join(wrapped)+++def test_an_unfinished_byte_run_is_not_settled_before_it_completes():+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")++ class Tokenizer(_Tokenizer):+ pieces = {2: "Hi ", 3: b"\xf0", 4: b"\x9f", 5: b"\x99", 6: b"\x82"}+ pieces.update({7: b"\xf0", 8: b"\x9f", 9: b"\x99", 10: b"\x83", 11: " done"})++ def decode(self, ids, **kwargs):+ # Like SentencePiece byte fallback: an unfinished byte run is all replacements.+ out, run = [], b""+ for token in ids:+ piece = self.pieces[int(token)]+ if isinstance(piece, bytes):+ run += piece+ continue+ out += [self._flush(run), piece]+ run = b""+ return "".join(out + [self._flush(run)])++ @staticmethod+ def _flush(run):+ try:+ return run.decode("utf-8")+ except UnicodeDecodeError:+ return "\ufffd" * len(run)++ streamer = inf.TextIteratorStreamer(Tokenizer(), skip_prompt = False)+ wrapped = inf._StopSequenceStreamer(streamer, ["never"])+ for token in range(2, 12):+ wrapped.put(torch.tensor([token]))+ wrapped.end()+ assert "".join(wrapped) == "Hi \U0001f642\U0001f643 done"+++def test_stop_matching_decodes_a_bounded_window_per_token():+ inf = pytest.importorskip("core.inference.inference")+ torch = pytest.importorskip("torch")++ class Tokenizer(_Tokenizer):+ decoded = 0++ def decode(self, ids, **kwargs):+ self.decoded += len(ids)+ return super().decode(ids, **kwargs)++ tokenizer = Tokenizer()+ words = 2000+ tokenizer.pieces = {i: "word " for i in range(2, words + 2)}+ tokenizer.pieces.update({words + 2 + i: ch for i, ch in enumerate("STOP")})+ streamer = inf.TextIteratorStreamer(tokenizer, skip_prompt = False)+ wrapped = inf._StopSequenceStreamer(streamer, ["STOP"])+ for token in range(2, words + 2):+ wrapped.put(torch.tensor([token]))+ assert not wrapped.matched.is_set()+ assert tokenizer.decoded < 10 * words+ for token in range(words + 2, words + 6):+ wrapped.put(torch.tensor([token]))+ assert wrapped.matched.is_set()+ wrapped.end()+ assert "".join(wrapped) == "word " * words