Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Match stopping criteria separately for each generated sequence (#11039)

Vineeth Sai Varikuntlaa1a38e92e970
diff --git a/tests/test_stopping_criteria_device.py b/tests/test_stopping_criteria_device.py
index ee388948386..118f61c224a 100644
--- a/tests/test_stopping_criteria_device.py
+++ b/tests/test_stopping_criteria_device.py
@@ -67,3 +67,20 @@ def test_stop_token_follows_the_input_device():
assert criteria[0](torch.tensor([9, 9, 2]), None) is True
assert criteria[0].stop_token.device.type == "cpu"
+
+
+ "stop_word, rows, expected",
+ [
+ ("eos_token", [[7, 8, 2], [7, 8, 3]], [True, False]),
+ ("eos_token", [[7, 8, 3], [7, 8, 2]], [False, True]),
+ ("<|stop|>", [[7, 100, 101], [7, 100, 8]], [True, False]),
+ ("<|stop|>", [[7, 100, 8], [7, 100, 101]], [False, True]),
+ ("<|stop|>", [[100], [101]], [False, False]),
+ ],
+)
+def test_stopping_criteria_matches_each_sequence(stop_word, rows, expected):
+ criteria = create_stopping_criteria(_FakeTokenizer(), stop_word = stop_word)
+ # Exercise Transformers' public aggregation, as generate() does.
+ result = criteria(torch.tensor(rows), scores = None)
+ torch.testing.assert_close(result, torch.tensor(expected))
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index dc8f21d9963..229df1be557 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -2883,16 +2883,17 @@ def create_stopping_criteria(tokenizer, stop_word = "eos_token"):
self.length = self.stop_token.shape[0]
self.single_match = self.length == 1
- def __call__(self, input_ids: LongTensor, scores: FloatTensor) -> bool:
- input_ids = input_ids.ravel()
+ def __call__(self, input_ids: LongTensor, scores: FloatTensor):
+ single_sequence = input_ids.ndim == 1
+ if single_sequence:
+ input_ids = input_ids.unsqueeze(0)
if self.stop_token.device != input_ids.device:
self.stop_token = self.stop_token.to(input_ids.device)
- last_token = input_ids[-1]
- if self.single_match and (last_token == self.stop_token): return True
-
- if input_ids.shape[0] >= self.length and \
- (input_ids[-self.length:] == self.stop_token).all(): return True
- return False
+ if input_ids.shape[-1] < self.length:
+ matches = torch.zeros(input_ids.shape[0], dtype = torch.bool, device = input_ids.device)
+ else:
+ matches = (input_ids[:, -self.length:] == self.stop_token).all(dim = -1)
+ return matches.item() if single_sequence else matches
stopping_criteria = StoppingCriteriaList([StoppingCriteriaSub(stops = stop_word)])
return stopping_criteria