Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Order the Ollama stop-token scan instead of leaving it to a set (#10101)

Vineeth Sai Varikuntlabc87b07e5169
diff --git a/.github/workflows/consolidated-tests-ci.yml b/.github/workflows/consolidated-tests-ci.yml
index 67eb6c724d7..437f10ff893 100644
--- a/.github/workflows/consolidated-tests-ci.yml
+++ b/.github/workflows/consolidated-tests-ci.yml
@@ -443,6 +443,7 @@ jobs:
tests/utils/test_trunc_normal_patch.py \
tests/python/test_fast_language_model_text_only.py \
tests/test_bad_mappings_redirect.py \
+ tests/test_ollama_eos_token_order.py \
tests/test_prefetch_snapshot_scope.py \
tests/test_gemma_2b_mapper_key.py \
tests/test_raw_text_json_loading.py \
diff --git a/tests/test_ollama_eos_token_order.py b/tests/test_ollama_eos_token_order.py
new file mode 100644
index 00000000000..11242eaa1ad
--- /dev/null
+++ b/tests/test_ollama_eos_token_order.py
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""get_ollama_eos_tokens must not depend on set iteration order.
+
+Its prefix collapse rewrites `joined_text` as it walks the token list, so a family member
+has to be seen before any shorter token sharing its prefix; a Gemma-shaped vocabulary gave
+`["<eos>", "<unk>", "<unused"]` or `["<eos>", "<un"]` depending on PYTHONHASHSEED.
+chat_templates.py needs a GPU to import, so the function is ast-extracted (the way
+tests/test_bad_mappings_redirect.py extracts its own).
+"""
+
+import ast
+import json
+import os
+import subprocess
+import sys
+
+_CHAT_TEMPLATES = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "chat_templates.py")
+
+# `<unk>` shares its first three characters with the `<unused*>` family: that is what
+# makes the order matter.
+_TOKENS = ["<pad>", "<eos>", "<unk>", "<unused0>", "<unused1>", "<unused2>"]
+_BOS = "<pad>"
+
+_RUNNER = """
+import ast, json, sys
+source, tokens, bos = json.loads(sys.argv[1]), json.loads(sys.argv[2]), json.loads(sys.argv[3])
+namespace = {}
+exec(compile(source, "get_ollama_eos_tokens", "exec"), namespace)
+
+
+class _Tokenizer:
+ def __init__(self, tokens, bos_token):
+ self._tokens, self.bos_token = tokens, bos_token
+
+ @property
+ def added_tokens_decoder(self):
+ return {index: token for index, token in enumerate(self._tokens)}
+
+
+print(json.dumps(sorted(namespace["get_ollama_eos_tokens"](_Tokenizer(tokens, bos), []))))
+"""
+
+
+def _extract_source():
+ with open(_CHAT_TEMPLATES, encoding = "utf-8") as f:
+ source = f.read()
+ for node in ast.parse(source).body:
+ if isinstance(node, ast.FunctionDef) and node.name == "get_ollama_eos_tokens":
+ return ast.get_source_segment(source, node)
+ raise AssertionError("get_ollama_eos_tokens not found in chat_templates.py")
+
+
+def _run(seed):
+ environment = dict(os.environ, PYTHONHASHSEED = str(seed))
+ completed = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ _RUNNER,
+ json.dumps(_extract_source()),
+ json.dumps(_TOKENS),
+ json.dumps(_BOS),
+ ],
+ capture_output = True,
+ text = True,
+ env = environment,
+ check = True,
+ )
+ return json.loads(completed.stdout)
+
+
+def test_ollama_eos_tokens_do_not_depend_on_hash_seed():
+ results = [_run(seed) for seed in range(8)]
+ assert len(set(map(tuple, results))) == 1, (
+ "get_ollama_eos_tokens returned different stop tokens for the same tokenizer "
+ f"under different PYTHONHASHSEEDs: {sorted(set(map(tuple, results)))}"
+ )
+
+
+def test_ollama_eos_tokens_keep_unk_beside_an_unused_family():
+ for seed in range(8):
+ assert _run(seed) == ["<eos>", "<unk>", "<unused"]
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index 82ae96f6259..58936f9d825 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -2371,8 +2371,15 @@ def get_ollama_eos_tokens(tokenizer, extra_eos_tokens = []):
added_tokens_decoder = tokenizer.added_tokens_decoder.values()
added_tokens_decoder = [str(x) for x in added_tokens_decoder]
- # Remove added_tokens_decoder duplicates
- added_tokens_decoder = list(set(added_tokens_decoder) - set(extra_eos_tokens))
+ # Longest first, never through a set: the collapse below rewrites joined_text as it goes,
+ # so a family member must be seen before any shorter token sharing its prefix (`<unused0>`
+ # before `<unk>`), and set() order over strings varies with PYTHONHASHSEED.
+ skip_eos_tokens = set(extra_eos_tokens)
+ added_tokens_decoder = sorted(
+ (x for x in dict.fromkeys(added_tokens_decoder) if x not in skip_eos_tokens),
+ key = len,
+ reverse = True,
+ )
# Remove BOS
if getattr(tokenizer, "bos_token", None) is not None: