Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Create the SentencePiece scratch directory without a check-then-create race (#12400)

Daniel Han8568ae303830
diff --git a/tests/python/test_get_chat_template_sharegpt_mapping.py b/tests/python/test_get_chat_template_sharegpt_mapping.py
index e3f85a5abe6..bc37b865d72 100644
--- a/tests/python/test_get_chat_template_sharegpt_mapping.py
+++ b/tests/python/test_get_chat_template_sharegpt_mapping.py
@@ -30,6 +30,13 @@ TEMPLATES = [
]
[email protected](autouse = True)
+def _scratch_cwd(tmp_path, monkeypatch):
+ # get_chat_template's SentencePiece fix writes a scratch dir under the working directory;
+ # keep each test's copy out of the checkout and away from other xdist workers.
+ monkeypatch.chdir(tmp_path)
+
+
def _tokenizer():
vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}
return PreTrainedTokenizerFast(
diff --git a/tests/saving/test_fix_sentencepiece_tokenizer_guard.py b/tests/saving/test_fix_sentencepiece_tokenizer_guard.py
index 0bbfeb468a8..27e69ab6cdc 100644
--- a/tests/saving/test_fix_sentencepiece_tokenizer_guard.py
+++ b/tests/saving/test_fix_sentencepiece_tokenizer_guard.py
@@ -310,3 +310,52 @@ def test_only_applied_mappings_are_patched(tmp_path, monkeypatch):
result = _read_pieces(f"{loaded[-1]}/tokenizer.model")
assert result.count("X") == 1 and "Y" in result and "aa" in result, result
assert tok is not None
+
+
+def test_a_scratch_directory_created_by_another_process_is_reused(tmp_path, monkeypatch):
+ """Two processes sharing a working directory both see the scratch dir missing; the
+ second makedirs must not raise. Simulated by reporting the directory absent after
+ another process has already created it."""
+ import unsloth.tokenizer_utils as tokenizer_utils
+
+ _stub_auto_tokenizer(monkeypatch)
+ old, new = _tokenizers()
+ location = str(tmp_path / "_unsloth_sentencepiece_temp")
+ os.makedirs(location)
+ real_exists = os.path.exists
+ monkeypatch.setattr(
+ tokenizer_utils.os.path,
+ "exists",
+ lambda path: False if os.fspath(path) == location else real_exists(path),
+ )
+
+ fix_sentencepiece_tokenizer(old, new, {"</s>": "<|im_end|>"}, temporary_location = location)
+
+ assert old.saved_to
+
+
+def test_every_scratch_makedirs_tolerates_an_existing_directory():
+ """convert_to_fast_tokenizer shares the pattern but needs a real slow tokenizer to reach
+ it, so pin the call shape for every makedirs in the module instead."""
+ import ast
+ import inspect
+
+ import unsloth.tokenizer_utils as tokenizer_utils
+
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(tokenizer_utils)))
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "makedirs"
+ ]
+ assert calls, "no makedirs call found; update this test"
+ racy = [
+ node.lineno
+ for node in calls
+ if not any(
+ kw.arg == "exist_ok" and isinstance(kw.value, ast.Constant) and kw.value.value is True
+ for kw in node.keywords
+ )
+ ]
+ assert not racy, f"makedirs without exist_ok = True at tokenizer_utils.py lines {racy}"
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 1ea3406c42c..26c564dc1d1 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -490,8 +490,8 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_sen
return slow_tokenizer
name = slow_tokenizer.name_or_path.replace("/", "_")
- if not os.path.exists(temporary_location):
- os.makedirs(temporary_location)
+ # exist_ok: a check-then-create races when two processes share a working directory.
+ os.makedirs(temporary_location, exist_ok = True)
new_location = f"{temporary_location}/{name}"
slow_tokenizer.save_pretrained(new_location)
fast_tokenizer.save_pretrained(new_location)
@@ -655,8 +655,8 @@ def fix_sentencepiece_tokenizer(
# This will only work for older SentencePiece versions <= 3.20.3
from transformers.utils import sentencepiece_model_pb2
- if not os.path.exists(temporary_location):
- os.makedirs(temporary_location)
+ # exist_ok: a check-then-create races when two processes share a working directory.
+ os.makedirs(temporary_location, exist_ok = True)
# Fresh per-call subdir so concurrent or repeated calls cannot clobber each other's tokenizer.model or leak stale files, without deleting anything the caller owns.
temporary_location = tempfile.mkdtemp(prefix = "tokenizer_", dir = temporary_location)