Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Create the SentencePiece scratch directory without a check-then-create race (#12400)
Daniel Han8568ae303830
diff --git a/tests/python/test_get_chat_template_sharegpt_mapping.py b/tests/python/test_get_chat_template_sharegpt_mapping.pyindex e3f85a5abe6..bc37b865d72 100644--- a/tests/python/test_get_chat_template_sharegpt_mapping.py+++ b/tests/python/test_get_chat_template_sharegpt_mapping.py@@ -30,6 +30,13 @@ TEMPLATES = [][email protected](autouse = True)+def _scratch_cwd(tmp_path, monkeypatch):+ # get_chat_template's SentencePiece fix writes a scratch dir under the working directory;+ # keep each test's copy out of the checkout and away from other xdist workers.+ monkeypatch.chdir(tmp_path)++def _tokenizer():vocab = {"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}return PreTrainedTokenizerFast(diff --git a/tests/saving/test_fix_sentencepiece_tokenizer_guard.py b/tests/saving/test_fix_sentencepiece_tokenizer_guard.pyindex 0bbfeb468a8..27e69ab6cdc 100644--- a/tests/saving/test_fix_sentencepiece_tokenizer_guard.py+++ b/tests/saving/test_fix_sentencepiece_tokenizer_guard.py@@ -310,3 +310,52 @@ def test_only_applied_mappings_are_patched(tmp_path, monkeypatch):result = _read_pieces(f"{loaded[-1]}/tokenizer.model")assert result.count("X") == 1 and "Y" in result and "aa" in result, resultassert tok is not None+++def test_a_scratch_directory_created_by_another_process_is_reused(tmp_path, monkeypatch):+ """Two processes sharing a working directory both see the scratch dir missing; the+ second makedirs must not raise. Simulated by reporting the directory absent after+ another process has already created it."""+ import unsloth.tokenizer_utils as tokenizer_utils++ _stub_auto_tokenizer(monkeypatch)+ old, new = _tokenizers()+ location = str(tmp_path / "_unsloth_sentencepiece_temp")+ os.makedirs(location)+ real_exists = os.path.exists+ monkeypatch.setattr(+ tokenizer_utils.os.path,+ "exists",+ lambda path: False if os.fspath(path) == location else real_exists(path),+ )++ fix_sentencepiece_tokenizer(old, new, {"</s>": "<|im_end|>"}, temporary_location = location)++ assert old.saved_to+++def test_every_scratch_makedirs_tolerates_an_existing_directory():+ """convert_to_fast_tokenizer shares the pattern but needs a real slow tokenizer to reach+ it, so pin the call shape for every makedirs in the module instead."""+ import ast+ import inspect++ import unsloth.tokenizer_utils as tokenizer_utils++ calls = [+ node+ for node in ast.walk(ast.parse(inspect.getsource(tokenizer_utils)))+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "makedirs"+ ]+ assert calls, "no makedirs call found; update this test"+ racy = [+ node.lineno+ for node in calls+ if not any(+ kw.arg == "exist_ok" and isinstance(kw.value, ast.Constant) and kw.value.value is True+ for kw in node.keywords+ )+ ]+ assert not racy, f"makedirs without exist_ok = True at tokenizer_utils.py lines {racy}"diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 1ea3406c42c..26c564dc1d1 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -490,8 +490,8 @@ def convert_to_fast_tokenizer(slow_tokenizer, temporary_location = "_unsloth_senreturn slow_tokenizername = slow_tokenizer.name_or_path.replace("/", "_")- if not os.path.exists(temporary_location):- os.makedirs(temporary_location)+ # exist_ok: a check-then-create races when two processes share a working directory.+ os.makedirs(temporary_location, exist_ok = True)new_location = f"{temporary_location}/{name}"slow_tokenizer.save_pretrained(new_location)fast_tokenizer.save_pretrained(new_location)@@ -655,8 +655,8 @@ def fix_sentencepiece_tokenizer(# This will only work for older SentencePiece versions <= 3.20.3from transformers.utils import sentencepiece_model_pb2- if not os.path.exists(temporary_location):- os.makedirs(temporary_location)+ # exist_ok: a check-then-create races when two processes share a working directory.+ os.makedirs(temporary_location, exist_ok = True)# Fresh per-call subdir so concurrent or repeated calls cannot clobber each other's tokenizer.model or leak stale files, without deleting anything the caller owns.temporary_location = tempfile.mkdtemp(prefix = "tokenizer_", dir = temporary_location)