Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Repair the checks red on main after today's Mistral-format, GKD, laya, llm-compressor and HF token merges (#12218)
Daniel Haneb91752d4b80
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.pyindex 493dadb289d..f9136357f81 100644--- a/studio/backend/core/systemone/laya_runtime.py+++ b/studio/backend/core/systemone/laya_runtime.py@@ -151,9 +151,44 @@ def _laya():for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:del sys.modules[name]raise+ for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:+ sys.modules[name].open = _utf8_openreturn module+def _utf8_open(+ file,+ mode = "r",+ buffering = -1,+ encoding = None,+ errors = None,+ newline = None,+ closefd = True,+ opener = None,+):+ """``open`` for the vendored laya modules: text mode defaults to UTF-8.++ laya reads ``rl_agent_config.json`` and ``tokenizer_config.json`` with a bare ``open()``,+ which decodes with the locale's code page (ANSI on Windows, ASCII under a C locale). A+ checkpoint whose tokenizer config holds non-ASCII special tokens then fails to read, and+ ``_fix_tokenizer_config`` swallows that and skips the repair the model needs to load.+ The vendored files stay byte-identical to the wheel (vendor/README.md), so the encoding is+ supplied here, as each laya module's own ``open``.+ """+ if encoding is None and "b" not in mode:+ encoding = "utf-8"+ return open(+ file,+ mode,+ buffering,+ encoding = encoding,+ errors = errors,+ newline = newline,+ closefd = closefd,+ opener = opener,+ )++def is_cached(checkpoint: Checkpoint) -> bool:try:root = _checkpoint_dir(checkpoint, local_only = True)diff --git a/studio/backend/tests/test_text_io_encoding.py b/studio/backend/tests/test_text_io_encoding.pyindex 7f01e50953d..12bb7c189fa 100644--- a/studio/backend/tests/test_text_io_encoding.py+++ b/studio/backend/tests/test_text_io_encoding.py@@ -12,6 +12,8 @@ model config or path containing ``ä ö ü → 世`` mojibakes or raisesfrom __future__ import annotations+import sys+import subprocessimport astimport importlib.utilimport json@@ -34,6 +36,11 @@ BACKEND_ROOT = Path(__file__).resolve().parent.parent# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped._SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")+# Vendored packages kept byte-identical to their wheel and pinned by per-file hashes+# (vendor/README.md), mapped to the loader that gives their modules a UTF-8 `open`. They are+# fixed there, not in place; test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale checks it.+_UTF8_BY_LOADER = {"vendor/laya": "core/systemone/laya_runtime.py"}+# Path.open()'s signature is what tells it apart from other libraries' open(),# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...)._FILE_MODE_CHARS = set("rwxabt+")@@ -53,6 +60,10 @@ def _studio_sources() -> list[Path]:pathfor path in sorted(BACKEND_ROOT.rglob("*.py"))if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)+ and not any(+ path.relative_to(BACKEND_ROOT).as_posix().startswith(prefix + "/")+ for prefix in _UTF8_BY_LOADER+ )]@@ -823,3 +834,40 @@ def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path)assert writer.has("id:1") and writer.has("id:2")finally:writer.close()+++def test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale(tmp_path):+ """The loader in _UTF8_BY_LOADER really makes vendored laya decode its JSON as UTF-8.++ Run in a C locale with coercion and UTF-8 mode off, so a bare open() does not decode as+ UTF-8. laya's tokenizer repair must still read a config holding non-ASCII special tokens+ and write them back unchanged.+ """+ config = tmp_path / "tokenizer" / "tokenizer_config.json"+ config.parent.mkdir()+ config.write_text(+ json.dumps(+ {"tokenizer_class": None, "extra_special_tokens": ["ä", "世"]}, ensure_ascii = False+ ),+ encoding = "utf-8",+ )+ # C locale with coercion and UTF-8 mode off decodes a bare open() as ASCII on Linux and+ # macOS; on Windows it stays the ANSI code page, which mis-decodes the same bytes instead of+ # raising, and the exact-token assertions below catch that. A UTF-8 locale has nothing to test.+ script = (+ "import codecs, locale, sys\n"+ "if codecs.lookup(locale.getencoding()).name == 'utf-8': sys.exit(3)\n"+ f"sys.path.insert(0, {str(BACKEND_ROOT)!r})\n"+ "from core.systemone import laya_runtime\n"+ f"laya_runtime._laya().agent._fix_tokenizer_config({str(tmp_path)!r})\n"+ )+ env = {**os.environ, "LC_ALL": "C", "LANG": "C", "PYTHONCOERCECLOCALE": "0", "PYTHONUTF8": "0"}+ run = subprocess.run(+ [sys.executable, "-c", script], env = env, capture_output = True, text = True, encoding = "utf-8"+ )+ if run.returncode == 3:+ pytest.skip("this platform's locale is UTF-8 even under LC_ALL=C")+ assert run.returncode == 0, run.stderr[-3000:]+ repaired = json.loads(config.read_text(encoding = "utf-8"))+ assert repaired["tokenizer_class"] == "PreTrainedTokenizerFast"+ assert repaired["extra_special_tokens"] == {"extra_0": "ä", "extra_1": "世"}diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.pyindex 7060f406125..5cc26b32c4c 100644--- a/tests/python/test_gkd_chunked_jsd.py+++ b/tests/python/test_gkd_chunked_jsd.py@@ -11,6 +11,7 @@ import textwrapimport typesimport pytest+from real_accelerator import has_real_cudaimport torchos.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")@@ -482,7 +483,7 @@ def test_dense_fallback_projects_on_the_head_device():@pytest.mark.skipif(- not torch.cuda.is_available(),+ not has_real_cuda(),reason = "needs a second device to split hidden states from the head",)def test_chunked_loss_with_heads_on_another_device():@@ -539,7 +540,7 @@ def test_minicpm3_scales_hidden_states_before_the_head_so_falls_back():@pytest.mark.skipif(- not torch.cuda.is_available(),+ not has_real_cuda(),reason = "needs a second device to split hidden states from the head",)def test_dense_fallback_colocates_with_the_labels():diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex 6669e4d1b52..e8879cc4e3f 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -54,7 +54,9 @@ def _function_source(name: str) -> str:def _load(*names, **env):"""Exec the named top-level functions against `env` and return the namespace."""- namespace = dict(env)+ # save.py imports this from models.mistral_format (#12144) and calls it on every merge and+ # GGUF path; none of these fixtures is a Mistral-format view, so it never refuses here.+ namespace = {"raise_if_merging_mistral_format_view": lambda model, save_method: None, **env}for name in names:exec(compile(_function_source(name), str(_SAVE_PY), "exec"), namespace)return namespacediff --git a/tests/studio/test_hf_token_validation_tick_contract.py b/tests/studio/test_hf_token_validation_tick_contract.pyindex 022de483c48..d5cd31a9835 100644--- a/tests/studio/test_hf_token_validation_tick_contract.py+++ b/tests/studio/test_hf_token_validation_tick_contract.py@@ -1,11 +1,13 @@"""Contracts for the Hugging Face token validation indicator."""+import refrom pathlib import PathREPO = Path(__file__).resolve().parents[2]GENERAL_TAB = REPO / "studio/frontend/src/features/settings/tabs/general-tab.tsx"VALIDATION_HOOK = REPO / "studio/frontend/src/hooks/use-hf-token-validation.ts"+HF_TOKEN_SHAPE = REPO / "studio/frontend/src/lib/hf-token-shape.ts"TOKEN_INDICATOR = REPO / "studio/frontend/src/features/hub/components/hf-token-indicator.tsx"EN_LOCALE = REPO / "studio/frontend/src/i18n/locales/en.ts"RUN_PREVIEW = REPO / "studio/frontend/src/features/studio/wizard/run-preview-card.tsx"@@ -52,7 +54,18 @@ def test_validation_result_must_belong_to_the_current_normalized_token():assert "useDebouncedValue(normalizedToken, 500)" in sourceassert 'normalizedToken && !normalizedToken.startsWith("hf_")' in sourceassert 'error: "Token must start with hf_."' in source- assert "if (!COMPLETE_HF_TOKEN.test(normalizedToken)) return INITIAL" in source+ # #12159 moved the shape check into lib/hf-token-shape.ts and widened it to OAuth tokens.+ assert "if (!isCompleteHfTokenShape(normalizedToken)) return INITIAL" in source+ shape = re.search(+ r"const COMPLETE_HF_TOKEN = /(.+)/;", HF_TOKEN_SHAPE.read_text(encoding = "utf-8")+ )+ assert shape is not None+ complete = re.compile(shape.group(1))+ assert complete.fullmatch("hf_" + "a" * 34)+ assert complete.fullmatch("hf_oauth_" + "a" * 30)+ assert not complete.fullmatch("hf_oauth_" + "a" * 29)+ assert not complete.fullmatch("hf_" + "a" * 33)+ assert not complete.fullmatch("hf_" + "a" * 34 + "-")assert "if (completed.token !== normalizedToken)" in sourceassert "if (completed.token !== debouncedToken)" not in sourcediff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.pyindex d2685d87937..1c9633a51ef 100644--- a/tests/test_merged_hub_destination.py+++ b/tests/test_merged_hub_destination.py@@ -165,6 +165,8 @@ def saving(monkeypatch, tmp_path):_normalize_compressed_method = lambda method: None,_normalize_torchao_method = lambda method: None,_is_qwen3_5_vlm = lambda model: False,+ # A Mistral-format view (#12144) refuses merged saves; these fixtures are not one.+ raise_if_merging_mistral_format_view = lambda model, save_method: None,logger = SimpleNamespace(warning_once = lambda *args: None),# save_method="lora" leaves this module for the adapter save rather than the merge,# so record the handover instead of re-implementing it.diff --git a/tests/test_mistral_format_mistral4.py b/tests/test_mistral_format_mistral4.pyindex 8b715a4d07b..b248c4b51b8 100644--- a/tests/test_mistral_format_mistral4.py+++ b/tests/test_mistral_format_mistral4.py@@ -10,6 +10,7 @@ import osimport reimport pytest+from real_accelerator import has_real_cudatorch = pytest.importorskip("torch")transformers = pytest.importorskip("transformers")@@ -485,7 +486,7 @@ def _fp8_blocks(weight):return q.reshape(rows, cols).contiguous(), scale.contiguous(), dequant[email protected](not torch.cuda.is_available(), reason = "FP8 loading needs a GPU")[email protected](not has_real_cuda(), reason = "FP8 loading needs a GPU")def test_fp8_checkpoint_dequantizes_exactly_and_loads_natively(tmp_path, hub_cache):from transformers import FineGrainedFP8Config@@ -641,7 +642,7 @@ with torch.no_grad():"""[email protected](not torch.cuda.is_available(), reason = "FastModel needs a GPU")[email protected](not has_real_cuda(), reason = "FastModel needs a GPU")def test_lora_adapter_trained_on_a_view_reloads_in_a_fresh_process(tmp_path):# The adapter's base is the view, whose index still names Mistral's tensors.import subprocessdiff --git a/unsloth/save.py b/unsloth/save.pyindex 31eb7b12cde..04adfe65de5 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -1614,6 +1614,7 @@ def _llm_compressor_imports_in_subprocess():stdout = subprocess.PIPE,stderr = subprocess.DEVNULL,text = True,+ encoding = "utf-8",timeout = 600,)if completed.returncode != 0: