Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Repair the checks red on main after today's Mistral-format, GKD, laya, llm-compressor and HF token merges (#12218)

Daniel Haneb91752d4b80
diff --git a/studio/backend/core/systemone/laya_runtime.py b/studio/backend/core/systemone/laya_runtime.py
index 493dadb289d..f9136357f81 100644
--- a/studio/backend/core/systemone/laya_runtime.py
+++ b/studio/backend/core/systemone/laya_runtime.py
@@ -151,9 +151,44 @@ def _laya():
for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
del sys.modules[name]
raise
+ for name in [n for n in sys.modules if n == "laya" or n.startswith("laya.")]:
+ sys.modules[name].open = _utf8_open
return module
+def _utf8_open(
+ file,
+ mode = "r",
+ buffering = -1,
+ encoding = None,
+ errors = None,
+ newline = None,
+ closefd = True,
+ opener = None,
+):
+ """``open`` for the vendored laya modules: text mode defaults to UTF-8.
+
+ laya reads ``rl_agent_config.json`` and ``tokenizer_config.json`` with a bare ``open()``,
+ which decodes with the locale's code page (ANSI on Windows, ASCII under a C locale). A
+ checkpoint whose tokenizer config holds non-ASCII special tokens then fails to read, and
+ ``_fix_tokenizer_config`` swallows that and skips the repair the model needs to load.
+ The vendored files stay byte-identical to the wheel (vendor/README.md), so the encoding is
+ supplied here, as each laya module's own ``open``.
+ """
+ if encoding is None and "b" not in mode:
+ encoding = "utf-8"
+ return open(
+ file,
+ mode,
+ buffering,
+ encoding = encoding,
+ errors = errors,
+ newline = newline,
+ closefd = closefd,
+ opener = opener,
+ )
+
+
def is_cached(checkpoint: Checkpoint) -> bool:
try:
root = _checkpoint_dir(checkpoint, local_only = True)
diff --git a/studio/backend/tests/test_text_io_encoding.py b/studio/backend/tests/test_text_io_encoding.py
index 7f01e50953d..12bb7c189fa 100644
--- a/studio/backend/tests/test_text_io_encoding.py
+++ b/studio/backend/tests/test_text_io_encoding.py
@@ -12,6 +12,8 @@ model config or path containing ``ä ö ü → 世`` mojibakes or raises
from __future__ import annotations
+import sys
+import subprocess
import ast
import importlib.util
import json
@@ -34,6 +36,11 @@ BACKEND_ROOT = Path(__file__).resolve().parent.parent
# Not runtime source. Shipped plugins under plugins/*/src are, so only builds are skipped.
_SKIPPED_DIRS = ("node_modules", "build", "tests", "__pycache__")
+# Vendored packages kept byte-identical to their wheel and pinned by per-file hashes
+# (vendor/README.md), mapped to the loader that gives their modules a UTF-8 `open`. They are
+# fixed there, not in place; test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale checks it.
+_UTF8_BY_LOADER = {"vendor/laya": "core/systemone/laya_runtime.py"}
+
# Path.open()'s signature is what tells it apart from other libraries' open(),
# e.g. fitz.open(stream=...) and av.open(..., metadata_errors=...).
_FILE_MODE_CHARS = set("rwxabt+")
@@ -53,6 +60,10 @@ def _studio_sources() -> list[Path]:
path
for path in sorted(BACKEND_ROOT.rglob("*.py"))
if not any(part in _SKIPPED_DIRS for part in path.relative_to(BACKEND_ROOT).parts)
+ and not any(
+ path.relative_to(BACKEND_ROOT).as_posix().startswith(prefix + "/")
+ for prefix in _UTF8_BY_LOADER
+ )
]
@@ -823,3 +834,40 @@ def test_an_unparseably_nested_document_is_discarded_not_raised(tmp_path: Path)
assert writer.has("id:1") and writer.has("id:2")
finally:
writer.close()
+
+
+def test_vendored_laya_reads_utf8_config_under_a_non_utf8_locale(tmp_path):
+ """The loader in _UTF8_BY_LOADER really makes vendored laya decode its JSON as UTF-8.
+
+ Run in a C locale with coercion and UTF-8 mode off, so a bare open() does not decode as
+ UTF-8. laya's tokenizer repair must still read a config holding non-ASCII special tokens
+ and write them back unchanged.
+ """
+ config = tmp_path / "tokenizer" / "tokenizer_config.json"
+ config.parent.mkdir()
+ config.write_text(
+ json.dumps(
+ {"tokenizer_class": None, "extra_special_tokens": ["ä", "世"]}, ensure_ascii = False
+ ),
+ encoding = "utf-8",
+ )
+ # C locale with coercion and UTF-8 mode off decodes a bare open() as ASCII on Linux and
+ # macOS; on Windows it stays the ANSI code page, which mis-decodes the same bytes instead of
+ # raising, and the exact-token assertions below catch that. A UTF-8 locale has nothing to test.
+ script = (
+ "import codecs, locale, sys\n"
+ "if codecs.lookup(locale.getencoding()).name == 'utf-8': sys.exit(3)\n"
+ f"sys.path.insert(0, {str(BACKEND_ROOT)!r})\n"
+ "from core.systemone import laya_runtime\n"
+ f"laya_runtime._laya().agent._fix_tokenizer_config({str(tmp_path)!r})\n"
+ )
+ env = {**os.environ, "LC_ALL": "C", "LANG": "C", "PYTHONCOERCECLOCALE": "0", "PYTHONUTF8": "0"}
+ run = subprocess.run(
+ [sys.executable, "-c", script], env = env, capture_output = True, text = True, encoding = "utf-8"
+ )
+ if run.returncode == 3:
+ pytest.skip("this platform's locale is UTF-8 even under LC_ALL=C")
+ assert run.returncode == 0, run.stderr[-3000:]
+ repaired = json.loads(config.read_text(encoding = "utf-8"))
+ assert repaired["tokenizer_class"] == "PreTrainedTokenizerFast"
+ assert repaired["extra_special_tokens"] == {"extra_0": "ä", "extra_1": "世"}
diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.py
index 7060f406125..5cc26b32c4c 100644
--- a/tests/python/test_gkd_chunked_jsd.py
+++ b/tests/python/test_gkd_chunked_jsd.py
@@ -11,6 +11,7 @@ import textwrap
import types
import pytest
+from real_accelerator import has_real_cuda
import torch
os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
@@ -482,7 +483,7 @@ def test_dense_fallback_projects_on_the_head_device():
@pytest.mark.skipif(
- not torch.cuda.is_available(),
+ not has_real_cuda(),
reason = "needs a second device to split hidden states from the head",
)
def test_chunked_loss_with_heads_on_another_device():
@@ -539,7 +540,7 @@ def test_minicpm3_scales_hidden_states_before_the_head_so_falls_back():
@pytest.mark.skipif(
- not torch.cuda.is_available(),
+ not has_real_cuda(),
reason = "needs a second device to split hidden states from the head",
)
def test_dense_fallback_colocates_with_the_labels():
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index 6669e4d1b52..e8879cc4e3f 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -54,7 +54,9 @@ def _function_source(name: str) -> str:
def _load(*names, **env):
"""Exec the named top-level functions against `env` and return the namespace."""
- namespace = dict(env)
+ # save.py imports this from models.mistral_format (#12144) and calls it on every merge and
+ # GGUF path; none of these fixtures is a Mistral-format view, so it never refuses here.
+ namespace = {"raise_if_merging_mistral_format_view": lambda model, save_method: None, **env}
for name in names:
exec(compile(_function_source(name), str(_SAVE_PY), "exec"), namespace)
return namespace
diff --git a/tests/studio/test_hf_token_validation_tick_contract.py b/tests/studio/test_hf_token_validation_tick_contract.py
index 022de483c48..d5cd31a9835 100644
--- a/tests/studio/test_hf_token_validation_tick_contract.py
+++ b/tests/studio/test_hf_token_validation_tick_contract.py
@@ -1,11 +1,13 @@
"""Contracts for the Hugging Face token validation indicator."""
+import re
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
GENERAL_TAB = REPO / "studio/frontend/src/features/settings/tabs/general-tab.tsx"
VALIDATION_HOOK = REPO / "studio/frontend/src/hooks/use-hf-token-validation.ts"
+HF_TOKEN_SHAPE = REPO / "studio/frontend/src/lib/hf-token-shape.ts"
TOKEN_INDICATOR = REPO / "studio/frontend/src/features/hub/components/hf-token-indicator.tsx"
EN_LOCALE = REPO / "studio/frontend/src/i18n/locales/en.ts"
RUN_PREVIEW = REPO / "studio/frontend/src/features/studio/wizard/run-preview-card.tsx"
@@ -52,7 +54,18 @@ def test_validation_result_must_belong_to_the_current_normalized_token():
assert "useDebouncedValue(normalizedToken, 500)" in source
assert 'normalizedToken && !normalizedToken.startsWith("hf_")' in source
assert 'error: "Token must start with hf_."' in source
- assert "if (!COMPLETE_HF_TOKEN.test(normalizedToken)) return INITIAL" in source
+ # #12159 moved the shape check into lib/hf-token-shape.ts and widened it to OAuth tokens.
+ assert "if (!isCompleteHfTokenShape(normalizedToken)) return INITIAL" in source
+ shape = re.search(
+ r"const COMPLETE_HF_TOKEN = /(.+)/;", HF_TOKEN_SHAPE.read_text(encoding = "utf-8")
+ )
+ assert shape is not None
+ complete = re.compile(shape.group(1))
+ assert complete.fullmatch("hf_" + "a" * 34)
+ assert complete.fullmatch("hf_oauth_" + "a" * 30)
+ assert not complete.fullmatch("hf_oauth_" + "a" * 29)
+ assert not complete.fullmatch("hf_" + "a" * 33)
+ assert not complete.fullmatch("hf_" + "a" * 34 + "-")
assert "if (completed.token !== normalizedToken)" in source
assert "if (completed.token !== debouncedToken)" not in source
diff --git a/tests/test_merged_hub_destination.py b/tests/test_merged_hub_destination.py
index d2685d87937..1c9633a51ef 100644
--- a/tests/test_merged_hub_destination.py
+++ b/tests/test_merged_hub_destination.py
@@ -165,6 +165,8 @@ def saving(monkeypatch, tmp_path):
_normalize_compressed_method = lambda method: None,
_normalize_torchao_method = lambda method: None,
_is_qwen3_5_vlm = lambda model: False,
+ # A Mistral-format view (#12144) refuses merged saves; these fixtures are not one.
+ raise_if_merging_mistral_format_view = lambda model, save_method: None,
logger = SimpleNamespace(warning_once = lambda *args: None),
# save_method="lora" leaves this module for the adapter save rather than the merge,
# so record the handover instead of re-implementing it.
diff --git a/tests/test_mistral_format_mistral4.py b/tests/test_mistral_format_mistral4.py
index 8b715a4d07b..b248c4b51b8 100644
--- a/tests/test_mistral_format_mistral4.py
+++ b/tests/test_mistral_format_mistral4.py
@@ -10,6 +10,7 @@ import os
import re
import pytest
+from real_accelerator import has_real_cuda
torch = pytest.importorskip("torch")
transformers = pytest.importorskip("transformers")
@@ -485,7 +486,7 @@ def _fp8_blocks(weight):
return q.reshape(rows, cols).contiguous(), scale.contiguous(), dequant
[email protected](not torch.cuda.is_available(), reason = "FP8 loading needs a GPU")
[email protected](not has_real_cuda(), reason = "FP8 loading needs a GPU")
def test_fp8_checkpoint_dequantizes_exactly_and_loads_natively(tmp_path, hub_cache):
from transformers import FineGrainedFP8Config
@@ -641,7 +642,7 @@ with torch.no_grad():
"""
[email protected](not torch.cuda.is_available(), reason = "FastModel needs a GPU")
[email protected](not has_real_cuda(), reason = "FastModel needs a GPU")
def test_lora_adapter_trained_on_a_view_reloads_in_a_fresh_process(tmp_path):
# The adapter's base is the view, whose index still names Mistral's tensors.
import subprocess
diff --git a/unsloth/save.py b/unsloth/save.py
index 31eb7b12cde..04adfe65de5 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -1614,6 +1614,7 @@ def _llm_compressor_imports_in_subprocess():
stdout = subprocess.PIPE,
stderr = subprocess.DEVNULL,
text = True,
+ encoding = "utf-8",
timeout = 600,
)
if completed.returncode != 0: