Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Scope UNSLOTH_HIGH_PRECISION_LAYERNORM to the load that sets it (#12873)
Daniel Han899d7ae2db67
diff --git a/tests/test_layernorm_precision_env_scope.py b/tests/test_layernorm_precision_env_scope.pynew file mode 100644index 00000000000..a7408a35f74--- /dev/null+++ b/tests/test_layernorm_precision_env_scope.py@@ -0,0 +1,171 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""UNSLOTH_HIGH_PRECISION_LAYERNORM is per load: a leak gave the next family float32 norms+("float != BFloat16"), and a cached recompile must replay the family's own decision."""++import os+import subprocess+import sys+import textwrap++import pytest++ENV = "UNSLOTH_HIGH_PRECISION_LAYERNORM"+++def clean_env(monkeypatch):+ monkeypatch.delenv(ENV, raising = False)+++def test_load_scope_restores_unset_value(clean_env):+ from unsloth.models.loader_utils import _restore_load_scoped_env++ @_restore_load_scoped_env+ def load():+ os.environ[ENV] = "1"+ return "model"++ assert load() == "model"+ assert ENV not in os.environ+++def test_load_scope_restores_user_value_on_error(monkeypatch):+ from unsloth.models.loader_utils import _restore_load_scoped_env++ monkeypatch.setenv(ENV, "1")++ @_restore_load_scoped_env+ def load():+ os.environ[ENV] = "0"+ raise ValueError("boom")++ with pytest.raises(ValueError):+ load()+ assert os.environ[ENV] == "1"+++def test_fast_model_from_pretrained_is_load_scoped():+ from unsloth.models.loader import FastModel++ fn, names = FastModel.from_pretrained, []+ while fn is not None:+ names.append(getattr(fn.__code__, "co_qualname", fn.__code__.co_name))+ fn = getattr(fn, "__wrapped__", None)+ assert "_restore_load_scoped_env.<locals>._wrapper" in names, names+++def _fake_compiler(detects):+ """Mimic zoo: check norms on a type's first compile only, OR-ing into the env."""+ patched = set()++ def compile_one(model_type, **kwargs):+ if model_type in patched:+ return+ patched.add(model_type)+ high = os.environ.get(ENV, "0") == "1" or model_type in detects+ os.environ[ENV] = "1" if high else "0"++ return compile_one+++def compile_fn(monkeypatch, clean_env):+ import unsloth.models._utils as U++ monkeypatch.setattr(U, "_HIGH_PRECISION_LAYERNORM_MODEL_TYPES", set())+ monkeypatch.setattr(U, "_run_temporary_patches", lambda *a, **k: None)+ monkeypatch.setattr(U, "_unsloth_compile_transformers", _fake_compiler({"qwen3_5"}))++ def run(model_types):+ return U.unsloth_compile_transformers(None, "m", list(model_types))++ return run+++def test_cached_compile_replays_high_precision(compile_fn):+ compile_fn(["siglip", "qwen3_5", "qwen3_5_text"])+ assert os.environ[ENV] == "1"+ os.environ.pop(ENV) # what the load scope does after the first load+ compile_fn(["siglip", "qwen3_5", "qwen3_5_text"]) # compiler returns early now+ assert os.environ[ENV] == "1"+++def test_other_family_is_not_upcast(compile_fn):+ compile_fn(["siglip", "qwen3_5"])+ os.environ.pop(ENV)+ compile_fn(["siglip", "qwen3"])+ assert os.environ.get(ENV, "0") == "0"+++def test_inherited_value_is_kept_but_not_attributed(compile_fn):+ import unsloth.models._utils as U++ os.environ[ENV] = "1" # e.g. the loader's Gemma 3 branch+ compile_fn(["siglip", "gemma3"])+ assert os.environ[ENV] == "1"+ assert U._HIGH_PRECISION_LAYERNORM_MODEL_TYPES == set()+++_SEQUENCE = textwrap.dedent(+ """+ import os, sys, json, torch+ from unsloth import FastModel+ for name in sys.argv[1:]:+ m, _ = FastModel.from_pretrained(name, max_seq_length = 256, load_in_4bit = False)+ norms = sorted({str(p.dtype) for n, p in m.named_parameters() if "norm" in n.split(".")[-2].lower()})+ torch.manual_seed(0)+ with torch.no_grad():+ m(input_ids = torch.randint(0, 100, (1, 8), device = m.device))+ print("RESULT", json.dumps({"model": name, "norms": norms}), flush = True)+ del m+ """+)++Q35 = "trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration"+Q3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"+G3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"+LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"+++def _run_sequence(tmp_path, *models):+ env = dict(os.environ, UNSLOTH_COMPILE_LOCATION = str(tmp_path / "cache"))+ env.pop(ENV, None)+ proc = subprocess.run(+ [sys.executable, "-c", _SEQUENCE, *models],+ env = env,+ capture_output = True,+ text = True,+ timeout = 1800,+ )+ assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:]+ import json++ return [+ json.loads(l[len("RESULT ") :]) for l in proc.stdout.splitlines() if l.startswith("RESULT ")+ ]+++cuda = pytest.mark.skipif(+ not (os.environ.get("UNSLOTH_RUN_GPU_TESTS") == "1"),+ reason = "loads tiny checkpoints on a GPU; set UNSLOTH_RUN_GPU_TESTS=1",+)+++@cuda+ "first, second, first_fp32, second_fp32",+ [+ (Q35, Q3, True, False),+ (Q35, Q35, True, True),+ (G3, LLAMA, True, False),+ (G3, Q3, True, False),+ ],+)+def test_load_sequence_keeps_each_models_norm_dtype(+ tmp_path, first, second, first_fp32, second_fp32+):+ a, b = _run_sequence(tmp_path, first, second)+ assert ("torch.float32" in a["norms"]) == first_fp32, a+ assert ("torch.float32" in b["norms"]) == second_fp32, bdiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 5930706ea8c..4db94fc7342 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -5657,6 +5657,29 @@ def patch_fast_lora():peft.tuners.lora.bnb.Linear4bit.forward = fast_lora_forward+# Model types whose norms the compiler's check upcast to float32. The check runs only on a modeling file's first compile, so a later load of the same family replays it from here.+_HIGH_PRECISION_LAYERNORM_MODEL_TYPES = set()+++def _start_layernorm_check():+ # The compiler ORs its check into the inherited value, so clear it to see this type's own answer.+ prior = os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM")+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "0"+ return prior+++def _finish_layernorm_check(model_type, prior):+ detected = os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM", "0") == "1"+ if detected:+ _HIGH_PRECISION_LAYERNORM_MODEL_TYPES.add(model_type)+ if detected or prior == "1" or model_type in _HIGH_PRECISION_LAYERNORM_MODEL_TYPES:+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"+ elif prior is None:+ os.environ.pop("UNSLOTH_HIGH_PRECISION_LAYERNORM", None)+ else:+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = prior++def unsloth_compile_transformers(dtype,model_name,@@ -5719,6 +5742,7 @@ def unsloth_compile_transformers(_run_temporary_patches("pre_compile")for model_type in model_types:+ prior_high_precision = _start_layernorm_check()_unsloth_compile_transformers(model_type,sdpa_dynamic_mask = sdpa_dynamic_mask,@@ -5747,6 +5771,7 @@ def unsloth_compile_transformers(return_logits = return_logits,supports_sdpa = supports_sdpa,)+ _finish_layernorm_check(model_type, prior_high_precision)_run_temporary_patches("post_compile")return model_types, supports_sdpa[0]diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex a2c6301cb30..ef402a8f82b 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -56,6 +56,7 @@ from .loader_utils import (prepare_device_map,requested_device_map,_offline_aware_load,+ _restore_load_scoped_env,_resolve_checkpoint_tokenizer_name,_is_offline_related_error,)@@ -1720,6 +1721,7 @@ class FastModel(FastBaseModel):return FastBaseModel.for_training(model, use_gradient_checkpointing)@staticmethod+ @_restore_load_scoped_env@_offline_aware_load@mistral_format_redirect@track_explicit_4bit_requestdiff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex ad9119cc2b8..49f189317d0 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -3648,6 +3648,26 @@ def _note_offline_retry(error, retry_error):pass+# Set per load (family branches in FastModel.from_pretrained, the compiler's norm check) and read only during it. Left set, a later load of another family gets float32 norms beside 16 bit projections and fails with "float != BFloat16".+LOAD_SCOPED_ENV_VARS = ("UNSLOTH_HIGH_PRECISION_LAYERNORM",)+++def _restore_load_scoped_env(fn):+ @functools.wraps(fn)+ def _wrapper(*args, **kwargs):+ saved = {name: os.environ.get(name) for name in LOAD_SCOPED_ENV_VARS}+ try:+ return fn(*args, **kwargs)+ finally:+ for name, value in saved.items():+ if value is None:+ os.environ.pop(name, None)+ else:+ os.environ[name] = value++ return _wrapper++def _offline_aware_load(fn):"""Decide offline ONCE (local_files_only kwarg or env) and force it around the whole load. If we started online and hit a network error, retry once forced-offline. The network-up online path is unchanged: no window, no retry."""