Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Scope UNSLOTH_HIGH_PRECISION_LAYERNORM to the load that sets it (#12873)

Daniel Han899d7ae2db67
diff --git a/tests/test_layernorm_precision_env_scope.py b/tests/test_layernorm_precision_env_scope.py
new file mode 100644
index 00000000000..a7408a35f74
--- /dev/null
+++ b/tests/test_layernorm_precision_env_scope.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""UNSLOTH_HIGH_PRECISION_LAYERNORM is per load: a leak gave the next family float32 norms
+("float != BFloat16"), and a cached recompile must replay the family's own decision."""
+
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+ENV = "UNSLOTH_HIGH_PRECISION_LAYERNORM"
+
+
+def clean_env(monkeypatch):
+ monkeypatch.delenv(ENV, raising = False)
+
+
+def test_load_scope_restores_unset_value(clean_env):
+ from unsloth.models.loader_utils import _restore_load_scoped_env
+
+ @_restore_load_scoped_env
+ def load():
+ os.environ[ENV] = "1"
+ return "model"
+
+ assert load() == "model"
+ assert ENV not in os.environ
+
+
+def test_load_scope_restores_user_value_on_error(monkeypatch):
+ from unsloth.models.loader_utils import _restore_load_scoped_env
+
+ monkeypatch.setenv(ENV, "1")
+
+ @_restore_load_scoped_env
+ def load():
+ os.environ[ENV] = "0"
+ raise ValueError("boom")
+
+ with pytest.raises(ValueError):
+ load()
+ assert os.environ[ENV] == "1"
+
+
+def test_fast_model_from_pretrained_is_load_scoped():
+ from unsloth.models.loader import FastModel
+
+ fn, names = FastModel.from_pretrained, []
+ while fn is not None:
+ names.append(getattr(fn.__code__, "co_qualname", fn.__code__.co_name))
+ fn = getattr(fn, "__wrapped__", None)
+ assert "_restore_load_scoped_env.<locals>._wrapper" in names, names
+
+
+def _fake_compiler(detects):
+ """Mimic zoo: check norms on a type's first compile only, OR-ing into the env."""
+ patched = set()
+
+ def compile_one(model_type, **kwargs):
+ if model_type in patched:
+ return
+ patched.add(model_type)
+ high = os.environ.get(ENV, "0") == "1" or model_type in detects
+ os.environ[ENV] = "1" if high else "0"
+
+ return compile_one
+
+
+def compile_fn(monkeypatch, clean_env):
+ import unsloth.models._utils as U
+
+ monkeypatch.setattr(U, "_HIGH_PRECISION_LAYERNORM_MODEL_TYPES", set())
+ monkeypatch.setattr(U, "_run_temporary_patches", lambda *a, **k: None)
+ monkeypatch.setattr(U, "_unsloth_compile_transformers", _fake_compiler({"qwen3_5"}))
+
+ def run(model_types):
+ return U.unsloth_compile_transformers(None, "m", list(model_types))
+
+ return run
+
+
+def test_cached_compile_replays_high_precision(compile_fn):
+ compile_fn(["siglip", "qwen3_5", "qwen3_5_text"])
+ assert os.environ[ENV] == "1"
+ os.environ.pop(ENV) # what the load scope does after the first load
+ compile_fn(["siglip", "qwen3_5", "qwen3_5_text"]) # compiler returns early now
+ assert os.environ[ENV] == "1"
+
+
+def test_other_family_is_not_upcast(compile_fn):
+ compile_fn(["siglip", "qwen3_5"])
+ os.environ.pop(ENV)
+ compile_fn(["siglip", "qwen3"])
+ assert os.environ.get(ENV, "0") == "0"
+
+
+def test_inherited_value_is_kept_but_not_attributed(compile_fn):
+ import unsloth.models._utils as U
+
+ os.environ[ENV] = "1" # e.g. the loader's Gemma 3 branch
+ compile_fn(["siglip", "gemma3"])
+ assert os.environ[ENV] == "1"
+ assert U._HIGH_PRECISION_LAYERNORM_MODEL_TYPES == set()
+
+
+_SEQUENCE = textwrap.dedent(
+ """
+ import os, sys, json, torch
+ from unsloth import FastModel
+ for name in sys.argv[1:]:
+ m, _ = FastModel.from_pretrained(name, max_seq_length = 256, load_in_4bit = False)
+ norms = sorted({str(p.dtype) for n, p in m.named_parameters() if "norm" in n.split(".")[-2].lower()})
+ torch.manual_seed(0)
+ with torch.no_grad():
+ m(input_ids = torch.randint(0, 100, (1, 8), device = m.device))
+ print("RESULT", json.dumps({"model": name, "norms": norms}), flush = True)
+ del m
+ """
+)
+
+Q35 = "trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration"
+Q3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+G3 = "hf-internal-testing/tiny-random-Gemma3ForCausalLM"
+LLAMA = "hf-internal-testing/tiny-random-LlamaForCausalLM"
+
+
+def _run_sequence(tmp_path, *models):
+ env = dict(os.environ, UNSLOTH_COMPILE_LOCATION = str(tmp_path / "cache"))
+ env.pop(ENV, None)
+ proc = subprocess.run(
+ [sys.executable, "-c", _SEQUENCE, *models],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1800,
+ )
+ assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:]
+ import json
+
+ return [
+ json.loads(l[len("RESULT ") :]) for l in proc.stdout.splitlines() if l.startswith("RESULT ")
+ ]
+
+
+cuda = pytest.mark.skipif(
+ not (os.environ.get("UNSLOTH_RUN_GPU_TESTS") == "1"),
+ reason = "loads tiny checkpoints on a GPU; set UNSLOTH_RUN_GPU_TESTS=1",
+)
+
+
+@cuda
+ "first, second, first_fp32, second_fp32",
+ [
+ (Q35, Q3, True, False),
+ (Q35, Q35, True, True),
+ (G3, LLAMA, True, False),
+ (G3, Q3, True, False),
+ ],
+)
+def test_load_sequence_keeps_each_models_norm_dtype(
+ tmp_path, first, second, first_fp32, second_fp32
+):
+ a, b = _run_sequence(tmp_path, first, second)
+ assert ("torch.float32" in a["norms"]) == first_fp32, a
+ assert ("torch.float32" in b["norms"]) == second_fp32, b
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 5930706ea8c..4db94fc7342 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -5657,6 +5657,29 @@ def patch_fast_lora():
peft.tuners.lora.bnb.Linear4bit.forward = fast_lora_forward
+# Model types whose norms the compiler's check upcast to float32. The check runs only on a modeling file's first compile, so a later load of the same family replays it from here.
+_HIGH_PRECISION_LAYERNORM_MODEL_TYPES = set()
+
+
+def _start_layernorm_check():
+ # The compiler ORs its check into the inherited value, so clear it to see this type's own answer.
+ prior = os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM")
+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "0"
+ return prior
+
+
+def _finish_layernorm_check(model_type, prior):
+ detected = os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM", "0") == "1"
+ if detected:
+ _HIGH_PRECISION_LAYERNORM_MODEL_TYPES.add(model_type)
+ if detected or prior == "1" or model_type in _HIGH_PRECISION_LAYERNORM_MODEL_TYPES:
+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = "1"
+ elif prior is None:
+ os.environ.pop("UNSLOTH_HIGH_PRECISION_LAYERNORM", None)
+ else:
+ os.environ["UNSLOTH_HIGH_PRECISION_LAYERNORM"] = prior
+
+
def unsloth_compile_transformers(
dtype,
model_name,
@@ -5719,6 +5742,7 @@ def unsloth_compile_transformers(
_run_temporary_patches("pre_compile")
for model_type in model_types:
+ prior_high_precision = _start_layernorm_check()
_unsloth_compile_transformers(
model_type,
sdpa_dynamic_mask = sdpa_dynamic_mask,
@@ -5747,6 +5771,7 @@ def unsloth_compile_transformers(
return_logits = return_logits,
supports_sdpa = supports_sdpa,
)
+ _finish_layernorm_check(model_type, prior_high_precision)
_run_temporary_patches("post_compile")
return model_types, supports_sdpa[0]
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index a2c6301cb30..ef402a8f82b 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -56,6 +56,7 @@ from .loader_utils import (
prepare_device_map,
requested_device_map,
_offline_aware_load,
+ _restore_load_scoped_env,
_resolve_checkpoint_tokenizer_name,
_is_offline_related_error,
)
@@ -1720,6 +1721,7 @@ class FastModel(FastBaseModel):
return FastBaseModel.for_training(model, use_gradient_checkpointing)
@staticmethod
+ @_restore_load_scoped_env
@_offline_aware_load
@mistral_format_redirect
@track_explicit_4bit_request
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index ad9119cc2b8..49f189317d0 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -3648,6 +3648,26 @@ def _note_offline_retry(error, retry_error):
pass
+# Set per load (family branches in FastModel.from_pretrained, the compiler's norm check) and read only during it. Left set, a later load of another family gets float32 norms beside 16 bit projections and fails with "float != BFloat16".
+LOAD_SCOPED_ENV_VARS = ("UNSLOTH_HIGH_PRECISION_LAYERNORM",)
+
+
+def _restore_load_scoped_env(fn):
+ @functools.wraps(fn)
+ def _wrapper(*args, **kwargs):
+ saved = {name: os.environ.get(name) for name in LOAD_SCOPED_ENV_VARS}
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ for name, value in saved.items():
+ if value is None:
+ os.environ.pop(name, None)
+ else:
+ os.environ[name] = value
+
+ return _wrapper
+
+
def _offline_aware_load(fn):
"""Decide offline ONCE (local_files_only kwarg or env) and force it around the whole load. If we started online and hit a network error, retry once forced-offline. The network-up online path is unchanged: no window, no retry."""