Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Refuse fp8 training layers under tensor parallel instead of silently training in 16 bit (#16881)

Daniel Hanfab4676b0fd0
diff --git a/tests/test_parallel_capabilities_precision.py b/tests/test_parallel_capabilities_precision.py
index 5fe63525292..9790914fc9c 100644
--- a/tests/test_parallel_capabilities_precision.py
+++ b/tests/test_parallel_capabilities_precision.py
@@ -82,10 +82,27 @@ def _float8_full():
return model
+def _float8_subclass():
+ # torchao's own conversion (no Unsloth markers), then a renamed Float8Linear subclass.
+ from torchao.float8 import Float8LinearConfig, convert_to_float8_training
+ from torchao.float8.float8_linear import Float8Linear
+
+ model = convert_to_float8_training(_model(), config = Float8LinearConfig(emulate = True))
+ renamed = type("MyFp8Linear", (Float8Linear,), {})
+ swapped = 0
+ for module in model.modules():
+ if type(module) is Float8Linear:
+ module.__class__ = renamed
+ swapped += 1
+ assert swapped
+ return model
+
+
BUILDERS = {
"fp8_lora_attached": _fp8_lora_attached,
"fp8_lora_requested": _fp8_lora_requested,
"float8_full": _float8_full,
+ "float8_subclass": _float8_subclass,
}
LAYOUTS = {
"cp2": (dict(tp = 1, dp = 1, world = 2, cp = 2), "cp x precision"),
diff --git a/tests/test_tp_env_gloo.py b/tests/test_tp_env_gloo.py
index 25581bf26c0..9075d924d64 100644
--- a/tests/test_tp_env_gloo.py
+++ b/tests/test_tp_env_gloo.py
@@ -122,6 +122,7 @@ def _case_units(rank, world, tmp):
wrapped = tp_env.tensor_parallel_env_load(loader)
out["refuse_vllm"] = _raised(lambda: wrapped("m", fast_inference = True))
out["refuse_8bit"] = _raised(lambda: wrapped("m", load_in_8bit = True))
+ out["refuse_float8"] = _raised(lambda: wrapped("m", full_finetuning = True, float8_training = True))
layout(UNSLOTH_TP_SIZE = 2, UNSLOTH_FSDP2 = 1, UNSLOTH_FSDP2_SHARDED_LOAD = 1, WORLD_SIZE = 2)
out["refuse_fsdp2_sharded_load"] = _raised(lambda: wrapped("m", False))
os.environ.pop("UNSLOTH_FSDP2_SHARDED_LOAD", None)
@@ -616,6 +617,7 @@ def test_env_layout_parsing_and_refusals():
assert res["fsdp2"] == (2, 2, 4, True)
assert "native FSDP2 only" in res["accelerate_fsdp"]
assert "vLLM" in res["refuse_vllm"] and "UNSLOTH_TP_SIZE=2" in res["refuse_vllm"]
+ assert "float8_training" in res["refuse_float8"] and "UNSLOTH_FSDP2=1" in res["refuse_float8"]
assert "8-bit weights" in res["refuse_8bit"]
assert "UNSLOTH_FSDP2_SHARDED_LOAD=1" in (res["refuse_fsdp2_sharded_load"] or ""), res[
"refuse_fsdp2_sharded_load"
diff --git a/tests/test_tp_low_precision_refused_gloo.py b/tests/test_tp_low_precision_refused_gloo.py
new file mode 100644
index 00000000000..ee74c46a9d6
--- /dev/null
+++ b/tests/test_tp_low_precision_refused_gloo.py
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""apply_tensor_parallel on a model whose Linear layers train in fp8 (torchao Float8Linear from
+float8_training, or fp8 LoRA frozen bases from enable_fp8_lora), gloo world of 2 on the CPU. Tensor parallel
+runs every projection through its own local-shard GEMM, so the fp8 path would never run (the model would
+silently train in 16 bit) and per-shard scales would differ from one process: it is refused before any weight
+is sharded, and the fp8 layers are left as they were."""
+
+import multiprocessing
+import os
+
+if multiprocessing.parent_process() is not None or getattr(
+ multiprocessing.current_process(), "_inheriting", False
+):
+ os.environ["CUDA_VISIBLE_DEVICES"] = "" # spawned gloo ranks: CPU only, also on GPU hosts
+
+import sys # noqa: E402
+import tempfile # noqa: E402
+
+import pytest # noqa: E402
+import torch # noqa: E402
+
+pytest.importorskip("torchao.float8")
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from test_tp_native_gloo import _free_port, _import_checkout_unsloth # noqa: E402
+
+
+def _model(kind):
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+
+ torch.manual_seed(0)
+ model = Qwen3ForCausalLM(
+ Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 16,
+ tie_word_embeddings = False,
+ )
+ ).to(torch.bfloat16)
+ if kind in ("float8_training", "float8_subclass"):
+ import dataclasses
+
+ from torchao.float8 import Float8LinearConfig
+ from torchao.float8.float8_linear import Float8Linear
+
+ from unsloth.models.float8_training import convert_model_to_float8_training
+
+ config = dataclasses.replace(Float8LinearConfig.from_recipe_name("rowwise"), emulate = True)
+ assert convert_model_to_float8_training(model, config, size_filter = False)
+ if kind == "float8_subclass":
+ # A renamed subclass still runs torchao's fp8 forward.
+ renamed = type("MyFp8Linear", (Float8Linear,), {})
+ for module in model.modules():
+ if type(module) is Float8Linear:
+ module.__class__ = renamed
+ return model
+ from peft import LoraConfig, get_peft_model
+
+ from unsloth.models.float8_training import enable_fp8_lora
+
+ model = get_peft_model(
+ model,
+ LoraConfig(
+ r = 8, lora_alpha = 16, target_modules = ["q_proj", "v_proj", "gate_proj", "down_proj"]
+ ),
+ )
+ assert enable_fp8_lora(model, "fp8_lora_tensorwise", allow_cpu = True)
+ return model
+
+
+def _worker(rank, world, port, tmp, kind):
+ import datetime
+
+ os.environ.update(
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ RANK = str(rank),
+ WORLD_SIZE = str(world),
+ LOCAL_RANK = str(rank),
+ UNSLOTH_FP8_LORA_FUSED_OPS = "0",
+ )
+ torch.set_num_threads(2)
+ error = _import_checkout_unsloth()
+ if error is not None:
+ torch.save({"skip": error}, os.path.join(tmp, f"result_{rank}.pt"))
+ return
+ import torch.distributed as dist
+ from torch.distributed.device_mesh import init_device_mesh
+
+ from unsloth.models import tp_native as tpn
+
+ dist.init_process_group(
+ "gloo", rank = rank, world_size = world, timeout = datetime.timedelta(seconds = 300)
+ )
+ try:
+ model = _model(kind)
+ before = {n: (tuple(p.shape), p.detach().clone()) for n, p in model.named_parameters()}
+ refused = None
+ try:
+ tpn.apply_tensor_parallel(
+ model, init_device_mesh("cpu", (world,), mesh_dim_names = ("tp",))["tp"]
+ )
+ except ValueError as e:
+ refused = str(e)
+ after = dict(model.named_parameters())
+ res = {
+ "refused": refused,
+ "dtensors": sum(tpn._is_dtensor(p) for p in model.parameters()),
+ "changed": [
+ n
+ for n, (shape, value) in before.items()
+ if n not in after
+ or tuple(after[n].shape) != shape
+ or tpn._is_dtensor(after[n])
+ or not torch.equal(after[n].detach(), value)
+ ],
+ "float8_linears": sum(
+ any(c.__name__ == "Float8Linear" for c in type(m).__mro__) for m in model.modules()
+ ),
+ "fp8_lora_bases": sum("_unsloth_fp8_lora" in m.__dict__ for m in model.modules()),
+ "is_tp": tpn.is_tensor_parallel(model),
+ }
+ torch.save(res, os.path.join(tmp, f"result_{rank}.pt"))
+ finally:
+ dist.destroy_process_group()
+
+
[email protected]("kind", ["float8_training", "float8_subclass", "fp8_lora"])
+def test_tp2_low_precision_model_is_refused_before_sharding(kind):
+ with tempfile.TemporaryDirectory() as tmp:
+ ctx = multiprocessing.get_context("spawn")
+ port = _free_port()
+ procs = [ctx.Process(target = _worker, args = (r, 2, port, tmp, kind)) for r in range(2)]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(600)
+ assert [p.exitcode for p in procs] == [0, 0]
+ results = [
+ torch.load(os.path.join(tmp, f"result_{r}.pt"), weights_only = False) for r in range(2)
+ ]
+ if "skip" in results[0]:
+ pytest.skip(f"import unsloth failed on this host: {results[0]['skip']}")
+ for res in results:
+ assert res["refused"] is not None and "fp8" in res["refused"], res
+ assert res["dtensors"] == 0 and res["changed"] == [] and not res["is_tp"], res
+ assert res["float8_linears"] + res["fp8_lora_bases"] > 0, res
diff --git a/unsloth/models/tp_env.py b/unsloth/models/tp_env.py
index 7fcfc7dbb4a..0eed5a56d10 100644
--- a/unsloth/models/tp_env.py
+++ b/unsloth/models/tp_env.py
@@ -115,6 +115,8 @@ _REFUSED_LOAD_FLAGS = (
("fast_inference", "vLLM (fast_inference = True)"),
("load_in_8bit", "8-bit weights (load_in_8bit = True; load in 4-bit or 16-bit)"),
("load_in_fp8", "fp8 weights (load_in_fp8)"),
+ # Refused again at Trainer init ("tp / pp x precision"); here before any shard is read.
+ ("float8_training", "fp8 training (float8_training; UNSLOTH_FSDP2=1 trains it data parallel)"),
)
_REFUSED_PIPELINE_LOAD_FLAGS = (
("full_finetuning", "full finetuning (full_finetuning = True; use UNSLOTH_FSDP2=1 instead)"),
diff --git a/unsloth/models/tp_native.py b/unsloth/models/tp_native.py
index 4833175bae4..1081498f8cd 100644
--- a/unsloth/models/tp_native.py
+++ b/unsloth/models/tp_native.py
@@ -898,6 +898,18 @@ def _refuse_base(
raise ValueError(
f"Unsloth: tensor parallelism v1 does not support packed MXFP4 weights ({path})."
)
+ # Every projection runs through the region's own local-shard GEMM, never the module's forward: fp8
+ # training layers would silently train in 16 bit, and per-shard scales would differ from one process.
+ float8_linear = any(
+ c.__name__ == "Float8Linear" and c.__module__.startswith("torchao.")
+ for c in type(base).__mro__
+ )
+ if "_unsloth_fp8_lora" in base.__dict__ or float8_linear:
+ raise ValueError(
+ f"Unsloth: tensor parallelism v1 does not support fp8 training layers ({path}: "
+ f"{'torchao Float8Linear' if float8_linear else 'fp8 LoRA base'}); its local-shard GEMMs "
+ "would bypass the fp8 path. Use UNSLOTH_FSDP2=1 (data parallel) for float8_training, or drop it."
+ )
for module in proj_module.modules():
forward_qat = any("FakeQuantized" in c.__name__ for c in type(module).__mro__)
if (
diff --git a/unsloth/parallel_capabilities.py b/unsloth/parallel_capabilities.py
index 021f400d566..29657ce3d39 100644
--- a/unsloth/parallel_capabilities.py
+++ b/unsloth/parallel_capabilities.py
@@ -1245,8 +1245,10 @@ def _low_precision(model) -> Optional[str]:
for m in model.modules():
if "_unsloth_fp8_lora" in getattr(m, "__dict__", {}):
return "fp8"
- cls = type(m)
- if cls.__name__ == "Float8Linear" and cls.__module__.startswith("torchao."):
+ if any(
+ c.__name__ == "Float8Linear" and c.__module__.startswith("torchao.")
+ for c in type(m).__mro__
+ ):
return "fp8"
except Exception:
pass