Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Apply the repo formatter to 26 files merged without it (#14817)

Daniel Handc80c0434ea1
diff --git a/tests/test_attention_dispatch_compiled_block.py b/tests/test_attention_dispatch_compiled_block.py
index 1a9c15b9ade..da1f94d30a0 100644
--- a/tests/test_attention_dispatch_compiled_block.py
+++ b/tests/test_attention_dispatch_compiled_block.py
@@ -98,10 +98,14 @@ def _compile_counts(fn, *inputs, dynamic):
def test_gqa_probe_takes_symbolic_head_sizes(dynamic):
# dynamic=True makes head_dim and the head counts SymInts; the constant-result probe must still trace.
def state(Q, K):
- takes = ad._sdpa_gqa_state(str(Q.device), Q.dtype, Q.shape[-1], Q.shape[1], K.shape[1], False, True)
+ takes = ad._sdpa_gqa_state(
+ str(Q.device), Q.dtype, Q.shape[-1], Q.shape[1], K.shape[1], False, True
+ )
return Q.sum() + K.sum() + (1 if takes else 2)
- inputs = [(torch.zeros(b, 8, t, 64), torch.zeros(b, 2, t, 64)) for b, t in ((3, 10), (5, 12), (6, 14))]
+ inputs = [
+ (torch.zeros(b, 8, t, 64), torch.zeros(b, 2, t, 64)) for b, t in ((3, 10), (5, 12), (6, 14))
+ ]
outs, graphs, breaks = _compile_counts(state, *inputs, dynamic = dynamic)
want = 1 if ad._sdpa_gqa_state("cpu", torch.float32, 64, 8, 2, False, True) else 2
assert [out.item() for out in outs] == [want] * 3
@@ -114,7 +118,9 @@ def test_gqa_probe_takes_symbolic_head_sizes(dynamic):
@pytest.mark.parametrize("dynamic", [False, True])
def test_varlen_probe_takes_symbolic_head_sizes(dynamic):
def beats(Q, K):
- return Q.sum() + K.sum() + (1 if ad._torch_varlen_beats_segments(Q, K, (5, 3), 2, True) else 2)
+ return (
+ Q.sum() + K.sum() + (1 if ad._torch_varlen_beats_segments(Q, K, (5, 3), 2, True) else 2)
+ )
Q, K = torch.zeros(1, 4, 8, 16), torch.zeros(1, 2, 8, 16)
outs, _, breaks = _compile_counts(beats, (Q, K), dynamic = dynamic)
@@ -125,7 +131,9 @@ def test_varlen_probe_takes_symbolic_head_sizes(dynamic):
def _has_real_cuda():
try:
- from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+ from real_accelerator import (
+ has_real_cuda,
+ ) # tests/_shared, on sys.path via tests/conftest.py
except Exception:
return torch.cuda.is_available()
return has_real_cuda()
@@ -167,7 +175,9 @@ def test_cuda_gqa_sdpa_compiles_fullgraph(dynamic):
for bsz, q_len in shapes:
g = torch.Generator(device = "cuda").manual_seed(q_len)
Q, K, V = (
- torch.randn(bsz, h, q_len, 64, device = "cuda", generator = g, dtype = torch.bfloat16).requires_grad_()
+ torch.randn(
+ bsz, h, q_len, 64, device = "cuda", generator = g, dtype = torch.bfloat16
+ ).requires_grad_()
for h in (8, 2, 2)
)
out = compiled(Q, K, V)
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 5418a6f9567..9c4e41801a6 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -220,7 +220,12 @@ def _install_blocks(M, model, monkeypatch):
return inner
-def _outer_loss(model, ids, fullgraph, backend = "aot_eager"):
+def _outer_loss(
+ model,
+ ids,
+ fullgraph,
+ backend = "aot_eager",
+):
from torch._dynamo.utils import counters
torch._dynamo.reset()
@@ -277,7 +282,9 @@ def test_outer_compile_traces_decode_aware_moe(request, monkeypatch, fullgraph):
want = model(input_ids = ids, labels = ids, use_cache = False).loss.detach()
block = type(model.model.layers[0].mlp)
monkeypatch.setattr(
- block, "forward", zoo_utils.torch_compiler_disable_unless_decode(recursive = True)(block.forward)
+ block,
+ "forward",
+ zoo_utils.torch_compiler_disable_unless_decode(recursive = True)(block.forward),
)
monkeypatch.setattr(M, "_lora_would_fold", lambda experts: False)
inner = _install_blocks(M, model, monkeypatch)
diff --git a/tests/test_bnb_integration_compile.py b/tests/test_bnb_integration_compile.py
index d503573b454..c75f9e757ae 100644
--- a/tests/test_bnb_integration_compile.py
+++ b/tests/test_bnb_integration_compile.py
@@ -532,7 +532,9 @@ def test_fast_linear_forward_sees_lora_updates(nf4_kernels):
@pytest.mark.parametrize("bnb_gemm", [True, False])
-def test_fast_linear_forward_lora_under_a_different_autocast_dtype(nf4_kernels, monkeypatch, bnb_gemm):
+def test_fast_linear_forward_lora_under_a_different_autocast_dtype(
+ nf4_kernels, monkeypatch, bnb_gemm
+):
# fp16 activations under bf16 autocast: every base path must hand the LoRA update X.dtype.
if not torch.cuda.is_bf16_supported():
pytest.skip("needs bf16")
@@ -544,7 +546,9 @@ def test_fast_linear_forward_lora_under_a_different_autocast_dtype(nf4_kernels,
monkeypatch.setattr(U, "_BNB_GEMM_4BIT_DEVICE", {})
monkeypatch.setattr(U, "_BNB_GEMM_4BIT_ARCHS", {torch.cuda.get_device_capability()})
matmul_4bit = U.bnb.matmul_4bit
- monkeypatch.setattr(U.bnb, "matmul_4bit", lambda *a, **k: calls.append(1) or matmul_4bit(*a, **k))
+ monkeypatch.setattr(
+ U.bnb, "matmul_4bit", lambda *a, **k: calls.append(1) or matmul_4bit(*a, **k)
+ )
_, block = _lora_block(dtype = torch.float16)
proj = block.o_proj
X = torch.randn(4, 1, 256, dtype = torch.float16, device = DEVICE)
diff --git a/tests/test_fp8_block_gemv.py b/tests/test_fp8_block_gemv.py
index ad85e860e2d..9a415baae8f 100644
--- a/tests/test_fp8_block_gemv.py
+++ b/tests/test_fp8_block_gemv.py
@@ -21,7 +21,11 @@ def _gemv_only(monkeypatch):
monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", False)
-def _quantize(N, K, seed = 0):
+def _quantize(
+ N,
+ K,
+ seed = 0,
+):
g = torch.Generator(device = "cuda").manual_seed(seed)
W = torch.randn(N, K, device = "cuda", generator = g)
nb, kb = triton.cdiv(N, 128), triton.cdiv(K, 128)
@@ -90,11 +94,17 @@ def test_routing_decode_only(monkeypatch):
real = F8.fp8_block_gemv
monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append(1) or real(*a))
with torch.no_grad():
- F8.fp8_block_quant_linear(torch.randn(8, 1, 1024, device = "cuda", dtype = torch.bfloat16), q, s)
+ F8.fp8_block_quant_linear(
+ torch.randn(8, 1, 1024, device = "cuda", dtype = torch.bfloat16), q, s
+ )
assert len(calls) == 1
- F8.fp8_block_quant_linear(torch.randn(2, 4, 1024, device = "cuda", dtype = torch.bfloat16), q, s)
- F8.fp8_block_quant_linear(torch.randn(17, 1, 1024, device = "cuda", dtype = torch.bfloat16), q, s)
- X =torch.randn(2, 1, 1024, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ F8.fp8_block_quant_linear(
+ torch.randn(2, 4, 1024, device = "cuda", dtype = torch.bfloat16), q, s
+ )
+ F8.fp8_block_quant_linear(
+ torch.randn(17, 1, 1024, device = "cuda", dtype = torch.bfloat16), q, s
+ )
+ X = torch.randn(2, 1, 1024, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
F8.fp8_block_quant_linear(X, q, s).sum().backward() # training keeps the autograd path
assert len(calls) == 1 and X.grad is not None
assert not F8.can_use_fp8_block_gemv(X.detach(), q, s.t().contiguous())
@@ -184,18 +194,26 @@ def test_router_order(monkeypatch, route, expect):
if F8.fp8_block_matmul is not F8.w8a8_block_fp8_matmul_triton:
pytest.skip("the fused small-M kernel replaces the Triton block matmul route only")
monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", True)
- fn = F8._fp8_block_small_m_first if route == "small_m_first" else F8._with_fp8_block_gemv(F8.fp8_torch_block_quant_forward)
+ fn = (
+ F8._fp8_block_small_m_first
+ if route == "small_m_first"
+ else F8._with_fp8_block_gemv(F8.fp8_torch_block_quant_forward)
+ )
q, s = _quantize(1024, 4096)
X = torch.randn(4, 1, 4096, device = "cuda", dtype = torch.bfloat16)
calls = []
real_gemv, real_small = F8.fp8_block_gemv, F8.fp8_block_small_m_linear
monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append("gemv") or real_gemv(*a))
- monkeypatch.setattr(F8, "fp8_block_small_m_linear", lambda *a: calls.append("small_m") or real_small(*a))
+ monkeypatch.setattr(
+ F8, "fp8_block_small_m_linear", lambda *a: calls.append("small_m") or real_small(*a)
+ )
with torch.no_grad():
y = fn(X, q, s)
assert calls == [expect]
# act_quant rounds X to e4m3 on the fused path: a few percent, against the GEMV's bf16 eps.
- assert _rel_err(y, _oracle(X, q, s)) <= (5e-2 if expect == "small_m" else torch.finfo(torch.bfloat16).eps)
+ assert _rel_err(y, _oracle(X, q, s)) <= (
+ 5e-2 if expect == "small_m" else torch.finfo(torch.bfloat16).eps
+ )
Xg = X.clone().requires_grad_()
calls.clear()
fn(Xg, q, s).sum().backward()
@@ -213,14 +231,19 @@ def test_router_small_m_first_falls_back(monkeypatch):
gemv = F8.fp8_block_gemv(X, q, s)
torch.testing.assert_close(compiled, gemv, atol = 0, rtol = 0)
monkeypatch.setattr(F8, "_fp8_device_lacks_kernel", lambda device: True)
- monkeypatch.setattr(F8, "fp8_block_small_m_linear", lambda *a: pytest.fail("fused kernel used below sm89"))
+ monkeypatch.setattr(
+ F8, "fp8_block_small_m_linear", lambda *a: pytest.fail("fused kernel used below sm89")
+ )
with torch.no_grad():
F8._fp8_block_small_m_first(X, _uncached(q), s)
def test_router_binding_follows_sm_count():
bound = F8.fp8_block_quant_linear
- if F8.fp8_block_matmul is not F8.w8a8_block_fp8_matmul_triton or bound.__wrapped__ is not F8.fp8_torch_block_quant_forward:
+ if (
+ F8.fp8_block_matmul is not F8.w8a8_block_fp8_matmul_triton
+ or bound.__wrapped__ is not F8.fp8_torch_block_quant_forward
+ ):
pytest.skip("FBGEMM / torchao block route: the GEMV wraps it")
big = torch.version.hip is None and min(F8._SM_COUNTS) >= 100
assert (bound is F8._fp8_block_small_m_first) == big
diff --git a/tests/test_fp8_block_small_m.py b/tests/test_fp8_block_small_m.py
index 94061b79a7a..7804da55472 100644
--- a/tests/test_fp8_block_small_m.py
+++ b/tests/test_fp8_block_small_m.py
@@ -26,10 +26,18 @@ def _no_block_gemv(monkeypatch):
monkeypatch.setattr(f, "_FP8_BLOCK_GEMV", False, raising = False)
-def _block_weight(N, K, scale_dtype = torch.float32):
+def _block_weight(
+ N,
+ K,
+ scale_dtype = torch.float32,
+):
W = torch.randn(N, K, device = "cuda") * 0.02
s = W.abs().view(N // 128, 128, K // 128, 128).amax((1, 3)) / 448.0
- q = (W.view(N // 128, 128, K // 128, 128) / s[:, None, :, None]).view(N, K).to(torch.float8_e4m3fn)
+ q = (
+ (W.view(N // 128, 128, K // 128, 128) / s[:, None, :, None])
+ .view(N, K)
+ .to(torch.float8_e4m3fn)
+ )
return q, s.contiguous().to(scale_dtype)
@@ -83,7 +91,9 @@ def test_small_m_matches_main(monkeypatch, dtype, scale_dtype, N, K, M):
# Only the fp32 sum order over K splits differs: bound is one output rounding plus fp32 sum error.
mag = torch.maximum(new.double().abs(), main.double().abs())
k_blocks = K // 128
- bound = torch.finfo(dtype).eps * mag + 2 * k_blocks * torch.finfo(torch.float32).eps * _oracle_abs(X, q, s)
+ bound = torch.finfo(dtype).eps * mag + 2 * k_blocks * torch.finfo(
+ torch.float32
+ ).eps * _oracle_abs(X, q, s)
assert ((new.double() - main.double()).abs() <= bound + 1e-30).all()
err_new = (new.double() - ref).abs().mean()
err_main = (main.double() - ref).abs().mean()
@@ -125,7 +135,9 @@ def test_pre_sm89_keeps_dequant_route(monkeypatch):
q, s = _block_weight(1024, 4096)
X = torch.randn(1, 2, 4096, device = "cuda", dtype = torch.bfloat16)
monkeypatch.setattr(f, "_fp8_device_lacks_kernel", lambda device: True)
- monkeypatch.setattr(f, "fp8_block_small_m_linear", lambda *a: pytest.fail("small-M kernel used below sm89"))
+ monkeypatch.setattr(
+ f, "fp8_block_small_m_linear", lambda *a: pytest.fail("small-M kernel used below sm89")
+ )
out = f.fp8_linear(X, q, s)
ref = X @ f._blockwise_weight_dequant_any_shape(q, s, [128, 128], X.dtype).t()
assert torch.equal(out, ref)
diff --git a/tests/test_fp8_rowwise_gemv_rows.py b/tests/test_fp8_rowwise_gemv_rows.py
index 58f3147fc87..d6e4df72e2a 100644
--- a/tests/test_fp8_rowwise_gemv_rows.py
+++ b/tests/test_fp8_rowwise_gemv_rows.py
@@ -27,7 +27,11 @@ def F():
return fp8
-def _weights(N, K, seed = 0):
+def _weights(
+ N,
+ K,
+ seed = 0,
+):
g = torch.Generator(device = "cuda").manual_seed(seed)
W = torch.randn(N, K, device = "cuda", generator = g) * 0.02
scale = (W.abs().amax(1, keepdim = True) / 448.0).float()
@@ -62,13 +66,16 @@ def test_rows_launch_widens_past_int32(F, monkeypatch, N, K, expected):
assert rec.calls[0][2]["LONG_INDEXING"] is expected
[email protected]("dtype, N, K, split", [
- (torch.bfloat16, 4096, 12288, 8), # long K (down_proj)
- (torch.float16, 1024, 4096, 8), # narrow N (kv_proj)
- (torch.bfloat16, 4096, 4096, 1),
- (torch.float16, 12288, 4096, 1),
- (torch.bfloat16, 512, 2048, 1),
-])
+ "dtype, N, K, split",
+ [
+ (torch.bfloat16, 4096, 12288, 8), # long K (down_proj)
+ (torch.float16, 1024, 4096, 8), # narrow N (kv_proj)
+ (torch.bfloat16, 4096, 4096, 1),
+ (torch.float16, 12288, 4096, 1),
+ (torch.bfloat16, 512, 2048, 1),
+ ],
+)
def test_split_k_rule(F, dtype, N, K, split):
assert F._fp8_rows_config(N, K, dtype)[2] == split
@@ -150,5 +157,9 @@ def test_rows_off_where_triton_miscompiles_the_gemv(F, monkeypatch):
monkeypatch.setattr(F, "_fp8_gemv_miscompiles", lambda device: True)
monkeypatch.setattr(F, "_triton_builds_blackwell", lambda: True)
Wq, scale = _weights(1024, 512)
- assert not F.can_use_fp8_rowwise_gemv(torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)
- assert F.can_use_fp8_rowwise_gemv(torch.randn(1, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale)
+ assert not F.can_use_fp8_rowwise_gemv(
+ torch.randn(4, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale
+ )
+ assert F.can_use_fp8_rowwise_gemv(
+ torch.randn(1, 1, 512, device = "cuda", dtype = torch.bfloat16), Wq, scale
+ )
diff --git a/tests/test_fsdp2_save_gloo.py b/tests/test_fsdp2_save_gloo.py
index 7d35883189e..556b7ffb512 100644
--- a/tests/test_fsdp2_save_gloo.py
+++ b/tests/test_fsdp2_save_gloo.py
@@ -579,7 +579,9 @@ def _case_dcp_quant_state(rank, world_size, m):
fully_shard(model, mesh = mesh)
m.install_dcp_bnb_state_dict()
save_opts = StateDictOptions(full_state_dict = True) # every rank gets the full dict
- saved = {k: v.clone() for k, v in dcp.get_model_state_dict(model, options = save_opts).items()}
+ saved = {
+ k: v.clone() for k, v in dcp.get_model_state_dict(model, options = save_opts).items()
+ }
states = [layer.weight.quant_state for layer in model]
before = [s.absmax.clone() for s in states]
results = {}
@@ -595,16 +597,18 @@ def _case_dcp_quant_state(rank, world_size, m):
key = next(k for k in bad if k.endswith("weight.absmax"))
bad[key] = bad[key][:-1].clone()
try:
- dcp.set_model_state_dict(
- model, bad, options = StateDictOptions(full_state_dict = True)
- )
+ dcp.set_model_state_dict(model, bad, options = StateDictOptions(full_state_dict = True))
out[f"mismatch_nested_{nested}"] = "loaded"
except ValueError as error:
out[f"mismatch_nested_{nested}"] = "ValueError" if "4-bit" in str(error) else str(error)
return out
-def _sharded_nf4(world_size, m, device = "cpu"):
+def _sharded_nf4(
+ world_size,
+ m,
+ device = "cpu",
+):
q = _load("fsdp2_qlora")
from torch.distributed.device_mesh import init_device_mesh
from torch.distributed.fsdp import fully_shard
@@ -628,7 +632,9 @@ def _legacy_and_mismatched_loads(rank, model, saved, out):
import torch.distributed.checkpoint.state_dict as dcp
def dequantized(state_dict):
- stats = {k[len("0.weight.") :]: v for k, v in state_dict.items() if k.startswith("0.weight.")}
+ stats = {
+ k[len("0.weight.") :]: v for k, v in state_dict.items() if k.startswith("0.weight.")
+ }
state = QuantState.from_dict(stats, device = "cpu")
packed = state_dict["0.weight"].cpu().reshape(-1, 1)
state.absmax, state.code = state.absmax.cpu(), state.code.cpu()
@@ -643,12 +649,13 @@ def _legacy_and_mismatched_loads(rank, model, saved, out):
for broadcast in (False, True):
options = StateDictOptions(full_state_dict = True, broadcast_from_rank0 = broadcast)
try:
- dcp.set_model_state_dict(model, legacy if (rank == 0 or not broadcast) else {}, options = options)
- after = dcp.get_model_state_dict(model, options = full)
- out[f"legacy_{broadcast}"] = (
- torch.equal(model[0].weight.quant_state.absmax, live_absmax)
- and torch.equal(dequantized(after), reference)
+ dcp.set_model_state_dict(
+ model, legacy if (rank == 0 or not broadcast) else {}, options = options
)
+ after = dcp.get_model_state_dict(model, options = full)
+ out[f"legacy_{broadcast}"] = torch.equal(
+ model[0].weight.quant_state.absmax, live_absmax
+ ) and torch.equal(dequantized(after), reference)
except Exception as error:
out[f"legacy_{broadcast}"] = f"{type(error).__name__}: {error}"
try:
@@ -657,7 +664,9 @@ def _legacy_and_mismatched_loads(rank, model, saved, out):
)
out[f"mismatch_{broadcast}"] = "loaded"
except ValueError as error:
- out[f"mismatch_{broadcast}"] = "ValueError" if "`0.weight`" in str(error) else str(error)
+ out[f"mismatch_{broadcast}"] = (
+ "ValueError" if "`0.weight`" in str(error) else str(error)
+ )
def _case_dcp_quant_state_missing(rank, world_size, m):
@@ -779,7 +788,10 @@ def _worker(rank, world_size, port, case, queue):
timeout = datetime.timedelta(seconds = 4) if case == "slow_writer" else None
dist.init_process_group(
- "nccl" if nccl else "gloo", rank = rank, world_size = world_size, **({"timeout": timeout} if timeout else {})
+ "nccl" if nccl else "gloo",
+ rank = rank,
+ world_size = world_size,
+ **({"timeout": timeout} if timeout else {}),
)
m = _load("fsdp2_save")
if case == "slow_writer":
@@ -811,7 +823,11 @@ def _worker(rank, world_size, port, case, queue):
queue.put((rank, {"error": f"{type(error).__name__}: {error}\n{traceback.format_exc()}"}))
-def _run(case, world_size = 2, cpu_only = True):
+def _run(
+ case,
+ world_size = 2,
+ cpu_only = True,
+):
import torch.multiprocessing as mp
ctx = mp.get_context("spawn")
diff --git a/tests/test_fused_lora_peft_fallbacks.py b/tests/test_fused_lora_peft_fallbacks.py
index a4e4caff307..29a5e59ce26 100644
--- a/tests/test_fused_lora_peft_fallbacks.py
+++ b/tests/test_fused_lora_peft_fallbacks.py
@@ -39,7 +39,12 @@ if has_real_cuda():
CASES += [("cuda", torch.float32, path) for path in _paths("cuda")]
-def _run(fn, model, X0, autocast_dtype = None):
+def _run(
+ fn,
+ model,
+ X0,
+ autocast_dtype = None,
+):
model.zero_grad(set_to_none = True)
torch.manual_seed(7)
X = X0.detach().clone().requires_grad_()
@@ -50,14 +55,25 @@ def _run(fn, model, X0, autocast_dtype = None):
out = fn(X * 1)
outs = out if isinstance(out, tuple) else (out,)
gen = torch.Generator(device = "cpu").manual_seed(11)
- loss = sum((o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum() for o in outs)
+ loss = sum(
+ (o.double() * torch.randn(o.shape, generator = gen, dtype = torch.float64).to(o.device)).sum()
+ for o in outs
+ )
loss.backward()
grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
model.zero_grad(set_to_none = True)
return [o.detach() for o in outs], X.grad.clone(), grads
-def _assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype = None, exact = True):
+def _assert_falls_back_to_peft(
+ monkeypatch,
+ model,
+ block,
+ path,
+ X0,
+ autocast_dtype = None,
+ exact = True,
+):
spy = _Spy(monkeypatch)
fused = _run(PATHS[path][0](block), model, X0, autocast_dtype)
assert spy.calls == [], "a fused LoRA Function ran"
@@ -98,7 +114,6 @@ def test_real_peft_variant_runs_peft_forward(monkeypatch, dropout):
pytest.skip("PEFT without KaSA")
from peft import LoraConfig, get_peft_model
-
torch.manual_seed(3407)
cfg = LoraConfig(
r = 4,
@@ -122,7 +137,9 @@ def test_real_peft_variant_runs_peft_forward(monkeypatch, dropout):
@pytest.mark.parametrize("device,dtype,path", CASES)
@pytest.mark.parametrize("state", ["disabled", "merged"])
@pytest.mark.parametrize("dropout,lora_bias", CONFIGS)
-def test_disabled_or_merged_adapter_backward_matches_peft(monkeypatch, device, dtype, path, state, dropout, lora_bias):
+def test_disabled_or_merged_adapter_backward_matches_peft(
+ monkeypatch, device, dtype, path, state, dropout, lora_bias
+):
if state == "merged" and lora_bias:
pytest.skip("PEFT refuses to merge a lora_B bias into a base layer without one")
model, block = _peft_block(device, dtype, dropout, lora_bias)
@@ -137,7 +154,9 @@ def test_disabled_or_merged_adapter_backward_matches_peft(monkeypatch, device, d
@pytest.mark.parametrize("device,dtype,path", CASES)
@pytest.mark.parametrize("dropout,lora_bias", CONFIGS)
-def test_trainable_base_weight_gets_its_gradient(monkeypatch, device, dtype, path, dropout, lora_bias):
+def test_trainable_base_weight_gets_its_gradient(
+ monkeypatch, device, dtype, path, dropout, lora_bias
+):
model, block = _peft_block(device, dtype, dropout, lora_bias)
for p in model.parameters():
p.requires_grad_(True)
@@ -151,9 +170,15 @@ def test_trainable_base_weight_gets_its_gradient(monkeypatch, device, dtype, pat
@pytest.mark.parametrize("dropout,lora_bias", CONFIGS)
@pytest.mark.parametrize(
"x_dtype,autocast_dtype",
- [(torch.float16, torch.bfloat16), (torch.bfloat16, torch.float16), (torch.float32, torch.float16)],
+ [
+ (torch.float16, torch.bfloat16),
+ (torch.bfloat16, torch.float16),
+ (torch.float32, torch.float16),
+ ],
)
-def test_autocast_dtype_other_than_input_runs_peft(monkeypatch, path, dropout, lora_bias, x_dtype, autocast_dtype):
+def test_autocast_dtype_other_than_input_runs_peft(
+ monkeypatch, path, dropout, lora_bias, x_dtype, autocast_dtype
+):
model, block = _peft_block("cuda", torch.bfloat16, dropout, lora_bias)
X0 = torch.randn(2, 5, H, device = "cuda", dtype = x_dtype)
_assert_falls_back_to_peft(monkeypatch, model, block, path, X0, autocast_dtype)
@@ -193,13 +218,17 @@ def _disable_or_merge(model, block, state):
@pytest.mark.parametrize("state", ["disabled", "merged"])
@pytest.mark.parametrize("grad_mode", ["no_grad", "inference_mode", "grad_frozen_input"])
@pytest.mark.parametrize("dropout", [0.0, 0.3])
-def test_disabled_or_merged_adapter_without_grad_stays_fused(monkeypatch, device, dtype, path, state, grad_mode, dropout):
+def test_disabled_or_merged_adapter_without_grad_stays_fused(
+ monkeypatch, device, dtype, path, state, grad_mode, dropout
+):
# The reference forward: no gradient flows, so the fused base matmul runs and equals PEFT bitwise.
model, block = _peft_block(device, dtype, dropout)
_disable_or_merge(model, block, state)
X = torch.randn(2, 5, H, device = device, dtype = dtype)
spy = _Spy(monkeypatch)
- ctx = {"no_grad": torch.no_grad, "inference_mode": torch.inference_mode}.get(grad_mode, torch.enable_grad)
+ ctx = {"no_grad": torch.no_grad, "inference_mode": torch.inference_mode}.get(
+ grad_mode, torch.enable_grad
+ )
with ctx():
fused = PATHS[path][0](block)(X)
assert [name for name, _ in spy.calls] == FUSED_NAMES[path]
@@ -215,7 +244,9 @@ def test_disabled_or_merged_adapter_without_grad_stays_fused(monkeypatch, device
# Each projection is PEFT's base_layer(X) bitwise; the MLP's Triton SwiGLU rounds its float32
# activation 1 ulp differently from torch's SiLU * up.
tol = {} if path == "mlp" else dict(rtol = 0, atol = 0)
- for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):
+ for a, b in zip(
+ fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)
+ ):
assert not a.requires_grad
torch.testing.assert_close(a, b, **tol)
@@ -230,10 +261,17 @@ def test_disabled_adapter_without_grad_on_4bit_base_stays_fused(monkeypatch, pat
def __init__(self):
super().__init__()
for name, (i, o) in {
- "q_proj": (H, H), "k_proj": (H, H), "v_proj": (H, H), "o_proj": (H, H),
- "gate_proj": (H, 96), "up_proj": (H, 96), "down_proj": (96, H),
+ "q_proj": (H, H),
+ "k_proj": (H, H),
+ "v_proj": (H, H),
+ "o_proj": (H, H),
+ "gate_proj": (H, 96),
+ "up_proj": (H, 96),
+ "down_proj": (96, H),
}.items():
- layer = bnb.nn.Linear4bit(i, o, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")
+ layer = bnb.nn.Linear4bit(
+ i, o, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4"
+ )
layer.weight = bnb.nn.Params4bit(
layer.weight.data.to(torch.bfloat16), requires_grad = False, quant_type = "nf4"
)
@@ -242,7 +280,9 @@ def test_disabled_adapter_without_grad_on_4bit_base_stays_fused(monkeypatch, pat
torch.manual_seed(3407)
base = _Block4bit().to("cuda")
- cfg = LoraConfig(r = 4, lora_alpha = 8, lora_dropout = 0.1, target_modules = TARGETS, init_lora_weights = False)
+ cfg = LoraConfig(
+ r = 4, lora_alpha = 8, lora_dropout = 0.1, target_modules = TARGETS, init_lora_weights = False
+ )
model = get_peft_model(base, cfg)
for name, p in model.named_parameters():
if "lora_" in name:
@@ -257,7 +297,9 @@ def test_disabled_adapter_without_grad_on_4bit_base_stays_fused(monkeypatch, pat
fused = PATHS[path][0](block)(X)
ref = PATHS[path][1](block)(X)
assert [name for name, _ in spy.calls] == FUSED_NAMES[path]
- for a, b in zip(fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)):
+ for a, b in zip(
+ fused if isinstance(fused, tuple) else (fused,), ref if isinstance(ref, tuple) else (ref,)
+ ):
torch.testing.assert_close(a, b, rtol = 0, atol = 0)
@@ -284,10 +326,19 @@ def test_routing_compiles_without_graph_breaks(monkeypatch):
def fn(X):
return fast_lora.apply_lora_o(block, X), fast_lora.apply_lora_qkv(block, X)
- def check(X, autocast_dtype = None, grad = True):
+ def check(
+ X,
+ autocast_dtype = None,
+ grad = True,
+ ):
torch._dynamo.reset()
compiled = torch.compile(fn, backend = "eager", fullgraph = True)
- with torch.set_grad_enabled(grad), torch.autocast("cpu", dtype = autocast_dtype or torch.bfloat16, enabled = autocast_dtype is not None):
+ with (
+ torch.set_grad_enabled(grad),
+ torch.autocast(
+ "cpu", dtype = autocast_dtype or torch.bfloat16, enabled = autocast_dtype is not None
+ ),
+ ):
got = compiled(X)
want = fn(X)
torch.testing.assert_close(got[0], want[0])
diff --git a/tests/test_fused_preference_loss.py b/tests/test_fused_preference_loss.py
index 75b1110e027..9af5a96e12c 100644
--- a/tests/test_fused_preference_loss.py
+++ b/tests/test_fused_preference_loss.py
@@ -715,7 +715,8 @@ def _zero_head_input_grad(module, grad_input, grad_output):
@pytest.mark.parametrize(
- "kind", ["full_backward_hook", "backward_pre_hook", "legacy_backward_hook", "global_backward_hook"]
+ "kind",
+ ["full_backward_hook", "backward_pre_hook", "legacy_backward_hook", "global_backward_hook"],
)
def test_backward_hooked_head_falls_back(dpo_module, monkeypatch, kind):
"""The kernel never calls the head module, so its module backward hooks would never fire."""
@@ -745,7 +746,6 @@ def test_backward_hooked_head_falls_back(dpo_module, monkeypatch, kind):
)
else:
from torch.nn.modules.module import register_module_full_backward_hook
-
def global_hook(module, grad_input, grad_output):
if module is head:
return _zero_head_input_grad(module, grad_input, grad_output)
diff --git a/tests/test_grouped_gemm_mixed_tma_device.py b/tests/test_grouped_gemm_mixed_tma_device.py
index fa885a0d3c9..dfb9d4ce66f 100644
--- a/tests/test_grouped_gemm_mixed_tma_device.py
+++ b/tests/test_grouped_gemm_mixed_tma_device.py
@@ -380,7 +380,6 @@ def test_sm_count_is_read_once_per_card(monkeypatch):
assert sorted(reads) == [0, 1]
-
def _real_autotune_cache():
spec = importlib.util.spec_from_file_location(
"_unsloth_ac_spoof", ROOT / "unsloth" / "kernels" / "moe" / "autotune_cache.py"
@@ -394,7 +393,16 @@ SMEM = {H100: 232448, A100: 166912, L4: 101376}
TILES = ("BLOCK_SIZE_M", "BLOCK_SIZE_N", "BLOCK_SIZE_K", "num_stages")
-def _launch_tiles(monkeypatch, caps, operand_index, M, N, K, stages, dtype = torch.bfloat16):
+def _launch_tiles(
+ monkeypatch,
+ caps,
+ operand_index,
+ M,
+ N,
+ K,
+ stages,
+ dtype = torch.bfloat16,
+):
"""Run the three non-autotuned impls on cuda:operand_index (current: cuda:0); return each launch's tiles."""
module, _ = _load(monkeypatch, caps)
real = torch.cuda.get_device_properties
@@ -431,7 +439,11 @@ def _launch_tiles(monkeypatch, caps, operand_index, M, N, K, stages, dtype = tor
out["dX"] = kernels[1]._grouped_gemm_dX_kernel.last_kwargs
module._grouped_gemm_dW_impl(X = X, dY = dY, **common)
per_expert = module._use_per_expert_dW_kernel()
- dw_kernel = kernels[1]._grouped_gemm_dW_per_expert_kernel if per_expert else kernels[1]._grouped_gemm_dW_kernel
+ dw_kernel = (
+ kernels[1]._grouped_gemm_dW_per_expert_kernel
+ if per_expert
+ else kernels[1]._grouped_gemm_dW_kernel
+ )
out["dW"] = dw_kernel.last_kwargs
return ac, {k: tuple(v[t] for t in TILES) for k, v in out.items()}
@@ -440,7 +452,12 @@ def test_caller_tiles_fitted_for_the_current_card_are_refit_for_the_operand_card
# Released unsloth_zoo passes no device: an H100-fitted bf16 dX (64, 64, 256, 4) needs 122,880 B,
# over the L4's 101,376 B. The launch on the L4 operand shrinks it to fit; the H100 copy is untouched.
ac, got = _launch_tiles(monkeypatch, [H100, L4], 1, 64, 64, 256, 4)
- assert ac._estimate_smem("dX", types.SimpleNamespace(**dict(zip(TILES, (64, 64, 256, 4)))), 2, 2, L4) > SMEM[L4]
+ assert (
+ ac._estimate_smem(
+ "dX", types.SimpleNamespace(**dict(zip(TILES, (64, 64, 256, 4)))), 2, 2, L4
+ )
+ > SMEM[L4]
+ )
assert got["dX"] == (64, 64, 256, 3)
for kind, tiles in got.items():
config = types.SimpleNamespace(**dict(zip(TILES, tiles)))
diff --git a/tests/test_grpo_routing_replay_trainer_hooks.py b/tests/test_grpo_routing_replay_trainer_hooks.py
index 640075842c1..bdd632e8e2d 100644
--- a/tests/test_grpo_routing_replay_trainer_hooks.py
+++ b/tests/test_grpo_routing_replay_trainer_hooks.py
@@ -41,15 +41,21 @@ LAYERS, EXPERTS, TOP_K, PAD = 2, 8, 2, 0
def _post_init_code():
# The string constant rl.py appends to RLTrainer_post for grpo_trainer.
for node in ast.walk(ast.parse(RL_PY.read_text(encoding = "utf-8"))):
- if isinstance(node, ast.Constant) and isinstance(node.value, str) \
- and "UNSLOTH_GRPO_ROUTING_REPLAY', '0') == '1' and hasattr(model, 'vllm_engine')" in node.value:
+ if (
+ isinstance(node, ast.Constant)
+ and isinstance(node.value, str)
+ and "UNSLOTH_GRPO_ROUTING_REPLAY', '0') == '1' and hasattr(model, 'vllm_engine')"
+ in node.value
+ ):
return node.value
raise AssertionError("routing replay post-init code not found in rl.py")
def _run_post_init(model, trainer_model = None):
- trainer = SimpleNamespace(model = trainer_model if trainer_model is not None else model,
- processing_class = SimpleNamespace(pad_token_id = PAD))
+ trainer = SimpleNamespace(
+ model = trainer_model if trainer_model is not None else model,
+ processing_class = SimpleNamespace(pad_token_id = PAD),
+ )
exec(_post_init_code(), {"os": os, "self": trainer, "model": model})
@@ -66,9 +72,18 @@ def replay(monkeypatch):
def _moe(seed = 0):
torch.manual_seed(seed)
config = Qwen3MoeConfig(
- vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,
- num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,
- num_experts = EXPERTS, num_experts_per_tok = TOP_K, norm_topk_prob = True, pad_token_id = PAD,
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ moe_intermediate_size = 16,
+ num_hidden_layers = LAYERS,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_experts = EXPERTS,
+ num_experts_per_tok = TOP_K,
+ norm_topk_prob = True,
+ pad_token_id = PAD,
)
model = Qwen3MoeForCausalLM(config).float().eval()
for layer in model.model.layers:
@@ -80,7 +95,9 @@ def _moe(seed = 0):
def _natural(model, ids):
captured = {}
handles = [
- layer.mlp.gate.register_forward_hook(lambda m, a, out, i = i: captured.__setitem__(i, out[2].clone()))
+ layer.mlp.gate.register_forward_hook(
+ lambda m, a, out, i = i: captured.__setitem__(i, out[2].clone())
+ )
for i, layer in enumerate(model.model.layers)
]
with torch.no_grad():
@@ -126,9 +143,15 @@ def test_peft_reference_scoring_routes_naturally(replay):
assert all(h for h, _ in hooks_before)
forced = ((natural_ids + 3) % EXPERTS).numpy().astype(np.int16)
- rr.record_routed_experts([SimpleNamespace(
- prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced[:-1])],
- )], pad_token_id = PAD)
+ rr.record_routed_experts(
+ [
+ SimpleNamespace(
+ prompt_token_ids = ids[:3],
+ outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced[:-1])],
+ )
+ ],
+ pad_token_id = PAD,
+ )
with torch.no_grad():
policy = model(input_ids = torch.tensor([ids])).logits
@@ -168,8 +191,13 @@ def test_reference_pass_restores_hook_order_with_prepended_hooks(replay):
router.register_forward_hook(lambda m, a, out: calls.append("appended"))
decoder.register_forward_pre_hook(lambda m, a, k: None, prepend = True, with_kwargs = True)
decoder.layers[0].register_forward_pre_hook(lambda m, a: None, prepend = True)
- tables = ("_forward_hooks", "_forward_hooks_with_kwargs", "_forward_hooks_always_called",
- "_forward_pre_hooks", "_forward_pre_hooks_with_kwargs")
+ tables = (
+ "_forward_hooks",
+ "_forward_hooks_with_kwargs",
+ "_forward_hooks_always_called",
+ "_forward_pre_hooks",
+ "_forward_pre_hooks_with_kwargs",
+ )
def orders():
return [[list(getattr(m, t, {}).items()) for t in tables] for m in model.modules()]
@@ -191,14 +219,27 @@ def _bias_router_model(model_type):
extra = {
"minimax_m2": dict(num_local_experts = EXPERTS),
"hy_v3": dict(num_local_experts = EXPERTS),
- "lfm2_moe": dict(num_experts = EXPERTS, use_expert_bias = True, num_dense_layers = 0,
- layer_types = ["full_attention"] * LAYERS),
+ "lfm2_moe": dict(
+ num_experts = EXPERTS,
+ use_expert_bias = True,
+ num_dense_layers = 0,
+ layer_types = ["full_attention"] * LAYERS,
+ ),
}[model_type]
try:
config = AutoConfig.for_model(
- model_type, vocab_size = 64, hidden_size = 32, intermediate_size = 64, moe_intermediate_size = 16,
- num_hidden_layers = LAYERS, num_attention_heads = 4, num_key_value_heads = 2, head_dim = 8,
- num_experts_per_tok = TOP_K, pad_token_id = PAD, **extra,
+ model_type,
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ moe_intermediate_size = 16,
+ num_hidden_layers = LAYERS,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 8,
+ num_experts_per_tok = TOP_K,
+ pad_token_id = PAD,
+ **extra,
)
except Exception as error:
pytest.skip(f"this transformers has no {model_type}: {error}")
@@ -217,21 +258,32 @@ def test_sigmoid_correction_bias_routers_route_naturally_with_a_warning(replay,
model = _bias_router_model(model_type)
routers = [m for m in model.modules() if "TopKRouter" in type(m).__name__]
if not routers:
- pytest.skip(f"{model_type} has no TopKRouter module in this transformers (gate inside the MoE block)")
+ pytest.skip(
+ f"{model_type} has no TopKRouter module in this transformers (gate inside the MoE block)"
+ )
ids = [5, 6, 7, 30, 31, 32]
with torch.no_grad():
natural = model(input_ids = torch.tensor([ids])).logits
name = type(routers[0]).__name__
- with pytest.warns(UserWarning, match = rf"routing replay is off for {len(routers)} router.*{name} \(routing replay has no weight formula"):
+ with pytest.warns(
+ UserWarning,
+ match = rf"routing replay is off for {len(routers)} router.*{name} \(routing replay has no weight formula",
+ ):
_run_post_init(model)
assert all(not r._forward_hooks for r in routers)
# A recorded rollout with other experts no longer changes anything.
layers = model.config.num_hidden_layers
forced = np.full((len(ids) - 1, layers, TOP_K), 0, dtype = np.int16)
forced[..., 1] = 1
- rr.record_routed_experts([SimpleNamespace(
- prompt_token_ids = ids[:3], outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced)],
- )], pad_token_id = PAD)
+ rr.record_routed_experts(
+ [
+ SimpleNamespace(
+ prompt_token_ids = ids[:3],
+ outputs = [SimpleNamespace(token_ids = ids[3:], routed_experts = forced)],
+ )
+ ],
+ pad_token_id = PAD,
+ )
with torch.no_grad():
torch.testing.assert_close(model(input_ids = torch.tensor([ids])).logits, natural)
@@ -254,7 +306,10 @@ def test_formula_that_only_fits_the_priming_input_is_rejected(replay):
gate = model.model.layers[0].mlp.gate
gate.__class__ = SigmoidTopKRouter
gate.weight.data.normal_(0, 1e-4)
- with pytest.warns(UserWarning, match = r"SigmoidTopKRouter \(the softmax_renorm weight formula does not reproduce"):
+ with pytest.warns(
+ UserWarning,
+ match = r"SigmoidTopKRouter \(the softmax_renorm weight formula does not reproduce",
+ ):
_run_post_init(model)
assert not gate._forward_hooks
assert model.model.layers[1].mlp.gate._forward_hooks # the real softmax router passes the check
diff --git a/tests/test_moe_autotune_operand_device.py b/tests/test_moe_autotune_operand_device.py
index 25fa946fca1..5c2d2cca404 100644
--- a/tests/test_moe_autotune_operand_device.py
+++ b/tests/test_moe_autotune_operand_device.py
@@ -52,7 +52,11 @@ def ac(monkeypatch, tmp_path):
sys.modules.pop(name, None)
-def _gpus(monkeypatch, cards, current = 0):
+def _gpus(
+ monkeypatch,
+ cards,
+ current = 0,
+):
def index(device = None):
if device is None:
return current
@@ -88,7 +92,9 @@ def test_fallback_configs_fit_the_operand_card(ac, monkeypatch, which, act, weig
_gpus(monkeypatch, [H100, L4], current = 0)
fn = ac._get_heuristic_configs if which == "heuristic" else ac._get_default_configs
for n, k in ((2 * inter, hidden), (hidden, inter)):
- configs = fn(n_dim = n, k_dim = k, dtype = act, weight_dtype = weight, device = torch.device("cuda", 1))
+ configs = fn(
+ n_dim = n, k_dim = k, dtype = act, weight_dtype = weight, device = torch.device("cuda", 1)
+ )
assert max(_smem(ac, configs, act, weight, L4)) <= L4[1]
@@ -110,11 +116,16 @@ def _big_configs():
KernelConfigBackward_dX,
KernelConfigForward,
)
-
return (
- KernelConfigForward(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 128, BLOCK_SIZE_K = 128, num_warps = 8, num_stages = 3),
- KernelConfigBackward_dX(BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4),
- KernelConfigBackward_dW(BLOCK_SIZE_M = 128, BLOCK_SIZE_N = 256, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4),
+ KernelConfigForward(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 128, BLOCK_SIZE_K = 128, num_warps = 8, num_stages = 3
+ ),
+ KernelConfigBackward_dX(
+ BLOCK_SIZE_M = 64, BLOCK_SIZE_N = 64, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4
+ ),
+ KernelConfigBackward_dW(
+ BLOCK_SIZE_M = 128, BLOCK_SIZE_N = 256, BLOCK_SIZE_K = 256, num_warps = 8, num_stages = 4
+ ),
)
@@ -126,7 +137,11 @@ def test_cache_lookup_and_tuning_follow_the_operand_card(ac, monkeypatch):
tuned_on = []
- def tune(*args, device = None, **kwargs):
+ def tune(
+ *args,
+ device = None,
+ **kwargs,
+ ):
tuned_on.append(device)
return ac._get_heuristic_configs(n_dim = 1536, k_dim = 2048, dtype = BF16, device = device)
diff --git a/tests/test_moe_dw_accumulate_node_hooks.py b/tests/test_moe_dw_accumulate_node_hooks.py
index a566235e679..d300d483956 100644
--- a/tests/test_moe_dw_accumulate_node_hooks.py
+++ b/tests/test_moe_dw_accumulate_node_hooks.py
@@ -54,7 +54,12 @@ def gg(monkeypatch):
def dX(dY, W, **kw):
return dY @ W[0]
- def dense_dW(X, dY, dW_dtype = None, **kw):
+ def dense_dW(
+ X,
+ dY,
+ dW_dtype = None,
+ **kw,
+ ):
return (dY.T @ X).unsqueeze(0).to(dW_dtype or X.dtype)
def dW_accumulate(dW, **kw):
@@ -84,7 +89,12 @@ def _scale(g, k):
return None if g is None else g * k
-def _run(gg, monkeypatch, enabled, hook = None):
+def _run(
+ gg,
+ monkeypatch,
+ enabled,
+ hook = None,
+):
monkeypatch.setenv("UNSLOTH_MOE_DW_ACCUMULATE", "1" if enabled else "0")
gen = torch.Generator().manual_seed(0)
X = torch.randn(4, 3, generator = gen, requires_grad = True)
@@ -93,7 +103,21 @@ def _run(gg, monkeypatch, enabled, hook = None):
G = torch.randn(4, 2, generator = gen)
cfg = _cfg()
Y = gg.GroupedGemm.apply(
- X, W, torch.tensor([4]), 1, None, False, False, None, False, cfg, cfg, cfg, False, False, False
+ X,
+ W,
+ torch.tensor([4]),
+ 1,
+ None,
+ False,
+ False,
+ None,
+ False,
+ cfg,
+ cfg,
+ cfg,
+ False,
+ False,
+ False,
)
handle = None
if hook == "acc_prehook":
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.py
index 635a24d3480..1ece6165003 100644
--- a/tests/test_mxfp4_packed_low_memory.py
+++ b/tests/test_mxfp4_packed_low_memory.py
@@ -62,14 +62,24 @@ def _headroom(mib, device):
torch.cuda.empty_cache()
-def _weight(N, K, device, seed = 0):
+def _weight(
+ N,
+ K,
+ device,
+ seed = 0,
+):
g = torch.Generator(device = device).manual_seed(seed)
packed = torch.randint(0, 256, (N, K // 2), dtype = torch.uint8, device = device, generator = g)
scale = torch.randint(118, 128, (N, K // 32), dtype = torch.uint8, device = device, generator = g)
return packed, scale
-def _single_shot(x, packed, scale, bias = None):
+def _single_shot(
+ x,
+ packed,
+ scale,
+ bias = None,
+):
# What #1899 runs below sm_80: the whole 16-bit weight, one cuBLAS call.
weight = mcl.dequantize_mxfp4_packed(packed, scale, x.dtype)
return torch.nn.functional.linear(x, weight, bias)
@@ -171,7 +181,9 @@ def test_compiled_forward_is_bounded_without_graph_breaks(sm75):
torch._dynamo.reset()
counters.clear()
compiled = torch.compile(module, fullgraph = True)
- torch.testing.assert_close(compiled(x), want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
+ torch.testing.assert_close(
+ compiled(x), want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item()
+ )
assert sum(counters["graph_break"].values()) == 0
with _headroom(64, device):
got = compiled(x)
@@ -186,7 +198,12 @@ def _fake_oom_dequant(monkeypatch, fail_rows):
real = mcl.dequantize_mxfp4_packed
pending, seen = list(fail_rows), []
- def fake(packed, scale, dtype = torch.bfloat16, out = None):
+ def fake(
+ packed,
+ scale,
+ dtype = torch.bfloat16,
+ out = None,
+ ):
seen.append(packed.shape[0])
if pending and packed.shape[0] == pending[0]:
pending.pop(0)
diff --git a/tests/test_qat_phone_deployment.py b/tests/test_qat_phone_deployment.py
index b1b61eb62aa..42d7105be64 100644
--- a/tests/test_qat_phone_deployment.py
+++ b/tests/test_qat_phone_deployment.py
@@ -438,9 +438,13 @@ def test_offload_hook_on_unselected_embedding_is_not_refused(qat_scheme):
pytest.importorskip("accelerate.hooks")
model = _offload_embedding(_tiny_gemma3(), place_submodules = True)
original = model.get_input_embeddings()
- model = _prepare_model_for_qat(model, qat_scheme) # Linear-only recipes never touch the embedding
+ model = _prepare_model_for_qat(
+ model, qat_scheme
+ ) # Linear-only recipes never touch the embedding
assert model.get_input_embeddings() is original
- assert _is_fake_quantized(model.model.layers[0].mlp.down_proj) # in_features 128 fits int4's group
+ assert _is_fake_quantized(
+ model.model.layers[0].mlp.down_proj
+ ) # in_features 128 fits int4's group
@pytest.mark.parametrize("convert", ["unsloth", "strip"])
@@ -458,7 +462,10 @@ def test_offloaded_embedding_with_offloaded_buffers_converts(convert):
scale = emb.embed_scale.detach().clone()
emb.register_buffer("embed_scale", scale.clone(), persistent = True)
prefix = "model.embed_tokens."
- weights = {prefix + "weight": emb.weight.detach().clone(), prefix + "embed_scale": scale.clone()}
+ weights = {
+ prefix + "weight": emb.weight.detach().clone(),
+ prefix + "embed_scale": scale.clone(),
+ }
emb.to("meta")
add_hook_to_module(
emb,
@@ -472,7 +479,9 @@ def test_offloaded_embedding_with_offloaded_buffers_converts(convert):
assert emb.embed_scale.device.type == "meta"
model = _prepare_model_for_qat(model, "int8-int4")
with torch.no_grad(): # the hook still serves forward from weights_map
- torch.testing.assert_close(model.get_input_embeddings()(ids), reference.get_input_embeddings()(ids))
+ torch.testing.assert_close(
+ model.get_input_embeddings()(ids), reference.get_input_embeddings()(ids)
+ )
if convert == "unsloth":
_convert_torchao_model(reference)
_convert_torchao_model(model)
@@ -498,28 +507,31 @@ def _delegating_override(self, input_ids):
def _install_override(emb, kind):
import types
-
if kind == "direct_weight":
emb.forward = types.MethodType(_direct_weight_override, emb)
elif kind == "delegating":
emb.forward = types.MethodType(_delegating_override, emb)
elif kind == "accelerate_over_override":
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
-
emb.forward = types.MethodType(_delegating_override, emb)
add_hook_to_module(emb, AlignDevicesHook(execution_device = "cpu", io_same_device = True))
elif kind == "forward_hook":
return emb.register_forward_hook(lambda m, a, out: out * 2)
elif kind == "unsloth_offload_hooks":
from unsloth.models.vision import _install_offload_embedding_hooks
-
assert _install_offload_embedding_hooks(emb, None, torch.device("cpu"))
return None
@pytest.mark.parametrize(
"kind",
- ["direct_weight", "delegating", "accelerate_over_override", "forward_hook", "unsloth_offload_hooks"],
+ [
+ "direct_weight",
+ "delegating",
+ "accelerate_over_override",
+ "forward_hook",
+ "unsloth_offload_hooks",
+ ],
)
@pytest.mark.parametrize("convert", ["unsloth", "strip"])
def test_embedding_with_instance_forward_or_hooks_is_left_out_of_qat(kind, convert):
@@ -540,7 +552,9 @@ def test_embedding_with_instance_forward_or_hooks_is_left_out_of_qat(kind, conve
emb = model.get_input_embeddings()
assert emb is original and not _is_fake_quantized(emb)
assert sum("leaves these embeddings in full precision" in str(w.message) for w in caught) == 1
- assert _is_fake_quantized(model.model.layers[0].self_attn.q_proj) # the rest of the recipe still applies
+ assert _is_fake_quantized(
+ model.model.layers[0].self_attn.q_proj
+ ) # the rest of the recipe still applies
with torch.no_grad():
torch.testing.assert_close(emb(ids), before)
if handle is not None:
@@ -570,7 +584,9 @@ def test_sequential_hook_with_align_devices_hook_is_supported(offload):
kwargs = {"execution_device": "cpu", "offload": offload}
if offload:
prefix = "model.embed_tokens."
- kwargs["weights_map"] = PrefixedDataset({prefix + "weight": emb.weight.detach().clone()}, prefix)
+ kwargs["weights_map"] = PrefixedDataset(
+ {prefix + "weight": emb.weight.detach().clone()}, prefix
+ )
accelerate_hooks.add_hook_to_module(emb, accelerate_hooks.AlignDevicesHook(**kwargs))
accelerate_hooks.add_hook_to_module(emb, accelerate_hooks.ModelHook(), append = True)
assert type(emb._hf_hook).__name__ == "SequentialHook"
@@ -620,7 +636,10 @@ def test_plain_embedding_keeps_its_accelerate_hook(offload):
emb = model.get_input_embeddings()
assert type(emb) is torch.nn.Embedding
accelerate_hooks.add_hook_to_module(
- emb, accelerate_hooks.AlignDevicesHook(execution_device = "cpu", offload = offload, io_same_device = True)
+ emb,
+ accelerate_hooks.AlignDevicesHook(
+ execution_device = "cpu", offload = offload, io_same_device = True
+ ),
)
model = _prepare_model_for_qat(model, "int8-int4")
emb = model.get_input_embeddings()
diff --git a/tests/test_remote_code_class_and_expert_submodules.py b/tests/test_remote_code_class_and_expert_submodules.py
index c2fcaa331b6..13ca047d58a 100644
--- a/tests/test_remote_code_class_and_expert_submodules.py
+++ b/tests/test_remote_code_class_and_expert_submodules.py
@@ -920,7 +920,9 @@ def test_requested_router_sibling_is_kept_when_attention_is_also_listed(leaves):
widened, _, _ = U.widen_target_regex_to_expert_submodules(
model, regex, unrequested, auto_regex = True, explicit_leaves = True
)
- assert not any(n.endswith(".block_sparse_moe.gate") and re.fullmatch(widened, n) for n in names)
+ assert not any(
+ n.endswith(".block_sparse_moe.gate") and re.fullmatch(widened, n) for n in names
+ )
def test_requested_router_sibling_peft_injects_real_hf_mixtral():
diff --git a/tests/test_triton_variant_budget.py b/tests/test_triton_variant_budget.py
index 3e5ffd75dad..87803204426 100644
--- a/tests/test_triton_variant_budget.py
+++ b/tests/test_triton_variant_budget.py
@@ -48,7 +48,6 @@ def _reset(*kernels):
for v in kernel.cache.values():
v.clear()
from unsloth.kernels import triton_launch
-
triton_launch._CACHE.clear()
@@ -61,4 +60,3 @@ def test_rms_layernorm_backward_one_variant_across_token_counts():
fast_rms_layernorm(norm, X).sum().backward()
torch.cuda.synchronize()
assert [_variants(k) for k in kernels] == [1, 1]
-
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 661604a4e9f..60e863ec2d6 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -152,10 +152,16 @@ def _autocast_dtype_mismatch(X):
# in X's dtype (in-place ops are not autocast), so a different autocast dtype cannot be fused.
# Mode flags only, no tensor work. Eager first asks whether any autocast is on (a private query, so not
# under Dynamo, which traces the public calls below as constants).
- if _is_any_autocast_enabled is not None and not _is_compiling() and not _is_any_autocast_enabled():
+ if (
+ _is_any_autocast_enabled is not None
+ and not _is_compiling()
+ and not _is_any_autocast_enabled()
+ ):
return False
device_type = "cuda" if X.is_cuda else X.device.type
- return torch.is_autocast_enabled(device_type) and torch.get_autocast_dtype(device_type) != X.dtype
+ return (
+ torch.is_autocast_enabled(device_type) and torch.get_autocast_dtype(device_type) != X.dtype
+ )
def _is_plain_lora(proj):
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 26460879eb9..2725793758a 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -534,7 +534,11 @@ def _small_m_config(M, N, K, device):
def can_use_fp8_block_small_m(X, weight, weight_scale, block_size):
# Eager only: under torch.compile the M bound below would guard on the symbolic M and recompile.
- if not _FP8_BLOCK_SMALL_M or fp8_block_matmul is not w8a8_block_fp8_matmul_triton or torch.compiler.is_compiling():
+ if (
+ not _FP8_BLOCK_SMALL_M
+ or fp8_block_matmul is not w8a8_block_fp8_matmul_triton
+ or torch.compiler.is_compiling()
+ ):
return False
if not X.is_cuda or X.dtype not in (torch.bfloat16, torch.float16):
return False
@@ -546,7 +550,9 @@ def can_use_fp8_block_small_m(X, weight, weight_scale, block_size):
if not (1 <= M <= _SMALL_M_MAX) or weight.shape[1] != K or K % 128 or N % 16:
return False
# Checkpoints store weight_scale_inv in bf16 too; the kernel promotes it exactly as the w8a8 one does.
- if weight_scale.dtype not in (torch.float32, torch.bfloat16, torch.float16) or tuple(weight_scale.shape) != (triton.cdiv(N, 128), K // 128):
+ if weight_scale.dtype not in (torch.float32, torch.bfloat16, torch.float16) or tuple(
+ weight_scale.shape
+ ) != (triton.cdiv(N, 128), K // 128):
return False
return not long_indexing(X, weight)
@@ -742,7 +748,12 @@ def _fp8_rowwise_gemv_kernel(
# Batched decode (2..16 rows): each weight tile is read once for every row, no 16-bit weight copy.
# UNSLOTH_FP8_GEMV_ROWS=0 restores the dequantize path.
_FP8_GEMV_MAX_ROWS = 16
-_FP8_GEMV_ROWS = os.environ.get("UNSLOTH_FP8_GEMV_ROWS", "1").strip().lower() not in ("0", "false", "no", "off")
+_FP8_GEMV_ROWS = os.environ.get("UNSLOTH_FP8_GEMV_ROWS", "1").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+)
@triton.jit(do_not_specialize = ["M"])
@@ -789,14 +800,18 @@ def _fp8_rowwise_gemv_rows_kernel(
if SPLIT_K == 1:
s = tl.load(s_ptr + offs_n, mask = offs_n < N, other = 0.0).to(tl.float32)
y = acc * s[None, :]
- tl.store(y_ptr + offs_m[:, None] * N + offs_n[None, :], y.to(y_ptr.dtype.element_ty), mask = mask)
+ tl.store(
+ y_ptr + offs_m[:, None] * N + offs_n[None, :], y.to(y_ptr.dtype.element_ty), mask = mask
+ )
else:
# fp32 partials, summed in a fixed order by the reduce kernel: deterministic, no atomics.
tl.store(p_ptr + (pid_k * M + offs_m[:, None]) * N + offs_n[None, :], acc, mask = mask)
@triton.jit(do_not_specialize = ["M"])
-def _fp8_rowwise_gemv_rows_reduce(p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr):
+def _fp8_rowwise_gemv_rows_reduce(
+ p_ptr, s_ptr, y_ptr, M, N, SPLIT_K: tl.constexpr, BLOCK: tl.constexpr
+):
offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offs < M * N
acc = tl.zeros((BLOCK,), dtype = tl.float32)
@@ -825,7 +840,9 @@ def _fp8_rowwise_gemv_rows(X, weight, scale, out, rows):
N, K = weight.shape
block_n, block_k, split_k, warps, stages = _fp8_rows_config(N, K, X.dtype)
partials = (
- torch.empty((split_k, rows, N), device = X.device, dtype = torch.float32) if split_k > 1 else out
+ torch.empty((split_k, rows, N), device = X.device, dtype = torch.float32)
+ if split_k > 1
+ else out
)
with _fp8_triton_device_context(X, static_device_count = True):
_fp8_rowwise_gemv_rows_kernel[(triton.cdiv(N, block_n), split_k)](
@@ -998,7 +1015,12 @@ def _fp8_block_size(weight, weight_scale):
def _fp8_block_gemv_weight_ok(weight, weight_scale):
# Cached per (weight, scale): 0 = never, else max decode rows.
- if not weight.is_cuda or weight.dtype != torch.float8_e4m3fn or weight.dim() != 2 or not weight.is_contiguous():
+ if (
+ not weight.is_cuda
+ or weight.dtype != torch.float8_e4m3fn
+ or weight.dim() != 2
+ or not weight.is_contiguous()
+ ):
return 0
N, K = weight.shape
# Only where never slower than the block GEMM (L4, RTX PRO 6000; L2 flushed and hot). Small parts cap
@@ -1068,7 +1090,9 @@ def _sm_count(device):
@_opaque_under_compile(
"fp8_block_gemv",
- lambda X, weight, weight_scale, group_n, group_k: X.new_empty(X.shape[:-1] + (weight.shape[0],)),
+ lambda X, weight, weight_scale, group_n, group_k: X.new_empty(
+ X.shape[:-1] + (weight.shape[0],)
+ ),
)
def _fp8_block_gemv(
X: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, group_n: int, group_k: int
@@ -1141,7 +1165,9 @@ def _fp8_block_small_m_first(X, weight, weight_scale):
# 100+ SM parts: the fused small-M kernel is faster, so try it first (eager only; compiled uses the GEMV).
if not (torch.is_grad_enabled() and X.requires_grad):
if (
- can_use_fp8_block_small_m(X, weight, weight_scale, _fp8_block_size(weight, weight_scale))
+ can_use_fp8_block_small_m(
+ X, weight, weight_scale, _fp8_block_size(weight, weight_scale)
+ )
and weight.is_contiguous()
and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)
):
diff --git a/unsloth/kernels/moe/grouped_gemm/interface.py b/unsloth/kernels/moe/grouped_gemm/interface.py
index d7291a68168..0ca2d5291a1 100644
--- a/unsloth/kernels/moe/grouped_gemm/interface.py
+++ b/unsloth/kernels/moe/grouped_gemm/interface.py
@@ -107,11 +107,12 @@ _OPERAND_TILE_FITS = {}
def _autotune_cache():
from .. import autotune_cache
-
return autotune_cache
-def _fit_tiles_to_operand_device(kind, operand, weight_dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages):
+def _fit_tiles_to_operand_device(
+ kind, operand, weight_dtype, BLOCK_SIZE_M, BLOCK_SIZE_N, BLOCK_SIZE_K, num_stages
+):
"""Caller-supplied tiles were fitted to the current card's shared memory when the caller passed no device
(unsloth_zoo <= 2026.10.7). If the operands sit on a different GPU model, shrink a copy to fit their card.
One GPU, or the same model as the current card: returned unchanged."""
@@ -122,7 +123,9 @@ def _fit_tiles_to_operand_device(kind, operand, weight_dtype, BLOCK_SIZE_M, BLOC
return tiles
current = torch.cuda.current_device()
index = _device_index(device)
- if index == current or _device_model(device) == _device_model(torch.device("cuda", current)):
+ if index == current or _device_model(device) == _device_model(
+ torch.device("cuda", current)
+ ):
return tiles
except Exception:
return tiles
@@ -151,7 +154,12 @@ def _fit_tiles_to_operand_device(kind, operand, weight_dtype, BLOCK_SIZE_M, BLOC
tuple(torch.cuda.get_device_capability(device)),
smem,
)
- fitted = (config.BLOCK_SIZE_M, config.BLOCK_SIZE_N, config.BLOCK_SIZE_K, config.num_stages)
+ fitted = (
+ config.BLOCK_SIZE_M,
+ config.BLOCK_SIZE_N,
+ config.BLOCK_SIZE_K,
+ config.num_stages,
+ )
except Exception:
fitted = tiles
_OPERAND_TILE_FITS[key] = fitted
diff --git a/unsloth/models/_routing_replay.py b/unsloth/models/_routing_replay.py
index 03574176813..1d4030ffbc2 100644
--- a/unsloth/models/_routing_replay.py
+++ b/unsloth/models/_routing_replay.py
@@ -103,7 +103,13 @@ def _router_extra_args(module, parent):
return extra
-def _prime_formula(rr, module, hook, layers, parent = None):
+def _prime_formula(
+ rr,
+ module,
+ hook,
+ layers,
+ parent = None,
+):
"""Run the zoo hook once, eagerly, on a synthetic router call whose replay is the natural
routing: it picks and caches module._unsloth_replay_formula exactly as the first real call
would. Returns None when primed, else why it could not be."""
@@ -177,7 +183,11 @@ def _check_formula(rr, module, x, extra):
li, wi, ii, dense = positions
logits = output[li].reshape(-1, output[li].shape[-1])
ids = output[ii].reshape(-1, output[ii].shape[-1]).long()
- weights = output[wi].reshape(logits.shape).gather(-1, ids) if dense else output[wi].reshape(ids.shape)
+ weights = (
+ output[wi].reshape(logits.shape).gather(-1, ids)
+ if dense
+ else output[wi].reshape(ids.shape)
+ )
candidate = fn(logits, ids).to(weights.dtype).float()
if not torch.allclose(candidate, weights.float(), rtol = 2e-2, atol = 2e-3):
return f"the {formula} weight formula does not reproduce its routing weights"
@@ -203,7 +213,9 @@ def prepare_routing_replay(model, before):
import unsloth_zoo.routing_replay as rr
except Exception:
rr = None
- if rr is not None and all(hasattr(rr, n) for n in ("_STATE", "_is_router", "_split_router_output", "_LAYER_INDEX_RE")):
+ if rr is not None and all(
+ hasattr(rr, n) for n in ("_STATE", "_is_router", "_split_router_output", "_LAYER_INDEX_RE")
+ ):
layer_of, parent_of = {}, {}
for name, module in model.named_modules():
match = rr._LAYER_INDEX_RE.search(f"{name}.")
@@ -212,7 +224,8 @@ def prepare_routing_replay(model, before):
for child in module.children():
parent_of[id(child)] = module
routers = [
- entry for entry in hooks
+ entry
+ for entry in hooks
if entry[1] == "_forward_hooks" and rr._is_router(entry[0]) and id(entry[0]) in layer_of
]
# One replay row per decoder layer index, every row the natural routing.
@@ -220,7 +233,9 @@ def prepare_routing_replay(model, before):
failed = []
for entry in routers:
module, name, extras, hook_id = entry
- why = _prime_formula(rr, module, module._forward_hooks[hook_id], layers, parent_of.get(id(module)))
+ why = _prime_formula(
+ rr, module, module._forward_hooks[hook_id], layers, parent_of.get(id(module))
+ )
if why is not None:
_remove_hook(*entry)
failed.append((type(module).__name__, why))
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b232a2d999d..af3000a45dd 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -7642,7 +7642,6 @@ def _is_accelerate_forward(module: torch.nn.Module, forward) -> bool:
def _is_class_forward(module: torch.nn.Module, forward) -> bool:
import types
-
return (
isinstance(forward, types.MethodType)
and forward.__self__ is module
@@ -7653,18 +7652,27 @@ def _is_class_forward(module: torch.nn.Module, forward) -> bool:
def _is_require_grads_hook(hook) -> bool:
"""transformers' enable_input_require_grads hook (make_inputs_require_grads, handles kept on the model) or
PEFT's fallback (make_inputs_require_grad, no handle kept). Both are re-registered after the QAT swap."""
- return getattr(hook, "__name__", None) in ("make_inputs_require_grads", "make_inputs_require_grad")
+ return getattr(hook, "__name__", None) in (
+ "make_inputs_require_grads",
+ "make_inputs_require_grad",
+ )
def _unowned_require_grads_hooks(originals: Dict[str, torch.nn.Module]):
"""PEFT's make_inputs_require_grad hooks per embedding: nobody holds their handle, so they can be re-registered."""
return {
- fqn: [h for h in module._forward_hooks.values() if getattr(h, "__name__", None) == "make_inputs_require_grad"]
+ fqn: [
+ h
+ for h in module._forward_hooks.values()
+ if getattr(h, "__name__", None) == "make_inputs_require_grad"
+ ]
for fqn, module in originals.items()
}
-def _reregister_unowned_require_grads_hooks(model: torch.nn.Module, originals, hooks_by_fqn) -> None:
+def _reregister_unowned_require_grads_hooks(
+ model: torch.nn.Module, originals, hooks_by_fqn
+) -> None:
for fqn, hooks in hooks_by_fqn.items():
if not hooks:
continue
@@ -7714,7 +7722,9 @@ def _flat_accelerate_hooks(hook):
def _offload_hook(module: torch.nn.Module):
"""The offloading AlignDevicesHook on module, also inside a SequentialHook, else None."""
hooks = [
- h for h in _flat_accelerate_hooks(getattr(module, "_hf_hook", None)) if getattr(h, "offload", False)
+ h
+ for h in _flat_accelerate_hooks(getattr(module, "_hf_hook", None))
+ if getattr(h, "offload", False)
]
return hooks[0] if hooks else None
@@ -7770,7 +7780,12 @@ def _exclude_unsupported_qat_embeddings(model: torch.nn.Module, torchao_config)
if not excluded:
return
torchao_config.base_config_and_filter_fns = [
- (c, f if f is None or isinstance(f, _SkipExcludedEmbeddings) else _SkipExcludedEmbeddings(f))
+ (
+ c,
+ f
+ if f is None or isinstance(f, _SkipExcludedEmbeddings)
+ else _SkipExcludedEmbeddings(f),
+ )
for c, f in configs
]
warnings.warn(
@@ -7836,7 +7851,9 @@ def _offload_hook_problem(module: torch.nn.Module):
return None
-def _check_offloaded_embeddings(originals: Dict[str, torch.nn.Module], base_config_and_filter_fns) -> None:
+def _check_offloaded_embeddings(
+ originals: Dict[str, torch.nn.Module], base_config_and_filter_fns
+) -> None:
"""Refuse, before QAT replaces anything, an offloaded embedding selected for QAT whose real weight could not
follow it. A filter of None selects Linears only, so it never selects an embedding."""
filters = [f for _, f in base_config_and_filter_fns if f is not None]
@@ -7870,7 +7887,6 @@ def _move_dispatch_hook(dst: torch.nn.Module, src: torch.nn.Module) -> None:
original_devices = dict(getattr(offload_hook, "original_devices", {}) or {})
try:
from accelerate.hooks import add_hook_to_module
-
add_hook_to_module(dst, hook)
if offload_hook is not None and original_devices:
offload_hook.original_devices = original_devices
@@ -7961,7 +7977,9 @@ def _restore_fake_quantized_embedding_subclasses(
if type(module) is not FakeQuantizedEmbedding:
continue
if type(original) is torch.nn.Embedding:
- _move_dispatch_hook(module, original) # torchao's own class already fake quantizes the lookup
+ _move_dispatch_hook(
+ module, original
+ ) # torchao's own class already fake quantizes the lookup
continue
module.__class__ = _fake_quantized_embedding_class(type(original))
_copy_embedding_subclass_state(module, original)
@@ -10689,11 +10707,19 @@ def widen_target_regex_to_expert_submodules(
# Requested expert siblings (Mixtral's router gate), as PEFT suffix matching would.
for name, module in model.named_modules():
parent, _, leaf = name.rpartition(".")
- if leaf in requested and any(re.fullmatch(p, parent) for p in native) and (
- isinstance(module, torch.nn.Linear)
- or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ if (
+ leaf in requested
+ and any(re.fullmatch(p, parent) for p in native)
+ and (
+ isinstance(module, torch.nn.Linear)
+ or isinstance(getattr(module, "base_layer", None), torch.nn.Linear)
+ )
):
- siblings.add(r"\.".join(r"\d+" if part.isdigit() else re.escape(part) for part in name.split(".")))
+ siblings.add(
+ r"\.".join(
+ r"\d+" if part.isdigit() else re.escape(part) for part in name.split(".")
+ )
+ )
if not parents:
return target_modules, detect_targets, []
# Requested names used as-is too, as plain PEFT does (Mixtral's w1 / w2 / w3).
diff --git a/unsloth/models/dpo.py b/unsloth/models/dpo.py
index 93a9ee35e62..4be13424001 100644
--- a/unsloth/models/dpo.py
+++ b/unsloth/models/dpo.py
@@ -84,7 +84,6 @@ def _uses_fsdp1(trainer, model):
def _global_module_backward_hooks():
# torch.nn.modules.module.register_module_full_backward_(pre_)hook applies to every module.
from torch.nn.modules import module as _module
-
return bool(
getattr(_module, "_global_backward_hooks", None)
or getattr(_module, "_global_backward_pre_hooks", None)
diff --git a/unsloth/models/fsdp2_save.py b/unsloth/models/fsdp2_save.py
index fe187aa11a0..3556b0a57b6 100644
--- a/unsloth/models/fsdp2_save.py
+++ b/unsloth/models/fsdp2_save.py
@@ -527,11 +527,13 @@ def _restore_quant_state(model, state_dict, options):
if broadcast:
# An object broadcast, not a tensor one: a CPU tensor cannot go through an NCCL group, and
# broadcast_object_list stages on whatever device the group's backend needs.
- has = [any(
- k == prefix + "weight" or k.startswith(prefix + "weight.")
- for prefix, _ in entries
- for k in state_dict
- )]
+ has = [
+ any(
+ k == prefix + "weight" or k.startswith(prefix + "weight.")
+ for prefix, _ in entries
+ for k in state_dict
+ )
+ ]
dist.broadcast_object_list(has, src = 0)
if not has[0]:
return
diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.py
index 7c6b76d8b6e..78947b0d14c 100644
--- a/unsloth/models/mxfp4_compressed_linear.py
+++ b/unsloth/models/mxfp4_compressed_linear.py
@@ -125,17 +125,10 @@ def mxfp4_packed_matmul(
for cuBLAS only as far as memory allows (row chunks, else the tile kernel); the rest decode it once for cuBLAS."""
rows = x.numel() // x.shape[-1] if x.shape[-1] else 0
# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.
- if (
- not transpose
- and _zoo_mxfp4_matmul is not None
- and rows <= _FUSED_MAX_ROWS
- and x.is_cuda
- ):
+ if not transpose and _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:
if mxfp4_gemm_available(x.device, x.dtype):
return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)
- if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(
- x.device, x.dtype
- ):
+ if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(x.device, x.dtype):
# Below sm_80 decode + cuBLAS beats the tile kernel, but only the tile kernel never builds the
# 16-bit weight: decode in chunks of output rows when the weight does not fit.
return _bounded_dense_matmul(x, packed, scale, out, bias)
@@ -172,7 +165,13 @@ def _free_bytes(device):
return max(0, free) + max(0, reserved - torch.cuda.memory_allocated(device))
-def _bounded_dense_matmul_eager(x, packed, scale, out = None, bias = None):
+def _bounded_dense_matmul_eager(
+ x,
+ packed,
+ scale,
+ out = None,
+ bias = None,
+):
"""``x @ W^T (+ bias)`` without ever holding more of the 16-bit W than fits: the whole W when it allocates
(one decode + one cuBLAS call), else row chunks of W written to ``y[..., r0:r1]`` (no cross-chunk sums),
else the tile kernel."""
@@ -237,10 +236,7 @@ try:
@torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul", mutates_args = ())
def _bounded_dense_matmul_op(
- x: torch.Tensor,
- packed: torch.Tensor,
- scale: torch.Tensor,
- bias: Optional[torch.Tensor],
+ x: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, bias: Optional[torch.Tensor]
) -> torch.Tensor:
return _bounded_dense_matmul_eager(x, packed, scale, None, bias)
diff --git a/unsloth/save.py b/unsloth/save.py
index 5f9d92761cd..5cda23858bc 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -6416,7 +6416,11 @@ def _fsdp2_peft_saved_frozen(model, arguments):
return include
-def _fsdp2_collective_save(trainable_only, offload_to_cpu = None, offload_4bit = False):
+def _fsdp2_collective_save(
+ trainable_only,
+ offload_to_cpu = None,
+ offload_4bit = False,
+):
"""Under FSDP2 every rank calls the save: the shards are gathered, rank 0 writes, the others
return None. `trainable_only(self, args, kwargs)` says whether the adapter alone is enough;
`offload_to_cpu(self, args, kwargs)` (default True) whether gathered weights go to the CPU.