Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Skip bitsandbytes' per-parameter device sync in non-paged optimizer steps (#13389)
Daniel Haned8e5e7f1a65
diff --git a/tests/test_bnb_optimizer_step_sync.py b/tests/test_bnb_optimizer_step_sync.pynew file mode 100644index 00000000000..1b6e7e4c297--- /dev/null+++ b/tests/test_bnb_optimizer_step_sync.py@@ -0,0 +1,184 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""GPU cases run in a subprocess: the patch rebinds bitsandbytes' Optimizer8bit.step and sync_gpu."""++import importlib.util+import json+import os+import subprocess+import sys+import textwrap+import types++import pytest++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_FIXES = os.path.join(_ROOT, "unsloth", "import_fixes.py")++_RUNNER = textwrap.dedent(+ """+ import importlib.util, json, sys+ import torch, bitsandbytes as bnb+ patched, optim_name, side_stream = sys.argv[1] == "patched", sys.argv[2], sys.argv[4] == "stream"+ if patched:+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[3])+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ fixes.patch_bitsandbytes_optimizer_step_sync()+ fixes.patch_bitsandbytes_optimizer_step_sync() # idempotent+ syncs = [0]+ real_sync = torch.cuda.synchronize+ def counting_sync(*a, **k):+ syncs[0] += 1+ return real_sync(*a, **k)+ torch.cuda.synchronize = counting_sync+ torch.manual_seed(0)+ shapes = [(2048, 1024), (16, 2048), (1024,), (64, 2)]+ params = [torch.nn.Parameter(torch.randn(s, device = "cuda")) for s in shapes]+ opt = getattr(bnb.optim, optim_name)(params, lr = 1e-2)+ per_step = []+ stream = torch.cuda.Stream() if side_stream else torch.cuda.current_stream()+ for i in range(10):+ g = torch.Generator(device = "cuda").manual_seed(i)+ for p in params:+ p.grad = torch.randn(p.shape, device = "cuda", generator = g)+ before = syncs[0]+ # Step 0 allocates optimizer state; stock bnb orders that against its kernels by stream, not sync, so+ # creating it on a side stream races in stock too. Later steps are the ones under test.+ with torch.cuda.stream(stream if i else torch.cuda.default_stream()):+ stream.wait_stream(torch.cuda.default_stream())+ opt.step()+ torch.cuda.default_stream().wait_stream(stream)+ per_step.append(syncs[0] - before)+ real_sync()+ out = [p.detach().float().cpu().flatten().tolist() for p in params]+ print("RESULT " + json.dumps({"params": out, "syncs": per_step, "is_paged": bool(opt.is_paged),+ "wrapped": getattr(type(opt).step, "_unsloth_no_sync", False)}))+ """+)+++def _run(+ mode,+ optim_name,+ stream = "default",+):+ proc = subprocess.run(+ [sys.executable, "-c", _RUNNER, mode, optim_name, _FIXES, stream],+ capture_output = True,+ text = True,+ timeout = 600,+ )+ line = next((l for l in proc.stdout.splitlines() if l.startswith("RESULT ")), None)+ assert line is not None, proc.stdout[-2000:] + proc.stderr[-4000:]+ return json.loads(line[len("RESULT ") :])+++def _gpu_ready():+ try:+ import bitsandbytes # noqa: F401+ import torch+ except Exception:+ return False+ return torch.cuda.is_available()++[email protected](not _gpu_ready(), reason = "bitsandbytes optimizers need a CUDA device")+ "optim_name", ["AdamW8bit", "Adam8bit", "Lion8bit", "AdamW", "PagedAdamW8bit"]+)+def test_step_matches_stock_and_drops_only_non_paged_syncs(optim_name):+ stock = _run("stock", optim_name)+ patched = _run("patched", optim_name)+ if not patched["wrapped"]:+ pytest.skip("this bitsandbytes inlines torch.cuda.synchronize; the patch leaves it alone")+ assert patched["params"] == stock["params"]+ if stock["is_paged"]:+ assert patched["syncs"] == stock["syncs"]+ else:+ assert min(stock["syncs"]) >= 4+ assert patched["syncs"] == [0] * 10++[email protected](not _gpu_ready(), reason = "bitsandbytes optimizers need a CUDA device")+def test_step_on_a_side_stream_keeps_every_sync():+ # bitsandbytes launches on the legacy default stream, so off it the per-parameter sync is the only ordering.+ stock = _run("stock", "AdamW8bit", "stream")+ patched = _run("patched", "AdamW8bit", "stream")+ if not patched["wrapped"]:+ pytest.skip("this bitsandbytes inlines torch.cuda.synchronize; the patch leaves it alone")+ assert patched["syncs"][1:] == stock["syncs"][1:] and min(stock["syncs"][1:]) >= 4+ assert patched["params"] == stock["params"]+++def _load_fixes():+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_step_sync", _FIXES)+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ return fixes+++def _fake_bnb(monkeypatch, tmp_path, source):+ path = tmp_path / f"fake_optimizer_{abs(hash(source))}.py"+ path.write_text(textwrap.dedent(source))+ spec = importlib.util.spec_from_file_location("bitsandbytes.optim.optimizer", path)+ optimizer = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(optimizer)+ package, optim = types.ModuleType("bitsandbytes"), types.ModuleType("bitsandbytes.optim")+ package.optim, optim.optimizer = optim, optimizer+ for name, module in (+ ("bitsandbytes", package),+ ("bitsandbytes.optim", optim),+ ("bitsandbytes.optim.optimizer", optimizer),+ ):+ monkeypatch.setitem(sys.modules, name, module)+ return optimizer+++_NEW_STEP = """+def sync_gpu(t):+ pass++class Optimizer8bit:+ def step(self, closure=None):+ for p in self.params:+ sync_gpu(p)+"""++_OLD_STEP = """+class Optimizer8bit:+ def step(self, closure=None):+ for p in self.params:+ torch.cuda.synchronize()+"""+++def test_patch_is_idempotent_and_skips_a_bitsandbytes_without_sync_gpu(monkeypatch, tmp_path):+ fixes = _load_fixes()+ optimizer = _fake_bnb(monkeypatch, tmp_path, _NEW_STEP)+ stock = optimizer.Optimizer8bit.step+ fixes.patch_bitsandbytes_optimizer_step_sync()+ patched = optimizer.Optimizer8bit.step+ assert patched._unsloth_no_sync and patched.__wrapped__ is stock+ fixes.patch_bitsandbytes_optimizer_step_sync()+ assert optimizer.Optimizer8bit.step is patched++ # An older bitsandbytes (inline torch.cuda.synchronize, no sync_gpu helper) is left untouched.+ old = _fake_bnb(monkeypatch, tmp_path, _OLD_STEP)+ before = old.Optimizer8bit.step+ fixes.patch_bitsandbytes_optimizer_step_sync()+ assert old.Optimizer8bit.step is beforediff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 37537bbcafa..a3ca27973a8 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -730,6 +730,10 @@ from .import_fixes import patch_bitsandbytes_paged_optimizer_resumepatch_bitsandbytes_paged_optimizer_resume()del patch_bitsandbytes_paged_optimizer_resume+from .import_fixes import patch_bitsandbytes_optimizer_step_sync++patch_bitsandbytes_optimizer_step_sync()+del patch_bitsandbytes_optimizer_step_syncfrom .models import *from .models import __version__diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 62be4fb7ed9..fa4e7ee91b4 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -5219,6 +5219,58 @@ def patch_bitsandbytes_paged_optimizer_resume():Optimizer8bit.load_state_dict = _unsloth_load_state_dict+def patch_bitsandbytes_optimizer_step_sync():+ """Skip Optimizer8bit.step's per-parameter sync_gpu for non-paged optimizers on the default stream.+ Paged state needs it to order prefetch (bnb 41a9c708); non-paged it only stalls the host, output bit-identical."""+ if "bitsandbytes" not in sys.modules:+ return+ try:+ import bitsandbytes.optim.optimizer as bnb_optimizer+ except Exception:+ return+ Optimizer8bit = getattr(bnb_optimizer, "Optimizer8bit", None)+ step = getattr(Optimizer8bit, "step", None)+ sync_gpu = getattr(bnb_optimizer, "sync_gpu", None)+ if step is None or sync_gpu is None or getattr(step, "_unsloth_no_sync", False):+ return+ try:+ if "sync_gpu(p)" not in inspect.getsource(step):+ return+ except (OSError, TypeError):+ return+ import contextvars++ skip = contextvars.ContextVar("unsloth_bnb_skip_sync", default = False)++ @functools.wraps(sync_gpu)+ def _unsloth_sync_gpu(t):+ if skip.get():+ return None+ return sync_gpu(t)++ import torch++ def _on_default_streams(optimizer):+ # bnb kernels run on the legacy default stream; off it the sync is the only ordering.+ devices = {p.device for group in optimizer.param_groups for p in group["params"]}+ return bool(devices) and all(+ d.type == "cuda" and torch.cuda.current_stream(d) == torch.cuda.default_stream(d)+ for d in devices+ )++ @functools.wraps(step)+ def _unsloth_step(self, *args, **kwargs):+ token = skip.set(not getattr(self, "is_paged", False) and _on_default_streams(self))+ try:+ return step(self, *args, **kwargs)+ finally:+ skip.reset(token)++ _unsloth_step._unsloth_no_sync = True+ bnb_optimizer.sync_gpu = _unsloth_sync_gpu+ Optimizer8bit.step = _unsloth_step++def _raised_in_triton_host_build(exc):"""Triton builds each kernel's launcher with the host C compiler in triton/runtime/build.py."""seen = set()