Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Skip bitsandbytes' per-parameter device sync in non-paged optimizer steps (#13389)

Daniel Haned8e5e7f1a65
diff --git a/tests/test_bnb_optimizer_step_sync.py b/tests/test_bnb_optimizer_step_sync.py
new file mode 100644
index 00000000000..1b6e7e4c297
--- /dev/null
+++ b/tests/test_bnb_optimizer_step_sync.py
@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""GPU cases run in a subprocess: the patch rebinds bitsandbytes' Optimizer8bit.step and sync_gpu."""
+
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+import textwrap
+import types
+
+import pytest
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_FIXES = os.path.join(_ROOT, "unsloth", "import_fixes.py")
+
+_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, json, sys
+ import torch, bitsandbytes as bnb
+ patched, optim_name, side_stream = sys.argv[1] == "patched", sys.argv[2], sys.argv[4] == "stream"
+ if patched:
+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_under_test", sys.argv[3])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes.patch_bitsandbytes_optimizer_step_sync()
+ fixes.patch_bitsandbytes_optimizer_step_sync() # idempotent
+ syncs = [0]
+ real_sync = torch.cuda.synchronize
+ def counting_sync(*a, **k):
+ syncs[0] += 1
+ return real_sync(*a, **k)
+ torch.cuda.synchronize = counting_sync
+ torch.manual_seed(0)
+ shapes = [(2048, 1024), (16, 2048), (1024,), (64, 2)]
+ params = [torch.nn.Parameter(torch.randn(s, device = "cuda")) for s in shapes]
+ opt = getattr(bnb.optim, optim_name)(params, lr = 1e-2)
+ per_step = []
+ stream = torch.cuda.Stream() if side_stream else torch.cuda.current_stream()
+ for i in range(10):
+ g = torch.Generator(device = "cuda").manual_seed(i)
+ for p in params:
+ p.grad = torch.randn(p.shape, device = "cuda", generator = g)
+ before = syncs[0]
+ # Step 0 allocates optimizer state; stock bnb orders that against its kernels by stream, not sync, so
+ # creating it on a side stream races in stock too. Later steps are the ones under test.
+ with torch.cuda.stream(stream if i else torch.cuda.default_stream()):
+ stream.wait_stream(torch.cuda.default_stream())
+ opt.step()
+ torch.cuda.default_stream().wait_stream(stream)
+ per_step.append(syncs[0] - before)
+ real_sync()
+ out = [p.detach().float().cpu().flatten().tolist() for p in params]
+ print("RESULT " + json.dumps({"params": out, "syncs": per_step, "is_paged": bool(opt.is_paged),
+ "wrapped": getattr(type(opt).step, "_unsloth_no_sync", False)}))
+ """
+)
+
+
+def _run(
+ mode,
+ optim_name,
+ stream = "default",
+):
+ proc = subprocess.run(
+ [sys.executable, "-c", _RUNNER, mode, optim_name, _FIXES, stream],
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ line = next((l for l in proc.stdout.splitlines() if l.startswith("RESULT ")), None)
+ assert line is not None, proc.stdout[-2000:] + proc.stderr[-4000:]
+ return json.loads(line[len("RESULT ") :])
+
+
+def _gpu_ready():
+ try:
+ import bitsandbytes # noqa: F401
+ import torch
+ except Exception:
+ return False
+ return torch.cuda.is_available()
+
+
[email protected](not _gpu_ready(), reason = "bitsandbytes optimizers need a CUDA device")
+ "optim_name", ["AdamW8bit", "Adam8bit", "Lion8bit", "AdamW", "PagedAdamW8bit"]
+)
+def test_step_matches_stock_and_drops_only_non_paged_syncs(optim_name):
+ stock = _run("stock", optim_name)
+ patched = _run("patched", optim_name)
+ if not patched["wrapped"]:
+ pytest.skip("this bitsandbytes inlines torch.cuda.synchronize; the patch leaves it alone")
+ assert patched["params"] == stock["params"]
+ if stock["is_paged"]:
+ assert patched["syncs"] == stock["syncs"]
+ else:
+ assert min(stock["syncs"]) >= 4
+ assert patched["syncs"] == [0] * 10
+
+
[email protected](not _gpu_ready(), reason = "bitsandbytes optimizers need a CUDA device")
+def test_step_on_a_side_stream_keeps_every_sync():
+ # bitsandbytes launches on the legacy default stream, so off it the per-parameter sync is the only ordering.
+ stock = _run("stock", "AdamW8bit", "stream")
+ patched = _run("patched", "AdamW8bit", "stream")
+ if not patched["wrapped"]:
+ pytest.skip("this bitsandbytes inlines torch.cuda.synchronize; the patch leaves it alone")
+ assert patched["syncs"][1:] == stock["syncs"][1:] and min(stock["syncs"][1:]) >= 4
+ assert patched["params"] == stock["params"]
+
+
+def _load_fixes():
+ spec = importlib.util.spec_from_file_location("unsloth_import_fixes_step_sync", _FIXES)
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ return fixes
+
+
+def _fake_bnb(monkeypatch, tmp_path, source):
+ path = tmp_path / f"fake_optimizer_{abs(hash(source))}.py"
+ path.write_text(textwrap.dedent(source))
+ spec = importlib.util.spec_from_file_location("bitsandbytes.optim.optimizer", path)
+ optimizer = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(optimizer)
+ package, optim = types.ModuleType("bitsandbytes"), types.ModuleType("bitsandbytes.optim")
+ package.optim, optim.optimizer = optim, optimizer
+ for name, module in (
+ ("bitsandbytes", package),
+ ("bitsandbytes.optim", optim),
+ ("bitsandbytes.optim.optimizer", optimizer),
+ ):
+ monkeypatch.setitem(sys.modules, name, module)
+ return optimizer
+
+
+_NEW_STEP = """
+def sync_gpu(t):
+ pass
+
+class Optimizer8bit:
+ def step(self, closure=None):
+ for p in self.params:
+ sync_gpu(p)
+"""
+
+_OLD_STEP = """
+class Optimizer8bit:
+ def step(self, closure=None):
+ for p in self.params:
+ torch.cuda.synchronize()
+"""
+
+
+def test_patch_is_idempotent_and_skips_a_bitsandbytes_without_sync_gpu(monkeypatch, tmp_path):
+ fixes = _load_fixes()
+ optimizer = _fake_bnb(monkeypatch, tmp_path, _NEW_STEP)
+ stock = optimizer.Optimizer8bit.step
+ fixes.patch_bitsandbytes_optimizer_step_sync()
+ patched = optimizer.Optimizer8bit.step
+ assert patched._unsloth_no_sync and patched.__wrapped__ is stock
+ fixes.patch_bitsandbytes_optimizer_step_sync()
+ assert optimizer.Optimizer8bit.step is patched
+
+ # An older bitsandbytes (inline torch.cuda.synchronize, no sync_gpu helper) is left untouched.
+ old = _fake_bnb(monkeypatch, tmp_path, _OLD_STEP)
+ before = old.Optimizer8bit.step
+ fixes.patch_bitsandbytes_optimizer_step_sync()
+ assert old.Optimizer8bit.step is before
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 37537bbcafa..a3ca27973a8 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -730,6 +730,10 @@ from .import_fixes import patch_bitsandbytes_paged_optimizer_resume
patch_bitsandbytes_paged_optimizer_resume()
del patch_bitsandbytes_paged_optimizer_resume
+from .import_fixes import patch_bitsandbytes_optimizer_step_sync
+
+patch_bitsandbytes_optimizer_step_sync()
+del patch_bitsandbytes_optimizer_step_sync
from .models import *
from .models import __version__
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 62be4fb7ed9..fa4e7ee91b4 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -5219,6 +5219,58 @@ def patch_bitsandbytes_paged_optimizer_resume():
Optimizer8bit.load_state_dict = _unsloth_load_state_dict
+def patch_bitsandbytes_optimizer_step_sync():
+ """Skip Optimizer8bit.step's per-parameter sync_gpu for non-paged optimizers on the default stream.
+ Paged state needs it to order prefetch (bnb 41a9c708); non-paged it only stalls the host, output bit-identical."""
+ if "bitsandbytes" not in sys.modules:
+ return
+ try:
+ import bitsandbytes.optim.optimizer as bnb_optimizer
+ except Exception:
+ return
+ Optimizer8bit = getattr(bnb_optimizer, "Optimizer8bit", None)
+ step = getattr(Optimizer8bit, "step", None)
+ sync_gpu = getattr(bnb_optimizer, "sync_gpu", None)
+ if step is None or sync_gpu is None or getattr(step, "_unsloth_no_sync", False):
+ return
+ try:
+ if "sync_gpu(p)" not in inspect.getsource(step):
+ return
+ except (OSError, TypeError):
+ return
+ import contextvars
+
+ skip = contextvars.ContextVar("unsloth_bnb_skip_sync", default = False)
+
+ @functools.wraps(sync_gpu)
+ def _unsloth_sync_gpu(t):
+ if skip.get():
+ return None
+ return sync_gpu(t)
+
+ import torch
+
+ def _on_default_streams(optimizer):
+ # bnb kernels run on the legacy default stream; off it the sync is the only ordering.
+ devices = {p.device for group in optimizer.param_groups for p in group["params"]}
+ return bool(devices) and all(
+ d.type == "cuda" and torch.cuda.current_stream(d) == torch.cuda.default_stream(d)
+ for d in devices
+ )
+
+ @functools.wraps(step)
+ def _unsloth_step(self, *args, **kwargs):
+ token = skip.set(not getattr(self, "is_paged", False) and _on_default_streams(self))
+ try:
+ return step(self, *args, **kwargs)
+ finally:
+ skip.reset(token)
+
+ _unsloth_step._unsloth_no_sync = True
+ bnb_optimizer.sync_gpu = _unsloth_sync_gpu
+ Optimizer8bit.step = _unsloth_step
+
+
def _raised_in_triton_host_build(exc):
"""Triton builds each kernel's launcher with the host C compiler in triton/runtime/build.py."""
seen = set()