Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Sync before re-reading a buffer stored in an Inductor reduction loop (torch < 2.14) (#15000)
Daniel Han082c52a658cb
diff --git a/tests/test_inductor_readback_barrier.py b/tests/test_inductor_readback_barrier.pynew file mode 100644index 00000000000..14f52bb0cd7--- /dev/null+++ b/tests/test_inductor_readback_barrier.py@@ -0,0 +1,222 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""fix_inductor_reduction_readback_race: backport of pytorch/pytorch#190519 for torch < 2.14."""++import importlib.util+import os+import subprocess+import sys+import textwrap+import types++import pytest+import torch+from real_accelerator import has_real_cuda++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+_FIXES = os.path.join(_ROOT, "unsloth", "import_fixes.py")+++def _load_fixes(name):+ spec = importlib.util.spec_from_file_location(name, _FIXES)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++class _Buffer:+ def __init__(self):+ self.lines = []++ def writeline(self, line):+ self.lines.append(getattr(line, "line", line))+++class _CSE:+ def __init__(self, invalidated):+ self.invalidated_stores = set(invalidated)+++def _fake_kernel_class():+ class FakeTritonKernel:+ def __init__(self, invalidated):+ self.cse = _CSE(invalidated)+ self.loads, self.body = _Buffer(), _Buffer()++ def get_load_buffer(self, indexing):+ return self.body if indexing == "hoisted" else self.loads++ def load(self, name, index):+ buffer = self.get_load_buffer(index)+ buffer.writeline(f"tl.load({name})")+ return name++ return FakeTritonKernel+++class _Graph:+ def __init__(self, device):+ self.device = torch.device(device)++ def get_current_device_or_throw(self):+ return self.device+++def patched(monkeypatch):+ from torch._inductor import config as inductor_config+ from torch._inductor.codegen import triton as triton_codegen++ fake = _fake_kernel_class()+ monkeypatch.setattr(triton_codegen, "TritonKernel", fake)+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", None)+ fixes = _load_fixes("unsloth_import_fixes_readback")+ fixes.fix_inductor_reduction_readback_race()+ return fake, fixes, inductor_config+++def _lines(+ kernel,+ device,+ name,+ index = None,+):+ from torch._inductor.virtualized import V+ with V.set_graph_handler(_Graph(device)):+ kernel.load(name, index)+++def test_barrier_precedes_reload_of_an_invalidated_store(patched):+ fake, _, _ = patched+ kernel = fake({"buf0"})+ _lines(kernel, "cuda", "buf0")+ assert kernel.loads.lines == ["tl.debug_barrier()", "tl.load(buf0)"]+++def test_barrier_goes_to_the_buffer_the_load_goes_to(patched):+ fake, _, _ = patched+ kernel = fake({"buf0"})+ _lines(kernel, "cuda", "buf0", "hoisted")+ assert kernel.body.lines == ["tl.debug_barrier()", "tl.load(buf0)"]+ assert kernel.loads.lines == []+++def test_no_barrier_for_other_buffers_or_cpu(patched):+ fake, _, _ = patched+ kernel = fake({"buf0"})+ _lines(kernel, "cuda", "arg0_1")+ _lines(kernel, "cpu", "buf0")+ assert kernel.loads.lines == ["tl.load(arg0_1)", "tl.load(buf0)"]+++def test_idempotent_and_keys_the_compile_cache(patched):+ pytest.importorskip("torch._inductor.custom_graph_pass")+ fake, fixes, inductor_config = patched+ first = fake.load+ fixes.fix_inductor_reduction_readback_race()+ assert fake.load is first+ key = inductor_config.post_grad_custom_post_pass+ from torch._inductor.custom_graph_pass import CustomGraphPass++ assert isinstance(key, CustomGraphPass)+ assert key.uuid() == "unsloth-inductor-readback-barrier-v1"+ kernel = fake({"buf0"})+ _lines(kernel, "cuda", "buf0")+ assert kernel.loads.lines.count("tl.debug_barrier()") == 1+++def test_keeps_a_user_post_grad_pass(monkeypatch):+ from torch._inductor import config as inductor_config+ from torch._inductor.codegen import triton as triton_codegen++ monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())+ user_pass = object()+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", user_pass)+ _load_fixes("unsloth_import_fixes_readback_user").fix_inductor_reduction_readback_race()+ assert inductor_config.post_grad_custom_post_pass is user_pass+++def test_skips_a_torch_that_already_has_the_barrier(monkeypatch):+ from torch._inductor.codegen import triton as triton_codegen++ fake = _fake_kernel_class()++ def load(self, name, index): # mirrors torch >= 2.14+ if name in self.cse.invalidated_stores:+ pass+ return name++ fake.load = load+ monkeypatch.setattr(triton_codegen, "TritonKernel", fake)+ _load_fixes("unsloth_import_fixes_readback_fixed").fix_inductor_reduction_readback_race()+ assert fake.load is load+++def test_torch_without_custom_graph_pass_skips_the_fx_graph_cache():+ # torch 2.4 / 2.5 config is a plain module; newer torch lets TORCHINDUCTOR_FX_GRAPH_CACHE override writes.+ config = types.SimpleNamespace(fx_graph_cache = True, post_grad_custom_post_pass = None)+ fixes = _load_fixes("unsloth_import_fixes_readback_old")+ fixes._key_inductor_caches_on_readback_barrier(config, None)+ assert config.fx_graph_cache is False+ assert config.post_grad_custom_post_pass is None+++_GPU_RUNNER = textwrap.dedent(+ """+ import importlib.util, sys, torch+ from torch._inductor.utils import run_and_get_triton_code+ if sys.argv[1] == "patched":+ spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ fixes.fix_inductor_reduction_readback_race()++ def fn(a, b, w, bias):+ y = a + b+ mean = y.mean(-1, keepdim = True)+ var = y.var(-1, keepdim = True, unbiased = False)+ return y, (y - mean) * torch.rsqrt(var + 1e-6) * w + bias++ n = 4096+ inputs = [torch.randn(64, n, device = "cuda"), torch.randn(64, n, device = "cuda"),+ torch.randn(n, device = "cuda"), torch.randn(n, device = "cuda")]+ compiled = torch.compile(fn, backend = "inductor")+ code = run_and_get_triton_code(compiled, *inputs)+ for got, want in zip(compiled(*inputs), fn(*inputs)):+ torch.testing.assert_close(got, want)+ print("BARRIER" if "tl.debug_barrier()" in code else "NO_BARRIER")+ """+)++[email protected](not has_real_cuda(), reason = "Inductor Triton codegen needs a GPU")+def test_generated_reduction_syncs_before_the_reload(tmp_path):+ env = dict(+ os.environ,+ TORCHINDUCTOR_CACHE_DIR = str(tmp_path / "inductor"),+ TORCHINDUCTOR_FX_GRAPH_CACHE = "0",+ )+ run = lambda arm: subprocess.run(+ [sys.executable, "-c", _GPU_RUNNER, arm, _FIXES],+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ )+ patched = run("patched")+ assert patched.returncode == 0, patched.stderr[-3000:]+ assert patched.stdout.strip().splitlines()[-1] == "BARRIER"diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex e1ee188fc54..ab7f2ac6d53 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -383,6 +383,7 @@ from .import_fixes import (fix_rocm_windows_fused_sdpa,fix_triton_compiled_kernel_missing_attrs,fix_dynamo_config_thread_visibility,+ fix_inductor_reduction_readback_race,patch_trunc_normal_precision_issue,ignore_logger_messages,patch_ipykernel_hf_xet,@@ -481,6 +482,7 @@ fix_triton_compiled_kernel_missing_attrs()# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.fix_dynamo_config_thread_visibility()+fix_inductor_reduction_readback_race()patch_trunc_normal_precision_issue()ignore_logger_messages()patch_ipykernel_hf_xet()@@ -545,6 +547,7 @@ del fix_cudnn_sdpa_d256_masked_backwarddel fix_rocm_windows_fused_sdpadel fix_triton_compiled_kernel_missing_attrsdel fix_dynamo_config_thread_visibility+del fix_inductor_reduction_readback_racedel patch_trunc_normal_precision_issuedel ignore_logger_messagesdel patch_ipykernel_hf_xetdiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex f4d667f655f..84d19f8b18f 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -7253,6 +7253,86 @@ def fix_triton_compiled_kernel_missing_attrs():)+def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass):+ # Warm FX / AOTAutograd caches would return barrier-free kernels: a no-op pass's uuid keys them.+ if custom_graph_pass is None:+ # torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.+ inductor_config.fx_graph_cache = False+ return++ class ReadbackBarrierCacheKey(custom_graph_pass):+ def __call__(self, graph):+ return None++ def uuid(self):+ return "unsloth-inductor-readback-barrier-v1"++ if getattr(inductor_config, "post_grad_custom_post_pass", None) is None:+ inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()+++def fix_inductor_reduction_readback_race():+ """torch < 2.14 Inductor reloads a buffer stored by an earlier loop of the same reduction kernel with no+ barrier, and Triton may give the store and the reload different warps: silently wrong gradients+ (pytorch/pytorch#198562). Backports pytorch/pytorch#190519."""+ try:+ import torch+ from torch._inductor.codegen import triton as triton_codegen+ from torch._inductor import config as inductor_config+ from torch._inductor.virtualized import V+ except Exception:+ return+ kernel = getattr(triton_codegen, "TritonKernel", None)+ deferred_line = getattr(triton_codegen, "DeferredLine", None)+ if kernel is None or deferred_line is None or not hasattr(kernel, "get_load_buffer"):+ return+ if getattr(kernel.load, "_unsloth_readback_barrier", False):+ return+ try:+ if "invalidated_stores" in inspect.getsource(kernel.load):+ return+ except Exception:+ return++ original_load = kernel.load+ original_get_load_buffer = kernel.get_load_buffer++ @functools.wraps(original_load)+ def load(self, name, index, *args, **kwargs):+ previous = self.__dict__.get("_unsloth_readback_name")+ self._unsloth_readback_name = name+ try:+ return original_load(self, name, index, *args, **kwargs)+ finally:+ self._unsloth_readback_name = previous++ @functools.wraps(original_get_load_buffer)+ def get_load_buffer(self, indexing):+ buffer = original_get_load_buffer(self, indexing)+ name = self.__dict__.get("_unsloth_readback_name")+ if name is not None and name in self.cse.invalidated_stores:+ try:+ device = V.graph.get_current_device_or_throw().type+ except Exception:+ device = None+ if device != "cpu":+ buffer.writeline(deferred_line(name, "tl.debug_barrier()"))+ self._unsloth_readback_name = None+ return buffer++ load._unsloth_readback_barrier = True+ kernel.load = load+ kernel.get_load_buffer = get_load_buffer+ try:+ from torch._inductor.custom_graph_pass import CustomGraphPass+ except ImportError:+ CustomGraphPass = None+ _key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)+ logger.info(+ "Unsloth: Patched Inductor to sync before re-reading a buffer stored in a reduction loop."+ )++def fix_dynamo_config_thread_visibility():"""torch 2.12 made torch._dynamo/_inductor config overrides thread-local(ContextVars), so `config.recompile_limit = 1024` set on the main thread is