Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Sync before re-reading a buffer stored in an Inductor reduction loop (torch < 2.14) (#15000)

Daniel Han082c52a658cb
diff --git a/tests/test_inductor_readback_barrier.py b/tests/test_inductor_readback_barrier.py
new file mode 100644
index 00000000000..14f52bb0cd7
--- /dev/null
+++ b/tests/test_inductor_readback_barrier.py
@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""fix_inductor_reduction_readback_race: backport of pytorch/pytorch#190519 for torch < 2.14."""
+
+import importlib.util
+import os
+import subprocess
+import sys
+import textwrap
+import types
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_FIXES = os.path.join(_ROOT, "unsloth", "import_fixes.py")
+
+
+def _load_fixes(name):
+ spec = importlib.util.spec_from_file_location(name, _FIXES)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+class _Buffer:
+ def __init__(self):
+ self.lines = []
+
+ def writeline(self, line):
+ self.lines.append(getattr(line, "line", line))
+
+
+class _CSE:
+ def __init__(self, invalidated):
+ self.invalidated_stores = set(invalidated)
+
+
+def _fake_kernel_class():
+ class FakeTritonKernel:
+ def __init__(self, invalidated):
+ self.cse = _CSE(invalidated)
+ self.loads, self.body = _Buffer(), _Buffer()
+
+ def get_load_buffer(self, indexing):
+ return self.body if indexing == "hoisted" else self.loads
+
+ def load(self, name, index):
+ buffer = self.get_load_buffer(index)
+ buffer.writeline(f"tl.load({name})")
+ return name
+
+ return FakeTritonKernel
+
+
+class _Graph:
+ def __init__(self, device):
+ self.device = torch.device(device)
+
+ def get_current_device_or_throw(self):
+ return self.device
+
+
+def patched(monkeypatch):
+ from torch._inductor import config as inductor_config
+ from torch._inductor.codegen import triton as triton_codegen
+
+ fake = _fake_kernel_class()
+ monkeypatch.setattr(triton_codegen, "TritonKernel", fake)
+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", None)
+ fixes = _load_fixes("unsloth_import_fixes_readback")
+ fixes.fix_inductor_reduction_readback_race()
+ return fake, fixes, inductor_config
+
+
+def _lines(
+ kernel,
+ device,
+ name,
+ index = None,
+):
+ from torch._inductor.virtualized import V
+ with V.set_graph_handler(_Graph(device)):
+ kernel.load(name, index)
+
+
+def test_barrier_precedes_reload_of_an_invalidated_store(patched):
+ fake, _, _ = patched
+ kernel = fake({"buf0"})
+ _lines(kernel, "cuda", "buf0")
+ assert kernel.loads.lines == ["tl.debug_barrier()", "tl.load(buf0)"]
+
+
+def test_barrier_goes_to_the_buffer_the_load_goes_to(patched):
+ fake, _, _ = patched
+ kernel = fake({"buf0"})
+ _lines(kernel, "cuda", "buf0", "hoisted")
+ assert kernel.body.lines == ["tl.debug_barrier()", "tl.load(buf0)"]
+ assert kernel.loads.lines == []
+
+
+def test_no_barrier_for_other_buffers_or_cpu(patched):
+ fake, _, _ = patched
+ kernel = fake({"buf0"})
+ _lines(kernel, "cuda", "arg0_1")
+ _lines(kernel, "cpu", "buf0")
+ assert kernel.loads.lines == ["tl.load(arg0_1)", "tl.load(buf0)"]
+
+
+def test_idempotent_and_keys_the_compile_cache(patched):
+ pytest.importorskip("torch._inductor.custom_graph_pass")
+ fake, fixes, inductor_config = patched
+ first = fake.load
+ fixes.fix_inductor_reduction_readback_race()
+ assert fake.load is first
+ key = inductor_config.post_grad_custom_post_pass
+ from torch._inductor.custom_graph_pass import CustomGraphPass
+
+ assert isinstance(key, CustomGraphPass)
+ assert key.uuid() == "unsloth-inductor-readback-barrier-v1"
+ kernel = fake({"buf0"})
+ _lines(kernel, "cuda", "buf0")
+ assert kernel.loads.lines.count("tl.debug_barrier()") == 1
+
+
+def test_keeps_a_user_post_grad_pass(monkeypatch):
+ from torch._inductor import config as inductor_config
+ from torch._inductor.codegen import triton as triton_codegen
+
+ monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())
+ user_pass = object()
+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", user_pass)
+ _load_fixes("unsloth_import_fixes_readback_user").fix_inductor_reduction_readback_race()
+ assert inductor_config.post_grad_custom_post_pass is user_pass
+
+
+def test_skips_a_torch_that_already_has_the_barrier(monkeypatch):
+ from torch._inductor.codegen import triton as triton_codegen
+
+ fake = _fake_kernel_class()
+
+ def load(self, name, index): # mirrors torch >= 2.14
+ if name in self.cse.invalidated_stores:
+ pass
+ return name
+
+ fake.load = load
+ monkeypatch.setattr(triton_codegen, "TritonKernel", fake)
+ _load_fixes("unsloth_import_fixes_readback_fixed").fix_inductor_reduction_readback_race()
+ assert fake.load is load
+
+
+def test_torch_without_custom_graph_pass_skips_the_fx_graph_cache():
+ # torch 2.4 / 2.5 config is a plain module; newer torch lets TORCHINDUCTOR_FX_GRAPH_CACHE override writes.
+ config = types.SimpleNamespace(fx_graph_cache = True, post_grad_custom_post_pass = None)
+ fixes = _load_fixes("unsloth_import_fixes_readback_old")
+ fixes._key_inductor_caches_on_readback_barrier(config, None)
+ assert config.fx_graph_cache is False
+ assert config.post_grad_custom_post_pass is None
+
+
+_GPU_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, sys, torch
+ from torch._inductor.utils import run_and_get_triton_code
+ if sys.argv[1] == "patched":
+ spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes.fix_inductor_reduction_readback_race()
+
+ def fn(a, b, w, bias):
+ y = a + b
+ mean = y.mean(-1, keepdim = True)
+ var = y.var(-1, keepdim = True, unbiased = False)
+ return y, (y - mean) * torch.rsqrt(var + 1e-6) * w + bias
+
+ n = 4096
+ inputs = [torch.randn(64, n, device = "cuda"), torch.randn(64, n, device = "cuda"),
+ torch.randn(n, device = "cuda"), torch.randn(n, device = "cuda")]
+ compiled = torch.compile(fn, backend = "inductor")
+ code = run_and_get_triton_code(compiled, *inputs)
+ for got, want in zip(compiled(*inputs), fn(*inputs)):
+ torch.testing.assert_close(got, want)
+ print("BARRIER" if "tl.debug_barrier()" in code else "NO_BARRIER")
+ """
+)
+
+
[email protected](not has_real_cuda(), reason = "Inductor Triton codegen needs a GPU")
+def test_generated_reduction_syncs_before_the_reload(tmp_path):
+ env = dict(
+ os.environ,
+ TORCHINDUCTOR_CACHE_DIR = str(tmp_path / "inductor"),
+ TORCHINDUCTOR_FX_GRAPH_CACHE = "0",
+ )
+ run = lambda arm: subprocess.run(
+ [sys.executable, "-c", _GPU_RUNNER, arm, _FIXES],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ patched = run("patched")
+ assert patched.returncode == 0, patched.stderr[-3000:]
+ assert patched.stdout.strip().splitlines()[-1] == "BARRIER"
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index e1ee188fc54..ab7f2ac6d53 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -383,6 +383,7 @@ from .import_fixes import (
fix_rocm_windows_fused_sdpa,
fix_triton_compiled_kernel_missing_attrs,
fix_dynamo_config_thread_visibility,
+ fix_inductor_reduction_readback_race,
patch_trunc_normal_precision_issue,
ignore_logger_messages,
patch_ipykernel_hf_xet,
@@ -481,6 +482,7 @@ fix_triton_compiled_kernel_missing_attrs()
# Must run before unsloth_zoo's patch_torch_compile and the gpt-oss patches raise the dynamo
# recompile limits, so those settings reach the autograd worker threads on torch >= 2.12.
fix_dynamo_config_thread_visibility()
+fix_inductor_reduction_readback_race()
patch_trunc_normal_precision_issue()
ignore_logger_messages()
patch_ipykernel_hf_xet()
@@ -545,6 +547,7 @@ del fix_cudnn_sdpa_d256_masked_backward
del fix_rocm_windows_fused_sdpa
del fix_triton_compiled_kernel_missing_attrs
del fix_dynamo_config_thread_visibility
+del fix_inductor_reduction_readback_race
del patch_trunc_normal_precision_issue
del ignore_logger_messages
del patch_ipykernel_hf_xet
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index f4d667f655f..84d19f8b18f 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -7253,6 +7253,86 @@ def fix_triton_compiled_kernel_missing_attrs():
)
+def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass):
+ # Warm FX / AOTAutograd caches would return barrier-free kernels: a no-op pass's uuid keys them.
+ if custom_graph_pass is None:
+ # torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.
+ inductor_config.fx_graph_cache = False
+ return
+
+ class ReadbackBarrierCacheKey(custom_graph_pass):
+ def __call__(self, graph):
+ return None
+
+ def uuid(self):
+ return "unsloth-inductor-readback-barrier-v1"
+
+ if getattr(inductor_config, "post_grad_custom_post_pass", None) is None:
+ inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()
+
+
+def fix_inductor_reduction_readback_race():
+ """torch < 2.14 Inductor reloads a buffer stored by an earlier loop of the same reduction kernel with no
+ barrier, and Triton may give the store and the reload different warps: silently wrong gradients
+ (pytorch/pytorch#198562). Backports pytorch/pytorch#190519."""
+ try:
+ import torch
+ from torch._inductor.codegen import triton as triton_codegen
+ from torch._inductor import config as inductor_config
+ from torch._inductor.virtualized import V
+ except Exception:
+ return
+ kernel = getattr(triton_codegen, "TritonKernel", None)
+ deferred_line = getattr(triton_codegen, "DeferredLine", None)
+ if kernel is None or deferred_line is None or not hasattr(kernel, "get_load_buffer"):
+ return
+ if getattr(kernel.load, "_unsloth_readback_barrier", False):
+ return
+ try:
+ if "invalidated_stores" in inspect.getsource(kernel.load):
+ return
+ except Exception:
+ return
+
+ original_load = kernel.load
+ original_get_load_buffer = kernel.get_load_buffer
+
+ @functools.wraps(original_load)
+ def load(self, name, index, *args, **kwargs):
+ previous = self.__dict__.get("_unsloth_readback_name")
+ self._unsloth_readback_name = name
+ try:
+ return original_load(self, name, index, *args, **kwargs)
+ finally:
+ self._unsloth_readback_name = previous
+
+ @functools.wraps(original_get_load_buffer)
+ def get_load_buffer(self, indexing):
+ buffer = original_get_load_buffer(self, indexing)
+ name = self.__dict__.get("_unsloth_readback_name")
+ if name is not None and name in self.cse.invalidated_stores:
+ try:
+ device = V.graph.get_current_device_or_throw().type
+ except Exception:
+ device = None
+ if device != "cpu":
+ buffer.writeline(deferred_line(name, "tl.debug_barrier()"))
+ self._unsloth_readback_name = None
+ return buffer
+
+ load._unsloth_readback_barrier = True
+ kernel.load = load
+ kernel.get_load_buffer = get_load_buffer
+ try:
+ from torch._inductor.custom_graph_pass import CustomGraphPass
+ except ImportError:
+ CustomGraphPass = None
+ _key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
+ logger.info(
+ "Unsloth: Patched Inductor to sync before re-reading a buffer stored in a reduction loop."
+ )
+
+
def fix_dynamo_config_thread_visibility():
"""torch 2.12 made torch._dynamo/_inductor config overrides thread-local
(ContextVars), so `config.recompile_limit = 1024` set on the main thread is