Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

15.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""fix_inductor_reduction_readback_race: backport of pytorch/pytorch#190519 for torch < 2.14."""

import importlib.util
import os
import subprocess
import sys
import textwrap
import types

import pytest
import torch
from real_accelerator import has_real_cuda

_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_FIXES = os.path.join(_ROOT, "unsloth", "import_fixes.py")


def _load_fixes(name):
    spec = importlib.util.spec_from_file_location(name, _FIXES)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


class _Buffer:
    def __init__(self):
        self.lines = []

    def writeline(self, line):
        self.lines.append(getattr(line, "line", line))


class _CSE:
    def __init__(self, invalidated):
        self.invalidated_stores = set(invalidated)


def _fake_kernel_class():
    class FakeTritonKernel:
        def __init__(self, invalidated):
            self.cse = _CSE(invalidated)
            self.loads, self.body = _Buffer(), _Buffer()

        def get_load_buffer(self, indexing):
            return self.body if indexing == "hoisted" else self.loads

        def load(self, name, index):
            buffer = self.get_load_buffer(index)
            buffer.writeline(f"tl.load({name})")
            return name

    return FakeTritonKernel


class _Graph:
    def __init__(self, device):
        self.device = torch.device(device)

    def get_current_device_or_throw(self):
        return self.device


@pytest.fixture
def patched(monkeypatch):
    from torch._inductor import config as inductor_config
    from torch._inductor.codegen import triton as triton_codegen

    fake = _fake_kernel_class()
    monkeypatch.setattr(triton_codegen, "TritonKernel", fake)
    monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", None)
    fixes = _load_fixes("unsloth_import_fixes_readback")
    fixes.fix_inductor_reduction_readback_race()
    return fake, fixes, inductor_config


def _lines(
    kernel,
    device,
    name,
    index = None,
):
    from torch._inductor.virtualized import V
    with V.set_graph_handler(_Graph(device)):
        kernel.load(name, index)


def test_barrier_precedes_reload_of_an_invalidated_store(patched):
    fake, _, _ = patched
    kernel = fake({"buf0"})
    _lines(kernel, "cuda", "buf0")
    assert kernel.loads.lines == ["tl.debug_barrier()", "tl.load(buf0)"]


def test_barrier_goes_to_the_buffer_the_load_goes_to(patched):
    fake, _, _ = patched
    kernel = fake({"buf0"})
    _lines(kernel, "cuda", "buf0", "hoisted")
    assert kernel.body.lines == ["tl.debug_barrier()", "tl.load(buf0)"]
    assert kernel.loads.lines == []


def test_no_barrier_for_other_buffers_or_cpu(patched):
    fake, _, _ = patched
    kernel = fake({"buf0"})
    _lines(kernel, "cuda", "arg0_1")
    _lines(kernel, "cpu", "buf0")
    assert kernel.loads.lines == ["tl.load(arg0_1)", "tl.load(buf0)"]


def test_idempotent_and_keys_the_compile_cache(patched):
    pytest.importorskip("torch._inductor.custom_graph_pass")
    fake, fixes, inductor_config = patched
    first = fake.load
    fixes.fix_inductor_reduction_readback_race()
    assert fake.load is first
    key = inductor_config.post_grad_custom_post_pass
    from torch._inductor.custom_graph_pass import CustomGraphPass

    assert isinstance(key, CustomGraphPass)
    assert key.uuid() == "unsloth-inductor-readback-barrier-v1"
    kernel = fake({"buf0"})
    _lines(kernel, "cuda", "buf0")
    assert kernel.loads.lines.count("tl.debug_barrier()") == 1


def test_keeps_a_user_post_grad_pass(monkeypatch):
    from torch._inductor import config as inductor_config
    from torch._inductor.codegen import triton as triton_codegen

    monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())
    user_pass = object()
    monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", user_pass)
    _load_fixes("unsloth_import_fixes_readback_user").fix_inductor_reduction_readback_race()
    assert inductor_config.post_grad_custom_post_pass is user_pass


def test_skips_a_torch_that_already_has_the_barrier(monkeypatch):
    from torch._inductor.codegen import triton as triton_codegen

    fake = _fake_kernel_class()

    def load(self, name, index):  # mirrors torch >= 2.14
        if name in self.cse.invalidated_stores:
            pass
        return name

    fake.load = load
    monkeypatch.setattr(triton_codegen, "TritonKernel", fake)
    _load_fixes("unsloth_import_fixes_readback_fixed").fix_inductor_reduction_readback_race()
    assert fake.load is load


def test_torch_without_custom_graph_pass_skips_the_fx_graph_cache():
    # torch 2.4 / 2.5 config is a plain module; newer torch lets TORCHINDUCTOR_FX_GRAPH_CACHE override writes.
    config = types.SimpleNamespace(fx_graph_cache = True, post_grad_custom_post_pass = None)
    fixes = _load_fixes("unsloth_import_fixes_readback_old")
    fixes._key_inductor_caches_on_readback_barrier(config, None)
    assert config.fx_graph_cache is False
    assert config.post_grad_custom_post_pass is None


_GPU_RUNNER = textwrap.dedent(
    """
    import importlib.util, sys, torch
    from torch._inductor.utils import run_and_get_triton_code
    if sys.argv[1] == "patched":
        spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])
        fixes = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(fixes)
        fixes.fix_inductor_reduction_readback_race()

    def fn(a, b, w, bias):
        y = a + b
        mean = y.mean(-1, keepdim = True)
        var = y.var(-1, keepdim = True, unbiased = False)
        return y, (y - mean) * torch.rsqrt(var + 1e-6) * w + bias

    n = 4096
    inputs = [torch.randn(64, n, device = "cuda"), torch.randn(64, n, device = "cuda"),
              torch.randn(n, device = "cuda"), torch.randn(n, device = "cuda")]
    compiled = torch.compile(fn, backend = "inductor")
    code = run_and_get_triton_code(compiled, *inputs)
    for got, want in zip(compiled(*inputs), fn(*inputs)):
        torch.testing.assert_close(got, want)
    print("BARRIER" if "tl.debug_barrier()" in code else "NO_BARRIER")
    """
)


@pytest.mark.skipif(not has_real_cuda(), reason = "Inductor Triton codegen needs a GPU")
def test_generated_reduction_syncs_before_the_reload(tmp_path):
    env = dict(
        os.environ,
        TORCHINDUCTOR_CACHE_DIR = str(tmp_path / "inductor"),
        TORCHINDUCTOR_FX_GRAPH_CACHE = "0",
    )
    run = lambda arm: subprocess.run(
        [sys.executable, "-c", _GPU_RUNNER, arm, _FIXES],
        env = env,
        capture_output = True,
        text = True,
        timeout = 600,
    )
    patched = run("patched")
    assert patched.returncode == 0, patched.stderr[-3000:]
    assert patched.stdout.strip().splitlines()[-1] == "BARRIER"


def _user_pass_class(base_name = "CustomGraphPass"):
    custom_graph_pass = pytest.importorskip("torch._inductor.custom_graph_pass")
    base = getattr(custom_graph_pass, base_name, None)
    if base is None:
        pytest.skip(f"torch has no {base_name}")

    class UserPass(base):
        def __init__(self):
            self.calls = []

        def __call__(self, graph, *args, **kwargs):
            self.calls.append((graph, args, kwargs))

        def uuid(self):
            return "user-pass-v1"

    return UserPass


def _install_with(monkeypatch, existing):
    from torch._inductor import config as inductor_config
    from torch._inductor.codegen import triton as triton_codegen

    monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())
    monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", existing)
    fixes = _load_fixes("unsloth_import_fixes_readback_keyed")
    fixes.fix_inductor_reduction_readback_race()
    return fixes, inductor_config


def test_a_user_custom_graph_pass_keys_the_cache_on_the_barrier_too(monkeypatch):
    """A pre-set CustomGraphPass must carry the marker in its uuid and still run."""
    from torch._inductor.custom_graph_pass import CustomGraphPass

    user = _user_pass_class()()
    fixes, inductor_config = _install_with(monkeypatch, user)
    keyed = inductor_config.post_grad_custom_post_pass
    # torch keeps the cache only for CustomGraphPass instances with a truthy uuid.
    assert isinstance(keyed, CustomGraphPass) and keyed.uuid()
    assert keyed.uuid() != user.uuid()
    assert keyed.uuid() == ("unsloth-inductor-readback-barrier-v1", "user-pass-v1")
    graph = object()
    keyed(graph)
    assert user.calls == [(graph, (), {})]
    # Idempotent: no re-wrap.
    fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
    assert inductor_config.post_grad_custom_post_pass is keyed


def test_an_inference_aware_user_pass_still_gets_is_inference(monkeypatch):
    user = _user_pass_class("CustomInferenceAwareGraphPass")()  # skips on torch without it
    from torch._inductor.custom_graph_pass import CustomInferenceAwareGraphPass

    _, inductor_config = _install_with(monkeypatch, user)
    keyed = inductor_config.post_grad_custom_post_pass
    # post_grad.py passes is_inference only to CustomInferenceAwareGraphPass instances.
    assert isinstance(keyed, CustomInferenceAwareGraphPass)
    keyed("g", is_inference = True)
    assert user.calls == [("g", (), {"is_inference": True})]


def test_a_list_of_user_passes_gets_the_marker_appended(monkeypatch):
    user = _user_pass_class()()
    _, inductor_config = _install_with(monkeypatch, [user])
    passes = inductor_config.post_grad_custom_post_pass
    assert isinstance(passes, list) and passes[0] is user
    assert [p.uuid() for p in passes] == ["user-pass-v1", "unsloth-inductor-readback-barrier-v1"]


def test_a_raw_callable_user_pass_is_left_alone(monkeypatch):
    # torch bypasses the FX cache for raw callables; wrapping one would make it cacheable unkeyed.
    def raw(graph):
        return None

    _, inductor_config = _install_with(monkeypatch, raw)
    assert inductor_config.post_grad_custom_post_pass is raw


_CACHE_RUNNER = textwrap.dedent(
    """
    import importlib.util, json, sys, torch
    from torch._inductor import config
    from torch._inductor.custom_graph_pass import CustomGraphPass
    from torch._dynamo.utils import counters

    class UserPass(CustomGraphPass):
        def __call__(self, graph):
            return None

        def uuid(self):
            return "user-pass-v1"

    config.post_grad_custom_post_pass = UserPass()
    if sys.argv[1] == "patched":
        spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])
        fixes = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(fixes)
        fixes._key_inductor_caches_on_readback_barrier(config, CustomGraphPass)
    torch.compile(lambda x: (x.sin() + 1).sum(-1), backend = "inductor")(torch.randn(4, 8))
    print(json.dumps(dict(counters["inductor"])))
    """
)


def test_a_warm_cache_from_before_the_fix_is_not_served_with_a_user_pass(tmp_path):
    """End to end on CPU (the key, not the kernel): a cache warmed without the fix must miss."""
    pytest.importorskip("torch._inductor.custom_graph_pass")
    import json

    runner = tmp_path / "runner.py"
    runner.write_text(_CACHE_RUNNER)
    env = {
        **os.environ,
        "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),
        "TORCHINDUCTOR_FX_GRAPH_CACHE": "1",
        "TORCHINDUCTOR_AUTOGRAD_CACHE": "1",
        "CUDA_VISIBLE_DEVICES": "",
    }

    def run(arm):
        out = subprocess.run(
            [sys.executable, str(runner), arm, _FIXES],
            env = env,
            capture_output = True,
            text = True,
            timeout = 600,
        )
        assert out.returncode == 0, out.stderr[-2000:]
        return json.loads(out.stdout.strip().splitlines()[-1])

    warm = run("base")
    assert warm.get("fxgraph_cache_miss", 0) >= 1
    assert run("base").get("fxgraph_cache_hit", 0) >= 1  # the key is otherwise stable
    patched = run("patched")
    assert patched.get("fxgraph_cache_hit", 0) == 0
    assert patched.get("fxgraph_cache_miss", 0) >= 1


def test_a_wrapped_user_pass_that_later_opts_out_of_caching_still_bypasses(monkeypatch):
    """torch's contract: uuid() -> None skips the FX cache. The wrapper must forward that."""
    from torch._inductor.custom_graph_pass import CustomGraphPass

    user = _user_pass_class()()
    _, inductor_config = _install_with(monkeypatch, user)
    keyed = inductor_config.post_grad_custom_post_pass
    assert keyed.uuid() == ("unsloth-inductor-readback-barrier-v1", "user-pass-v1")
    user.uuid = lambda: None
    assert isinstance(keyed, CustomGraphPass)
    assert keyed.uuid() is None


_OPT_OUT_RUNNER = textwrap.dedent(
    """
    import importlib.util, json, sys, torch
    from torch._inductor import config
    from torch._inductor.custom_graph_pass import CustomGraphPass
    from torch._dynamo.utils import counters

    class TogglingPass(CustomGraphPass):
        enabled = True
        offset = 2.0

        def uuid(self):
            return "initial" if self.enabled else None

        def __call__(self, graph):
            for node in graph.nodes:
                if node.op == "call_function" and node.target == torch.ops.aten.add.Tensor:
                    node.args = (node.args[0], self.offset)

    user = TogglingPass()
    config.post_grad_custom_post_pass = user
    if sys.argv[1] == "patched":
        spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])
        fixes = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(fixes)
        fixes._key_inductor_caches_on_readback_barrier(config, CustomGraphPass)
    user.enabled = False  # opt out of caching, then change what the pass does
    results = []
    for offset in (2.0, 3.0):
        user.offset = offset
        torch._dynamo.reset()
        counters.clear()
        out = torch.compile(lambda x: x + 1, fullgraph = True)(torch.zeros(4))
        results.append({"out": out.tolist(), "inductor": dict(counters["inductor"])})
    print(json.dumps(results))
    """
)


def test_a_cache_opted_out_user_pass_is_never_served_a_stale_graph(tmp_path):
    """End to end on CPU: after uuid() -> None, a changed pass must recompile, not hit the cache."""
    pytest.importorskip("torch._inductor.custom_graph_pass")
    import json

    runner = tmp_path / "runner.py"
    runner.write_text(_OPT_OUT_RUNNER)

    def run(arm):
        env = {
            **os.environ,
            "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / f"inductor_{arm}"),
            "TORCHINDUCTOR_FX_GRAPH_CACHE": "1",
            "TORCHINDUCTOR_AUTOGRAD_CACHE": "1",
            "CUDA_VISIBLE_DEVICES": "",
        }
        out = subprocess.run(
            [sys.executable, str(runner), arm, _FIXES],
            env = env,
            capture_output = True,
            text = True,
            timeout = 600,
        )
        assert out.returncode == 0, out.stderr[-2000:]
        return json.loads(out.stdout.strip().splitlines()[-1])

    base = run("base")
    assert [r["out"] for r in base] == [[2.0] * 4, [3.0] * 4]  # the harness can see the change
    patched = run("patched")
    assert [r["out"] for r in patched] == [[2.0] * 4, [3.0] * 4]
    assert all(r["inductor"].get("fxgraph_cache_hit", 0) == 0 for r in patched)