Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Inductor readback barrier: key the compile cache when a user pass is already set (#15208)
Daniel Han7ebf5e4b6e2b
diff --git a/tests/test_inductor_readback_barrier.py b/tests/test_inductor_readback_barrier.pyindex 14f52bb0cd7..7fe2d0ee914 100644--- a/tests/test_inductor_readback_barrier.py+++ b/tests/test_inductor_readback_barrier.py@@ -220,3 +220,141 @@ def test_generated_reduction_syncs_before_the_reload(tmp_path):patched = run("patched")assert patched.returncode == 0, patched.stderr[-3000:]assert patched.stdout.strip().splitlines()[-1] == "BARRIER"+++def _user_pass_class(base_name = "CustomGraphPass"):+ custom_graph_pass = pytest.importorskip("torch._inductor.custom_graph_pass")+ base = getattr(custom_graph_pass, base_name, None)+ if base is None:+ pytest.skip(f"torch has no {base_name}")++ class UserPass(base):+ def __init__(self):+ self.calls = []++ def __call__(self, graph, *args, **kwargs):+ self.calls.append((graph, args, kwargs))++ def uuid(self):+ return "user-pass-v1"++ return UserPass+++def _install_with(monkeypatch, existing):+ from torch._inductor import config as inductor_config+ from torch._inductor.codegen import triton as triton_codegen++ monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", existing)+ fixes = _load_fixes("unsloth_import_fixes_readback_keyed")+ fixes.fix_inductor_reduction_readback_race()+ return fixes, inductor_config+++def test_a_user_custom_graph_pass_keys_the_cache_on_the_barrier_too(monkeypatch):+ """A pre-set CustomGraphPass must carry the marker in its uuid and still run."""+ from torch._inductor.custom_graph_pass import CustomGraphPass++ user = _user_pass_class()()+ fixes, inductor_config = _install_with(monkeypatch, user)+ keyed = inductor_config.post_grad_custom_post_pass+ # torch keeps the cache only for CustomGraphPass instances with a truthy uuid.+ assert isinstance(keyed, CustomGraphPass) and keyed.uuid()+ assert keyed.uuid() != user.uuid()+ assert keyed.uuid() == ("unsloth-inductor-readback-barrier-v1", "user-pass-v1")+ graph = object()+ keyed(graph)+ assert user.calls == [(graph, (), {})]+ # Idempotent: no re-wrap.+ fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)+ assert inductor_config.post_grad_custom_post_pass is keyed+++def test_an_inference_aware_user_pass_still_gets_is_inference(monkeypatch):+ user = _user_pass_class("CustomInferenceAwareGraphPass")() # skips on torch without it+ from torch._inductor.custom_graph_pass import CustomInferenceAwareGraphPass++ _, inductor_config = _install_with(monkeypatch, user)+ keyed = inductor_config.post_grad_custom_post_pass+ # post_grad.py passes is_inference only to CustomInferenceAwareGraphPass instances.+ assert isinstance(keyed, CustomInferenceAwareGraphPass)+ keyed("g", is_inference = True)+ assert user.calls == [("g", (), {"is_inference": True})]+++def test_a_list_of_user_passes_gets_the_marker_appended(monkeypatch):+ user = _user_pass_class()()+ _, inductor_config = _install_with(monkeypatch, [user])+ passes = inductor_config.post_grad_custom_post_pass+ assert isinstance(passes, list) and passes[0] is user+ assert [p.uuid() for p in passes] == ["user-pass-v1", "unsloth-inductor-readback-barrier-v1"]+++def test_a_raw_callable_user_pass_is_left_alone(monkeypatch):+ # torch bypasses the FX cache for raw callables; wrapping one would make it cacheable unkeyed.+ def raw(graph):+ return None++ _, inductor_config = _install_with(monkeypatch, raw)+ assert inductor_config.post_grad_custom_post_pass is raw+++_CACHE_RUNNER = textwrap.dedent(+ """+ import importlib.util, json, sys, torch+ from torch._inductor import config+ from torch._inductor.custom_graph_pass import CustomGraphPass+ from torch._dynamo.utils import counters++ class UserPass(CustomGraphPass):+ def __call__(self, graph):+ return None++ def uuid(self):+ return "user-pass-v1"++ config.post_grad_custom_post_pass = UserPass()+ if sys.argv[1] == "patched":+ spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])+ fixes = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(fixes)+ fixes._key_inductor_caches_on_readback_barrier(config, CustomGraphPass)+ torch.compile(lambda x: (x.sin() + 1).sum(-1), backend = "inductor")(torch.randn(4, 8))+ print(json.dumps(dict(counters["inductor"])))+ """+)+++def test_a_warm_cache_from_before_the_fix_is_not_served_with_a_user_pass(tmp_path):+ """End to end on CPU (the key, not the kernel): a cache warmed without the fix must miss."""+ pytest.importorskip("torch._inductor.custom_graph_pass")+ import json++ runner = tmp_path / "runner.py"+ runner.write_text(_CACHE_RUNNER)+ env = {+ **os.environ,+ "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),+ "TORCHINDUCTOR_FX_GRAPH_CACHE": "1",+ "TORCHINDUCTOR_AUTOGRAD_CACHE": "1",+ "CUDA_VISIBLE_DEVICES": "",+ }++ def run(arm):+ out = subprocess.run(+ [sys.executable, str(runner), arm, _FIXES],+ env = env,+ capture_output = True,+ text = True,+ timeout = 600,+ )+ assert out.returncode == 0, out.stderr[-2000:]+ return json.loads(out.stdout.strip().splitlines()[-1])++ warm = run("base")+ assert warm.get("fxgraph_cache_miss", 0) >= 1+ assert run("base").get("fxgraph_cache_hit", 0) >= 1 # the key is otherwise stable+ patched = run("patched")+ assert patched.get("fxgraph_cache_hit", 0) == 0+ assert patched.get("fxgraph_cache_miss", 0) >= 1diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 84d19f8b18f..9e23c474282 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -7259,16 +7259,54 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)# torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.inductor_config.fx_graph_cache = Falsereturn+ marker = "unsloth-inductor-readback-barrier-v1"class ReadbackBarrierCacheKey(custom_graph_pass):+ _unsloth_readback_barrier = True+def __call__(self, graph):return Nonedef uuid(self):- return "unsloth-inductor-readback-barrier-v1"+ return marker- if getattr(inductor_config, "post_grad_custom_post_pass", None) is None:+ existing = getattr(inductor_config, "post_grad_custom_post_pass", None)+ if existing is None:inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()+ return+ # A pre-set pass keys the cache by its uuid alone; raw callables / uuid-less passes bypass it.+ if isinstance(existing, (list, tuple)):+ if any(getattr(p, "_unsloth_readback_barrier", False) for p in existing):+ return+ inductor_config.post_grad_custom_post_pass = type(existing)(+ [*existing, ReadbackBarrierCacheKey()]+ )+ return+ if getattr(existing, "_unsloth_readback_barrier", False):+ return+ if not isinstance(existing, custom_graph_pass) or not existing.uuid():+ return+ base = custom_graph_pass+ try:+ from torch._inductor.custom_graph_pass import CustomInferenceAwareGraphPass+ if isinstance(existing, CustomInferenceAwareGraphPass):+ base = CustomInferenceAwareGraphPass # called with is_inference=+ except ImportError:+ pass++ class ReadbackBarrierKeyedPass(base):+ _unsloth_readback_barrier = True++ def __init__(self, inner):+ self.inner = inner++ def __call__(self, graph, *args, **kwargs):+ return self.inner(graph, *args, **kwargs)++ def uuid(self):+ return (marker, self.inner.uuid())++ inductor_config.post_grad_custom_post_pass = ReadbackBarrierKeyedPass(existing)def fix_inductor_reduction_readback_race():