Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Inductor readback barrier: key the compile cache when a user pass is already set (#15208)

Daniel Han7ebf5e4b6e2b
diff --git a/tests/test_inductor_readback_barrier.py b/tests/test_inductor_readback_barrier.py
index 14f52bb0cd7..7fe2d0ee914 100644
--- a/tests/test_inductor_readback_barrier.py
+++ b/tests/test_inductor_readback_barrier.py
@@ -220,3 +220,141 @@ def test_generated_reduction_syncs_before_the_reload(tmp_path):
patched = run("patched")
assert patched.returncode == 0, patched.stderr[-3000:]
assert patched.stdout.strip().splitlines()[-1] == "BARRIER"
+
+
+def _user_pass_class(base_name = "CustomGraphPass"):
+ custom_graph_pass = pytest.importorskip("torch._inductor.custom_graph_pass")
+ base = getattr(custom_graph_pass, base_name, None)
+ if base is None:
+ pytest.skip(f"torch has no {base_name}")
+
+ class UserPass(base):
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, graph, *args, **kwargs):
+ self.calls.append((graph, args, kwargs))
+
+ def uuid(self):
+ return "user-pass-v1"
+
+ return UserPass
+
+
+def _install_with(monkeypatch, existing):
+ from torch._inductor import config as inductor_config
+ from torch._inductor.codegen import triton as triton_codegen
+
+ monkeypatch.setattr(triton_codegen, "TritonKernel", _fake_kernel_class())
+ monkeypatch.setattr(inductor_config, "post_grad_custom_post_pass", existing)
+ fixes = _load_fixes("unsloth_import_fixes_readback_keyed")
+ fixes.fix_inductor_reduction_readback_race()
+ return fixes, inductor_config
+
+
+def test_a_user_custom_graph_pass_keys_the_cache_on_the_barrier_too(monkeypatch):
+ """A pre-set CustomGraphPass must carry the marker in its uuid and still run."""
+ from torch._inductor.custom_graph_pass import CustomGraphPass
+
+ user = _user_pass_class()()
+ fixes, inductor_config = _install_with(monkeypatch, user)
+ keyed = inductor_config.post_grad_custom_post_pass
+ # torch keeps the cache only for CustomGraphPass instances with a truthy uuid.
+ assert isinstance(keyed, CustomGraphPass) and keyed.uuid()
+ assert keyed.uuid() != user.uuid()
+ assert keyed.uuid() == ("unsloth-inductor-readback-barrier-v1", "user-pass-v1")
+ graph = object()
+ keyed(graph)
+ assert user.calls == [(graph, (), {})]
+ # Idempotent: no re-wrap.
+ fixes._key_inductor_caches_on_readback_barrier(inductor_config, CustomGraphPass)
+ assert inductor_config.post_grad_custom_post_pass is keyed
+
+
+def test_an_inference_aware_user_pass_still_gets_is_inference(monkeypatch):
+ user = _user_pass_class("CustomInferenceAwareGraphPass")() # skips on torch without it
+ from torch._inductor.custom_graph_pass import CustomInferenceAwareGraphPass
+
+ _, inductor_config = _install_with(monkeypatch, user)
+ keyed = inductor_config.post_grad_custom_post_pass
+ # post_grad.py passes is_inference only to CustomInferenceAwareGraphPass instances.
+ assert isinstance(keyed, CustomInferenceAwareGraphPass)
+ keyed("g", is_inference = True)
+ assert user.calls == [("g", (), {"is_inference": True})]
+
+
+def test_a_list_of_user_passes_gets_the_marker_appended(monkeypatch):
+ user = _user_pass_class()()
+ _, inductor_config = _install_with(monkeypatch, [user])
+ passes = inductor_config.post_grad_custom_post_pass
+ assert isinstance(passes, list) and passes[0] is user
+ assert [p.uuid() for p in passes] == ["user-pass-v1", "unsloth-inductor-readback-barrier-v1"]
+
+
+def test_a_raw_callable_user_pass_is_left_alone(monkeypatch):
+ # torch bypasses the FX cache for raw callables; wrapping one would make it cacheable unkeyed.
+ def raw(graph):
+ return None
+
+ _, inductor_config = _install_with(monkeypatch, raw)
+ assert inductor_config.post_grad_custom_post_pass is raw
+
+
+_CACHE_RUNNER = textwrap.dedent(
+ """
+ import importlib.util, json, sys, torch
+ from torch._inductor import config
+ from torch._inductor.custom_graph_pass import CustomGraphPass
+ from torch._dynamo.utils import counters
+
+ class UserPass(CustomGraphPass):
+ def __call__(self, graph):
+ return None
+
+ def uuid(self):
+ return "user-pass-v1"
+
+ config.post_grad_custom_post_pass = UserPass()
+ if sys.argv[1] == "patched":
+ spec = importlib.util.spec_from_file_location("fixes", sys.argv[2])
+ fixes = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(fixes)
+ fixes._key_inductor_caches_on_readback_barrier(config, CustomGraphPass)
+ torch.compile(lambda x: (x.sin() + 1).sum(-1), backend = "inductor")(torch.randn(4, 8))
+ print(json.dumps(dict(counters["inductor"])))
+ """
+)
+
+
+def test_a_warm_cache_from_before_the_fix_is_not_served_with_a_user_pass(tmp_path):
+ """End to end on CPU (the key, not the kernel): a cache warmed without the fix must miss."""
+ pytest.importorskip("torch._inductor.custom_graph_pass")
+ import json
+
+ runner = tmp_path / "runner.py"
+ runner.write_text(_CACHE_RUNNER)
+ env = {
+ **os.environ,
+ "TORCHINDUCTOR_CACHE_DIR": str(tmp_path / "inductor"),
+ "TORCHINDUCTOR_FX_GRAPH_CACHE": "1",
+ "TORCHINDUCTOR_AUTOGRAD_CACHE": "1",
+ "CUDA_VISIBLE_DEVICES": "",
+ }
+
+ def run(arm):
+ out = subprocess.run(
+ [sys.executable, str(runner), arm, _FIXES],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 600,
+ )
+ assert out.returncode == 0, out.stderr[-2000:]
+ return json.loads(out.stdout.strip().splitlines()[-1])
+
+ warm = run("base")
+ assert warm.get("fxgraph_cache_miss", 0) >= 1
+ assert run("base").get("fxgraph_cache_hit", 0) >= 1 # the key is otherwise stable
+ patched = run("patched")
+ assert patched.get("fxgraph_cache_hit", 0) == 0
+ assert patched.get("fxgraph_cache_miss", 0) >= 1
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 84d19f8b18f..9e23c474282 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -7259,16 +7259,54 @@ def _key_inductor_caches_on_readback_barrier(inductor_config, custom_graph_pass)
# torch < 2.6 keys no custom pass: skip the FX graph cache rather than reuse racy kernels.
inductor_config.fx_graph_cache = False
return
+ marker = "unsloth-inductor-readback-barrier-v1"
class ReadbackBarrierCacheKey(custom_graph_pass):
+ _unsloth_readback_barrier = True
+
def __call__(self, graph):
return None
def uuid(self):
- return "unsloth-inductor-readback-barrier-v1"
+ return marker
- if getattr(inductor_config, "post_grad_custom_post_pass", None) is None:
+ existing = getattr(inductor_config, "post_grad_custom_post_pass", None)
+ if existing is None:
inductor_config.post_grad_custom_post_pass = ReadbackBarrierCacheKey()
+ return
+ # A pre-set pass keys the cache by its uuid alone; raw callables / uuid-less passes bypass it.
+ if isinstance(existing, (list, tuple)):
+ if any(getattr(p, "_unsloth_readback_barrier", False) for p in existing):
+ return
+ inductor_config.post_grad_custom_post_pass = type(existing)(
+ [*existing, ReadbackBarrierCacheKey()]
+ )
+ return
+ if getattr(existing, "_unsloth_readback_barrier", False):
+ return
+ if not isinstance(existing, custom_graph_pass) or not existing.uuid():
+ return
+ base = custom_graph_pass
+ try:
+ from torch._inductor.custom_graph_pass import CustomInferenceAwareGraphPass
+ if isinstance(existing, CustomInferenceAwareGraphPass):
+ base = CustomInferenceAwareGraphPass # called with is_inference=
+ except ImportError:
+ pass
+
+ class ReadbackBarrierKeyedPass(base):
+ _unsloth_readback_barrier = True
+
+ def __init__(self, inner):
+ self.inner = inner
+
+ def __call__(self, graph, *args, **kwargs):
+ return self.inner(graph, *args, **kwargs)
+
+ def uuid(self):
+ return (marker, self.inner.uuid())
+
+ inductor_config.post_grad_custom_post_pass = ReadbackBarrierKeyedPass(existing)
def fix_inductor_reduction_readback_race():