Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Context parallel: accept Unsloth's fast Qwen3 attention and compiled FastModel attention (#16870)

Daniel Han5c9d1af6b095
diff --git a/tests/test_cp_unsloth_attention_gloo.py b/tests/test_cp_unsloth_attention_gloo.py
new file mode 100644
index 00000000000..3bb228b8695
--- /dev/null
+++ b/tests/test_cp_unsloth_attention_gloo.py
@@ -0,0 +1,331 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""CP over the fast Qwen3 attention and unsloth_zoo's compiled attention: two gloo CPU ranks vs one process.
+GPU-only Triton RoPE / RMSNorm are swapped for torch references; the attention path is the real one."""
+
+import importlib
+import inspect
+import sys
+import textwrap
+import types
+
+import pytest
+import torch
+import torch.distributed as dist
+import torch.nn.functional as F
+
+import unsloth # noqa: F401
+from unsloth import context_parallel as cp
+from unsloth.utils import attention_dispatch as ad
+from unsloth.utils import ulysses_hybrid as uh
+
+transformers = pytest.importorskip("transformers")
+generic = importlib.import_module("tests.test_cp_ulysses_generic_gloo")
+
+SEQ = 16
+HIDDEN = 32
+
+
+def _qwen3_config(kv_heads):
+ config = generic._config("qwen3")
+ config.hidden_size = HIDDEN
+ config.intermediate_size = 64
+ config.head_dim = 8
+ config.num_attention_heads = 4
+ config.num_key_value_heads = kv_heads
+ return config
+
+
+def _fast_class(cls, forward):
+ return type(f"Fast{cls.__name__}", (cls,), {"forward": forward})
+
+
+# unsloth_zoo's compiler template: the class forward only calls a module-level `<Class>_forward`.
+def _compiled_like(cls):
+ modeling = sys.modules[cls.__module__]
+ source = textwrap.dedent(inspect.getsource(cls.forward))
+ source = source.replace("def forward(", f"def {cls.__name__}_forward(", 1)
+ lines = [line for line in source.splitlines() if not line.lstrip().startswith("@")]
+ namespace = dict(vars(modeling))
+ namespace["__name__"] = "unsloth_compiled_module_fake"
+ exec("\n".join(lines), namespace)
+ exec(
+ textwrap.dedent(
+ f"""
+ def forward(self, hidden_states, position_embeddings, attention_mask, past_key_values = None, **kwargs):
+ return {cls.__name__}_forward(self, hidden_states = hidden_states,
+ position_embeddings = position_embeddings, attention_mask = attention_mask,
+ past_key_values = past_key_values, **kwargs)
+ """
+ ),
+ namespace,
+ )
+ return type(cls.__name__, (cls,), {"forward": namespace["forward"]})
+
+
+def _compile_like_model(model):
+ for module in cp._attention_modules(model):
+ module.__class__ = _compiled_like(type(module).__mro__[0])
+ return model
+
+
+class _Rotary:
+ def __init__(self, head_dim, theta):
+ self.head_dim, self.theta, self.asked = head_dim, theta, []
+
+ def extend_rope_embedding(self, x, seq_len):
+ self.asked.append(int(seq_len))
+
+ def get_cached(
+ self,
+ seq_len,
+ device_index = None,
+ ):
+ inv = 1.0 / (self.theta ** (torch.arange(0, self.head_dim, 2).float() / self.head_dim))
+ freqs = torch.outer(torch.arange(seq_len).float(), inv)
+ emb = torch.cat([freqs, freqs], dim = -1)
+ return emb.cos(), emb.sin()
+
+
+def _rope(Q, K, cos, sin, position_ids):
+ cos, sin = cos[position_ids].unsqueeze(1), sin[position_ids].unsqueeze(1)
+
+ def rot(x):
+ a, b = x.chunk(2, dim = -1)
+ return torch.cat([-b, a], dim = -1)
+
+ return Q * cos + rot(Q) * sin, K * cos + rot(K) * sin
+
+
+def _install_cpu_kernels():
+ from unsloth.kernels import qk_norm_rope
+ from unsloth.models import qwen3
+
+ qk_norm_rope.QK_NORM_ROPE_ENABLED = False
+ qwen3.fast_rms_layernorm = lambda norm, X, **_: norm(X)
+ qwen3.fast_rope_embedding = _rope
+ ad.HAS_FLASH_ATTENTION = False
+ ad.HAS_XFORMERS = False
+
+
+def _fast_qwen3_attention(kv_heads):
+ from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention
+ from unsloth.models.qwen3 import Qwen3Attention_fast_forward
+
+ config = _qwen3_config(kv_heads)
+ config._attn_implementation = "sdpa"
+ torch.manual_seed(0)
+ attn = Qwen3Attention(config, 0).float().train()
+ attn.__class__ = _fast_class(Qwen3Attention, Qwen3Attention_fast_forward)
+ attn.rotary_emb = _Rotary(config.head_dim, getattr(config, "rope_theta", 10000.0))
+ return attn
+
+
+def _wrap(attn):
+ root = torch.nn.Module()
+ root.config = attn.config
+ root.layers = torch.nn.ModuleList([torch.nn.Module()])
+ root.layers[0].self_attn = attn
+ return root
+
+
[email protected]("kv_heads", [2, 1])
+def test_unsloth_fast_qwen3_attention_passes_the_cp_check(kv_heads):
+ model = _wrap(_fast_qwen3_attention(kv_heads))
+ assert cp._supports_context_parallel(model, "ulysses")
+ assert cp._supports_context_parallel(model, "ring")
+ cp._check_cp_model(model, 2, "ulysses")
+
+
+def test_compiled_fastmodel_attention_is_hf_registry_attention():
+ model = _compile_like_model(generic._model("qwen3", "sdpa"))
+ assert all(uh.is_hf_registry_attention(m) for m in cp._attention_modules(model))
+ assert cp._generic_hf_attention(model)
+ cp._check_cp_model(model, 2, "ulysses")
+
+
+def test_a_fast_forward_without_the_cp_declaration_stays_refused():
+ from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention
+ from unsloth.models.qwen3 import Qwen3Attention_fast_forward
+
+ def undeclared(self, *args, **kwargs):
+ return Qwen3Attention_fast_forward(self, *args, **kwargs)
+
+ attn = _fast_qwen3_attention(2)
+ attn.__class__ = _fast_class(Qwen3Attention, undeclared)
+ model = _wrap(attn)
+ assert not cp._supports_context_parallel(model, "ulysses")
+ with pytest.raises(NotImplementedError, match = "context parallelism supports"):
+ cp._check_cp_model(model, 2, "ulysses")
+
+
+def _fast_parity_worker(rank, world, kv_heads, packed, out_dir):
+ _install_cpu_kernels()
+ attn = _fast_qwen3_attention(kv_heads)
+ model = _wrap(attn)
+ cp._check_cp_model(model, world, "ulysses")
+ bsz = 1 if packed else 2
+ x = torch.randn(bsz, SEQ, HIDDEN, generator = torch.Generator().manual_seed(1))
+ weight = torch.randn(bsz, SEQ, HIDDEN, generator = torch.Generator().manual_seed(2))
+ lengths = [6, 7, 3] if packed else None
+ positions = torch.arange(SEQ).expand(bsz, -1)
+
+ pieces = [(0, SEQ)] if lengths is None else []
+ start = 0
+ for n in lengths or []:
+ pieces.append((start, start + n))
+ start += n
+ full = torch.zeros(bsz, SEQ, HIDDEN)
+ x_ref = x.clone().requires_grad_(True)
+ for a, b in pieces:
+ out, _, _ = attn(
+ hidden_states = x_ref[:, a:b], position_ids = torch.arange(b - a).expand(bsz, -1)
+ )
+ full[:, a:b] = out.detach()
+ (out * weight[:, a:b]).sum().backward()
+ reference = {n: p.grad.clone() for n, p in attn.named_parameters()}
+ x_grad = x_ref.grad.clone()
+ attn.zero_grad(set_to_none = True)
+
+ calls = []
+ original = ad.ulysses_attention
+
+ def counted(*args, **kwargs):
+ calls.append(1)
+ return original(*args, **kwargs)
+
+ ad.ulysses_attention = counted
+ manager = generic._manager(world, rank)
+ manager.attach_attention_hooks(model)
+ inputs = {"input_ids": torch.zeros(bsz, SEQ, dtype = torch.long)}
+ if packed:
+ inputs["packed_seq_lengths"] = torch.tensor(lengths, dtype = torch.int32)
+ positions = cp._document_positions(lengths, SEQ, x.device)
+ inputs["position_ids"] = positions.clone()
+ local = SEQ // world
+ sl = slice(rank * local, (rank + 1) * local)
+ x_cp = x[:, sl].clone().requires_grad_(True)
+ attn.rotary_emb.asked.clear()
+ with manager.apply(inputs):
+ kwargs = {}
+ if packed:
+ kwargs["packed_seq_lengths"] = inputs["packed_seq_lengths"]
+ out, _, _ = attn(hidden_states = x_cp, position_ids = inputs["position_ids"], **kwargs)
+ (out * weight[:, sl]).sum().backward()
+ ad.ulysses_attention = original
+ assert calls == [1], calls
+ # The rotary table must cover the global positions, not only this shard's length.
+ assert attn.rotary_emb.asked and min(attn.rotary_emb.asked) >= SEQ, attn.rotary_emb.asked
+ torch.testing.assert_close(out.detach(), full[:, sl], rtol = 1e-5, atol = 1e-6)
+ gathered = [torch.zeros_like(x_cp.grad) for _ in range(world)]
+ dist.all_gather(gathered, x_cp.grad)
+ torch.testing.assert_close(torch.cat(gathered, dim = 1), x_grad, rtol = 1e-4, atol = 1e-6)
+ worst = 0.0
+ for name, param in attn.named_parameters():
+ grad = param.grad.clone()
+ dist.all_reduce(grad)
+ torch.testing.assert_close(grad, reference[name], rtol = 1e-4, atol = 1e-6, msg = name)
+ worst = max(worst, float((grad - reference[name]).abs().max()))
+ if rank == 0 and out_dir:
+ import os
+ with open(
+ os.path.join(out_dir, f"fast_qwen3_kv{kv_heads}_{'packed' if packed else 'rows'}.txt"),
+ "w",
+ ) as f:
+ f.write(f"ulysses calls {len(calls)} max grad diff {worst:.3e}\n")
+
+
+ "kv_heads, packed",
+ [
+ (2, False),
+ (1, False), # 1 kv head replicated over cp 2
+ (2, True),
+ ],
+)
+def test_unsloth_fast_qwen3_attention_matches_one_process(kv_heads, packed):
+ _qwen3_config(kv_heads) # skip before forking when the repo is unavailable
+ generic._spawn(_fast_parity_worker, 2, kv_heads, packed, generic._OUT)
+
+
+def _compiled_parity_worker(rank, world, packed, out_dir):
+ original = generic._model
+ generic._model = lambda kind, implementation: _compile_like_model(
+ original(kind, implementation)
+ )
+ try:
+ generic._parity_worker(rank, world, "qwen3", "sdpa", packed, out_dir)
+ finally:
+ generic._model = original
+
+
[email protected]("packed", [False, True])
+def test_compiled_fastmodel_attention_matches_one_process(packed):
+ generic._config("qwen3")
+ generic._spawn(_compiled_parity_worker, 2, packed, "")
+
+
+_RL_REFUSAL = "DPO / GRPO context parallelism supports Unsloth's Llama-style attention only"
+
+
+def test_rl_keeps_context_parallel_llama_only():
+ """Qwen3 is SFTTrainer only: DPO / GRPO need an RL parity run on the fast Qwen3 attention first."""
+ from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention
+ from unsloth.models.llama import LlamaAttention_fast_forward
+
+ qwen3 = _wrap(_fast_qwen3_attention(2))
+ cp._check_cp_model(qwen3, 2, "ulysses")
+ with pytest.raises(NotImplementedError, match = _RL_REFUSAL):
+ cp._check_cp_model(qwen3, 2, "ulysses", allow_hybrids = False)
+ llama = _fast_qwen3_attention(2)
+ llama.__class__ = _fast_class(Qwen3Attention, LlamaAttention_fast_forward)
+ cp._check_cp_model(_wrap(llama), 2, "ulysses", allow_hybrids = False)
+
+
+_REAL_COMPILED = r"""
+import os, sys
+os.environ["UNSLOTH_COMPILE_LOCATION"] = sys.argv[1]
+import unsloth
+from unsloth_zoo.compiler import unsloth_compile_transformers
+unsloth_compile_transformers(model_type = "qwen3", import_from_cache = False)
+import transformers.models.qwen3.modeling_qwen3 as modeling
+from unsloth.utils import ulysses_hybrid as uh
+forward = modeling.Qwen3Attention.forward
+caller = uh._registry_caller(forward)
+# The shape the one-step delegation follow relies on: the class forward only calls the module-level copy.
+assert forward.__module__ == "unsloth_compiled_module_qwen3", forward.__module__
+assert forward.__code__.co_names == ("Qwen3Attention_forward",), forward.__code__.co_names
+assert caller is not None and caller.__name__ == "Qwen3Attention_forward", caller
+assert caller.__module__ == "unsloth_compiled_module_qwen3", caller.__module__
+from tests import test_cp_ulysses_generic_gloo as g
+for implementation, packed in (("sdpa", False), ("sdpa", True), ("eager", False)):
+ g._spawn(g._parity_worker, 2, "qwen3", implementation, packed, "")
+print("REAL_COMPILED_OK")
+"""
+
+
+def test_real_zoo_compiled_qwen3_attention_matches_one_process(tmp_path):
+ """Real unsloth_zoo compiler output (not a copy): pins the class forward shape and runs Ulysses parity on it."""
+ import os
+ import subprocess
+
+ pytest.importorskip("unsloth_zoo.compiler")
+ generic._config("qwen3")
+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ # Fresh interpreter: no test-only CPU fallback switches from tests/conftest.py.
+ env = {k: v for k, v in os.environ.items() if k != "UNSLOTH_DISABLE_CPU_FALLBACK"}
+ env.update(
+ CUDA_VISIBLE_DEVICES = "",
+ PYTHONPATH = os.pathsep.join(filter(None, [root, os.environ.get("PYTHONPATH")])),
+ )
+ result = subprocess.run(
+ [sys.executable, "-c", _REAL_COMPILED, str(tmp_path / "compiled")],
+ cwd = root,
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1200,
+ )
+ assert result.returncode == 0 and "REAL_COMPILED_OK" in result.stdout, (
+ result.stdout[-3000:] + result.stderr[-3000:]
+ )
diff --git a/unsloth/context_parallel.py b/unsloth/context_parallel.py
index 56cfe6606ce..e44c966c796 100644
--- a/unsloth/context_parallel.py
+++ b/unsloth/context_parallel.py
@@ -275,10 +275,15 @@ def _attention_modules(model) -> list:
return [module for name, module in model.named_modules() if name.endswith("self_attn")]
-def _uses_fast_llama_attention(model) -> bool:
- from .models.llama import LlamaAttention_fast_forward, runs_fast_forward
+def _uses_unsloth_attention(model, rl = False) -> bool:
+ """Every attention layer runs a forward marked by context_parallel_attention (with rl: marked for DPO / GRPO)."""
+ from .models.llama import running_forward
+ from .utils.attention_dispatch import is_context_parallel_attention
+
modules = _attention_modules(model)
- return bool(modules) and all(runs_fast_forward(m, LlamaAttention_fast_forward) for m in modules)
+ return bool(modules) and all(
+ is_context_parallel_attention(running_forward(m), rl) for m in modules
+ )
def _generic_hf_attention(model) -> bool:
@@ -295,7 +300,7 @@ def _supports_context_parallel(
backend = "ring",
generic = True,
) -> bool:
- if _uses_fast_llama_attention(model):
+ if _uses_unsloth_attention(model):
return not _stateful_mixers(model)
if backend != "ulysses" or _hybrid_config_blocker(model) is not None:
return False
@@ -393,9 +398,8 @@ class ContextParallelManager:
for module, _ in _stateful_mixers(model):
if is_ulysses_gated_delta(module):
wrap_gated_delta_forward(module, get_cp_manager)
- # Unsloth's fast Llama forward has its own Ulysses path; every other Hugging Face attention layer
- # runs its own attention function after the exchange.
- generic = not _uses_fast_llama_attention(model)
+ # Fast forwards exchange inside run_attention; other HF layers run their attention after the exchange.
+ generic = not _uses_unsloth_attention(model)
for name, module in model.named_modules():
if name.endswith("self_attn") and (
type(module).__name__ in GATED_DELTA_ATTENTION_CLASSES
@@ -1237,7 +1241,7 @@ def _check_cp_model(
generic = (
allow_hybrids
and backend == "ulysses"
- and not _uses_fast_llama_attention(model)
+ and not _uses_unsloth_attention(model)
and _generic_hf_attention(model)
)
unsupported = _unsupported_attention(model_config, backend, generic = generic)
@@ -1266,14 +1270,20 @@ def _check_cp_model(
raise NotImplementedError(
f"Unsloth: Ulysses context parallelism does not support {blocker}."
)
- if not allow_hybrids and not _uses_fast_llama_attention(model) and _generic_hf_attention(model):
+ if not allow_hybrids and (
+ _uses_unsloth_attention(model)
+ and not _uses_unsloth_attention(model, rl = True)
+ or not _uses_unsloth_attention(model)
+ and _generic_hf_attention(model)
+ ):
+ # Fast forwards not declared for RL (Qwen3) and generic HF attention: SFTTrainer only.
raise NotImplementedError(
"Unsloth: DPO / GRPO context parallelism supports Unsloth's Llama-style attention only; generic "
"Ulysses on other Hugging Face attention models is verified for SFTTrainer."
)
if not _supports_context_parallel(model, backend):
raise NotImplementedError(
- "Unsloth: context parallelism supports Unsloth's Llama-style attention with either backend, "
+ "Unsloth: context parallelism supports Unsloth's fast attention (Llama-style, Qwen3) with either backend, "
"and with Ulysses any Hugging Face attention model (attention through ALL_ATTENTION_FUNCTIONS) "
"and the Qwen3-Next / Qwen3.5 Gated DeltaNet hybrids."
)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index fdf7b6e43cc..51071714d89 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -99,6 +99,7 @@ from ..utils.attention_dispatch import (
AttentionConfig,
AttentionContext,
HAS_XFORMERS,
+ context_parallel_attention,
run_attention,
SDPA,
select_attention_backend,
@@ -1094,6 +1095,7 @@ def fast_layernorm_compiled(layernorm, X):
# Ported from transformers models/llama/modeling_llama.py#L320
+@context_parallel_attention(rl = True)
def LlamaAttention_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -3417,13 +3419,20 @@ def runs_fast_forward(
):
"""True when `module` runs Unsloth's `fast` as its class `name`: directly, or through the gate that
pre_patch installs (which runs `fast` only on modules marked by mark_unsloth_fast_path)."""
+ return fast is not None and running_forward(module, name) is fast
+
+
+def running_forward(module, name = "forward"):
+ """The function `module` runs as `name`; for pre_patch's gate, the wrapped fast forward only on marked modules."""
method = getattr(type(module), name, None)
method = getattr(method, "__func__", method)
- if method is fast:
- return True
- if getattr(method, "_unsloth_gated", False) and getattr(method, "__wrapped__", None) is fast:
- return bool(getattr(module, "_unsloth_fast_path", False))
- return False
+ if getattr(method, "_unsloth_gated", False):
+ return (
+ getattr(method, "__wrapped__", None)
+ if getattr(module, "_unsloth_fast_path", False)
+ else None
+ )
+ return method
def mark_unsloth_fast_path(model):
diff --git a/unsloth/models/qwen3.py b/unsloth/models/qwen3.py
index 67d43c9a3a6..6d36c1c59eb 100644
--- a/unsloth/models/qwen3.py
+++ b/unsloth/models/qwen3.py
@@ -15,6 +15,7 @@ import os
from ._utils import __version__
from unsloth_zoo.utils import Version, _get_dtype
from ..utils.packing import get_packed_info_from_kwargs
+from ..context_parallel import get_cp_manager
from ..kernels import qk_norm_rope as _qk_norm_rope
from ..kernels.qk_norm_rope import fast_qk_norm_rope, decode_qk_norm_rope
from . import _decode_graphs
@@ -22,6 +23,7 @@ from .llama import _decode_attention_core, _decode_attention_span
from ..utils.attention_dispatch import (
AttentionConfig,
AttentionContext,
+ context_parallel_attention,
run_attention,
SDPA,
select_attention_backend,
@@ -68,6 +70,7 @@ except:
Qwen3FlashAttention2 = Qwen3Attention
+@context_parallel_attention
def Qwen3Attention_fast_forward(
self,
hidden_states: torch.Tensor,
@@ -109,13 +112,17 @@ def Qwen3Attention_fast_forward(
if past_key_value is not None:
kv_seq_len += past_key_value[0].shape[-2]
+ # Under context parallel this rank holds a shard of the sequence but its positions are global.
+ cp_manager = get_cp_manager()
+ rope_seq_len = kv_seq_len * cp_manager.size if cp_manager is not None else kv_seq_len
+
# Extend RoPE dynamically to fit in VRAM; useful for LongRoPE.
- if position_embeddings and kv_seq_len <= position_embeddings[0].shape[0]:
+ if position_embeddings and rope_seq_len <= position_embeddings[0].shape[0]:
cos, sin = position_embeddings
else:
rotary_emb = self.rotary_emb
- rotary_emb.extend_rope_embedding(V, seq_len = kv_seq_len)
- cos, sin = rotary_emb.get_cached(kv_seq_len, Q.device.index)
+ rotary_emb.extend_rope_embedding(V, seq_len = rope_seq_len)
+ cos, sin = rotary_emb.get_cached(rope_seq_len, Q.device.index)
rope_position_ids = position_ids if position_ids is not None else kwargs.get("position_ids")
if _qk_norm_rope.QK_NORM_ROPE_ENABLED:
diff --git a/unsloth/utils/attention_dispatch.py b/unsloth/utils/attention_dispatch.py
index b8fe615ddc2..8b1150dbe31 100644
--- a/unsloth/utils/attention_dispatch.py
+++ b/unsloth/utils/attention_dispatch.py
@@ -1333,6 +1333,20 @@ def _windowed_causal_mask(q_len: int, k_len: int, sliding_window: int, device) -
return mask
+def context_parallel_attention(forward = None, *, rl = False):
+ """Mark a forward that attends via run_attention and sizes RoPE by the global sequence under CP.
+ `rl = True` also opens DPO / GRPO: their decoder hooks need their own parity run per forward."""
+ if forward is None:
+ return lambda f: context_parallel_attention(f, rl = rl)
+ forward._unsloth_context_parallel = {"rl": bool(rl)}
+ return forward
+
+
+def is_context_parallel_attention(forward, rl = False) -> bool:
+ mark = getattr(forward, "_unsloth_context_parallel", None)
+ return isinstance(mark, dict) and (not rl or mark["rl"])
+
+
def run_attention(
*, config: AttentionConfig, context: AttentionContext, Q: Tensor, K: Tensor, V: Tensor
) -> Tensor:
@@ -1797,4 +1811,6 @@ __all__ = [
"select_attention_backend",
"resolve_prefix_seg_info",
"run_attention",
+ "context_parallel_attention",
+ "is_context_parallel_attention",
]
diff --git a/unsloth/utils/ulysses_hybrid.py b/unsloth/utils/ulysses_hybrid.py
index 25d4313cb04..f66fe36048f 100644
--- a/unsloth/utils/ulysses_hybrid.py
+++ b/unsloth/utils/ulysses_hybrid.py
@@ -5,6 +5,7 @@
runs its own attention function after the exchange) and for hybrid models, whose scans need the full sequence so
they run on heads."""
+import builtins
import functools
import inspect
@@ -241,13 +242,34 @@ _SHARD_LOCAL_KWARGS = (
)
-def _calls_attention_registry(function) -> bool:
+def _registry_caller(function, hops = 1):
+ """`function` or the single function it delegates to (unsloth_zoo's compiled `<Class>_forward`) that reads
+ ALL_ATTENTION_FUNCTIONS, else None. A forward calling several functions is not a delegate (fast forwards)."""
try:
function = inspect.unwrap(function)
except Exception:
- return False
+ return None
code = getattr(function, "__code__", None)
- return code is not None and "ALL_ATTENTION_FUNCTIONS" in code.co_names
+ if code is None:
+ return None
+ if "ALL_ATTENTION_FUNCTIONS" in code.co_names:
+ return function
+ if hops <= 0:
+ return None
+ globs = getattr(function, "__globals__", {})
+ callees = {
+ name for name in code.co_names if callable(globs.get(name)) or name in vars(builtins)
+ }
+ if len(callees) != 1:
+ return None
+ callee = globs.get(callees.pop())
+ if callee is None or callee is function or not inspect.isfunction(callee):
+ return None
+ return _registry_caller(callee, hops - 1)
+
+
+def _calls_attention_registry(function) -> bool:
+ return _registry_caller(function) is not None
def hf_attention_forward(module):
@@ -275,7 +297,7 @@ def is_hf_registry_attention(module) -> bool:
def _forward_globals(module) -> dict:
- return getattr(inspect.unwrap(hf_attention_forward(module)), "__globals__", {})
+ return getattr(_registry_caller(hf_attention_forward(module)), "__globals__", {})
def _route_hf_forward(module, get_cp_manager) -> None: