Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(fast): partition the decision head's attention by the head's own head count

AlKor137190eee35da1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c97f733d..f6b48815 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -133,6 +133,7 @@ jobs:
python tests/test_question_token_reuse.py
python tests/test_lazy_import.py
python tests/test_runtime_fixes.py
+ python tests/test_fast_head_partition.py
python tests/test_agent_gate.py
python tests/test_option_collapse.py
python tests/test_lang_guess.py
diff --git a/laya/fast.py b/laya/fast.py
index 7333ee57..6597c592 100644
--- a/laya/fast.py
+++ b/laya/fast.py
@@ -45,6 +45,13 @@ class FastLaya:
self._forward_lock = threading.RLock()
self.verbose = verbose
self.H, self.Dh, self.D = cfg.num_attention_heads, cfg.hidden_size // cfg.num_attention_heads, cfg.hidden_size
+ # The decision head is a separate nn.TransformerEncoderLayer built with its OWN head count
+ # (common.py: `nhead = max(1, d // 64)`), not the encoder's `cfg.num_attention_heads`. The
+ # two agree only when the encoder's head_dim is exactly 64 (every shipped checkpoint), so
+ # reusing `self.H`/`self.Dh` for the head's attention silently mixed the wrong channels on
+ # any checkpoint with a non-64 encoder head_dim. Partition the head by its own heads.
+ self.H_head = max(1, self.D // 64)
+ self.Dh_head = self.D // self.H_head
self.F = cfg.intermediate_size
self.eps = cfg.norm_eps
self.max_len = max_len
@@ -125,6 +132,15 @@ class FastLaya:
self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H, self.Dh, window=window, dtype=self.kdtype)
return self._attn_k[key]
+ def attn_k_head(self, B, L):
+ # The decision head's full attention, partitioned by the head's own heads (self.H_head),
+ # not the encoder's. Cached apart from `attn_k` so the two never share a kernel built for
+ # the wrong partition. The head always uses full attention (window 0).
+ key = (None, None, 0, "head") if L <= self.DYNAMIC_MAX_L else (B, L, 0, "head")
+ if key not in self._attn_k:
+ self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H_head, self.Dh_head, window=0, dtype=self.kdtype)
+ return self._attn_k[key]
+
# ------------------------------------------------------------------ encoder + head on padded [B, L]
def _encode(self, ids, lens, qtype):
"""ids [B,L] long (padded), lens [B] int32, qtype [B] long -> hidden [B, L, D] fp32"""
@@ -157,7 +173,7 @@ class FastLaya:
for j, h in enumerate(self.head):
self.k_ln_b(X, Y, h["n1w"], h["n1b"], Y)
self.k_in(Y, h["in_w"], h["in_b"], qkv)
- self.attn_k(B, L, 0)(qkv.view(B, L, 3, self.H, self.Dh), lens, O.view(B, L, D))
+ self.attn_k_head(B, L)(qkv.view(B, L, 3, self.H_head, self.Dh_head), lens, O.view(B, L, D))
self.k_out(O, h["out_w"], h["out_b"], Y)
self.k_addln_b(X, Y, h["n2w"], h["n2b"], Y) # X += attn ; Y = norm2(X)
self.k_ffn1(Y, h["l1w"], h["l1b"], F1)
diff --git a/tests/test_fast_head_partition.py b/tests/test_fast_head_partition.py
new file mode 100644
index 00000000..b86a27cb
--- /dev/null
+++ b/tests/test_fast_head_partition.py
@@ -0,0 +1,56 @@
+"""The fast path must partition the decision head's attention by the head's own head count.
+
+The TileLang fast path only runs on CUDA (importing `laya.fast` pulls in `tilelang`), so this is a
+source-level pin -- it reads the files as text, imports nothing, and runs in the ordinary CI lane.
+It guards the fix for the head/encoder head-count mismatch: the decision head is built with
+`nhead = max(1, d // 64)` (laya/common.py), which differs from the encoder's
+`cfg.num_attention_heads` whenever the encoder head_dim != 64, and reusing the encoder partition
+for the head's attention silently mixes the wrong channels on any such checkpoint.
+
+Run: python tests/test_fast_head_partition.py
+"""
+import os
+import re
+import sys
+
+_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+FAST = open(os.path.join(_ROOT, "laya", "fast.py"), encoding="utf-8").read()
+COMMON = open(os.path.join(_ROOT, "laya", "common.py"), encoding="utf-8").read()
+
+PASS, FAIL = [], []
+
+
+def check(name, cond, detail=""):
+ (PASS if cond else FAIL).append(name if cond else "%s %s" % (name, detail))
+
+
+# The decision head's own head partition is defined, by the same rule common.py builds it with.
+check("fast/defines H_head = max(1, self.D // 64)",
+ re.search(r"self\.H_head\s*=\s*max\(\s*1\s*,\s*self\.D\s*//\s*64\s*\)", FAST) is not None)
+check("fast/defines Dh_head from H_head",
+ re.search(r"self\.Dh_head\s*=\s*self\.D\s*//\s*self\.H_head", FAST) is not None)
+check("common/head built with nhead = max(1, d // 64)",
+ re.search(r"nhead\s*=\s*max\(\s*1\s*,\s*d\s*//\s*64\s*\)", COMMON) is not None,
+ "if this rule moves, fast.py's H_head must move with it")
+
+# The head's attention reshape and kernel use the head partition, not the encoder's.
+check("fast/head attention reshapes by H_head/Dh_head",
+ "qkv.view(B, L, 3, self.H_head, self.Dh_head)" in FAST)
+check("fast/head has its own attn kernel builder",
+ re.search(r"def attn_k_head\b", FAST) is not None)
+check("fast/head kernel is built with H_head/Dh_head",
+ re.search(r"attn_kernel\([^)]*self\.H_head,\s*self\.Dh_head", FAST) is not None)
+
+# The encoder's attention is unchanged -- it legitimately uses cfg.num_attention_heads.
+check("fast/encoder attention still uses self.H/self.Dh",
+ "qkv.view(B, L, 3, self.H, self.Dh)" in FAST)
+# And the head no longer reuses the encoder partition anywhere.
+check("fast/no head reshape left on the encoder partition",
+ FAST.count("qkv.view(B, L, 3, self.H, self.Dh)") == 1,
+ "exactly the encoder call should remain")
+
+
+print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))
+for f in FAIL:
+ print(" FAIL " + f)
+sys.exit(1 if FAIL else 0)