Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(fast): partition the decision head's attention by the head's own head count
AlKor137190eee35da1
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c97f733d..f6b48815 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -133,6 +133,7 @@ jobs:python tests/test_question_token_reuse.pypython tests/test_lazy_import.pypython tests/test_runtime_fixes.py+ python tests/test_fast_head_partition.pypython tests/test_agent_gate.pypython tests/test_option_collapse.pypython tests/test_lang_guess.pydiff --git a/laya/fast.py b/laya/fast.pyindex 7333ee57..6597c592 100644--- a/laya/fast.py+++ b/laya/fast.py@@ -45,6 +45,13 @@ class FastLaya:self._forward_lock = threading.RLock()self.verbose = verboseself.H, self.Dh, self.D = cfg.num_attention_heads, cfg.hidden_size // cfg.num_attention_heads, cfg.hidden_size+ # The decision head is a separate nn.TransformerEncoderLayer built with its OWN head count+ # (common.py: `nhead = max(1, d // 64)`), not the encoder's `cfg.num_attention_heads`. The+ # two agree only when the encoder's head_dim is exactly 64 (every shipped checkpoint), so+ # reusing `self.H`/`self.Dh` for the head's attention silently mixed the wrong channels on+ # any checkpoint with a non-64 encoder head_dim. Partition the head by its own heads.+ self.H_head = max(1, self.D // 64)+ self.Dh_head = self.D // self.H_headself.F = cfg.intermediate_sizeself.eps = cfg.norm_epsself.max_len = max_len@@ -125,6 +132,15 @@ class FastLaya:self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H, self.Dh, window=window, dtype=self.kdtype)return self._attn_k[key]+ def attn_k_head(self, B, L):+ # The decision head's full attention, partitioned by the head's own heads (self.H_head),+ # not the encoder's. Cached apart from `attn_k` so the two never share a kernel built for+ # the wrong partition. The head always uses full attention (window 0).+ key = (None, None, 0, "head") if L <= self.DYNAMIC_MAX_L else (B, L, 0, "head")+ if key not in self._attn_k:+ self._attn_k[key] = K.attn_kernel(key[0], key[1], self.H_head, self.Dh_head, window=0, dtype=self.kdtype)+ return self._attn_k[key]+# ------------------------------------------------------------------ encoder + head on padded [B, L]def _encode(self, ids, lens, qtype):"""ids [B,L] long (padded), lens [B] int32, qtype [B] long -> hidden [B, L, D] fp32"""@@ -157,7 +173,7 @@ class FastLaya:for j, h in enumerate(self.head):self.k_ln_b(X, Y, h["n1w"], h["n1b"], Y)self.k_in(Y, h["in_w"], h["in_b"], qkv)- self.attn_k(B, L, 0)(qkv.view(B, L, 3, self.H, self.Dh), lens, O.view(B, L, D))+ self.attn_k_head(B, L)(qkv.view(B, L, 3, self.H_head, self.Dh_head), lens, O.view(B, L, D))self.k_out(O, h["out_w"], h["out_b"], Y)self.k_addln_b(X, Y, h["n2w"], h["n2b"], Y) # X += attn ; Y = norm2(X)self.k_ffn1(Y, h["l1w"], h["l1b"], F1)diff --git a/tests/test_fast_head_partition.py b/tests/test_fast_head_partition.pynew file mode 100644index 00000000..b86a27cb--- /dev/null+++ b/tests/test_fast_head_partition.py@@ -0,0 +1,56 @@+"""The fast path must partition the decision head's attention by the head's own head count.++The TileLang fast path only runs on CUDA (importing `laya.fast` pulls in `tilelang`), so this is a+source-level pin -- it reads the files as text, imports nothing, and runs in the ordinary CI lane.+It guards the fix for the head/encoder head-count mismatch: the decision head is built with+`nhead = max(1, d // 64)` (laya/common.py), which differs from the encoder's+`cfg.num_attention_heads` whenever the encoder head_dim != 64, and reusing the encoder partition+for the head's attention silently mixes the wrong channels on any such checkpoint.++Run: python tests/test_fast_head_partition.py+"""+import os+import re+import sys++_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+FAST = open(os.path.join(_ROOT, "laya", "fast.py"), encoding="utf-8").read()+COMMON = open(os.path.join(_ROOT, "laya", "common.py"), encoding="utf-8").read()++PASS, FAIL = [], []+++def check(name, cond, detail=""):+ (PASS if cond else FAIL).append(name if cond else "%s %s" % (name, detail))+++# The decision head's own head partition is defined, by the same rule common.py builds it with.+check("fast/defines H_head = max(1, self.D // 64)",+ re.search(r"self\.H_head\s*=\s*max\(\s*1\s*,\s*self\.D\s*//\s*64\s*\)", FAST) is not None)+check("fast/defines Dh_head from H_head",+ re.search(r"self\.Dh_head\s*=\s*self\.D\s*//\s*self\.H_head", FAST) is not None)+check("common/head built with nhead = max(1, d // 64)",+ re.search(r"nhead\s*=\s*max\(\s*1\s*,\s*d\s*//\s*64\s*\)", COMMON) is not None,+ "if this rule moves, fast.py's H_head must move with it")++# The head's attention reshape and kernel use the head partition, not the encoder's.+check("fast/head attention reshapes by H_head/Dh_head",+ "qkv.view(B, L, 3, self.H_head, self.Dh_head)" in FAST)+check("fast/head has its own attn kernel builder",+ re.search(r"def attn_k_head\b", FAST) is not None)+check("fast/head kernel is built with H_head/Dh_head",+ re.search(r"attn_kernel\([^)]*self\.H_head,\s*self\.Dh_head", FAST) is not None)++# The encoder's attention is unchanged -- it legitimately uses cfg.num_attention_heads.+check("fast/encoder attention still uses self.H/self.Dh",+ "qkv.view(B, L, 3, self.H, self.Dh)" in FAST)+# And the head no longer reuses the encoder partition anywhere.+check("fast/no head reshape left on the encoder partition",+ FAST.count("qkv.view(B, L, 3, self.H, self.Dh)") == 1,+ "exactly the encoder call should remain")+++print("\n%d passed, %d failed" % (len(PASS), len(FAIL)))+for f in FAIL:+ print(" FAIL " + f)+sys.exit(1 if FAIL else 0)