Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Metal: trellis expert kernels (TQ2_T, TQK6, TQK7) and the block Hadamard rotation for Gyro

Niko1221672fd99ccce0
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txt
index 7a2e60b9a..220164d45 100644
--- a/metal/CMakeLists.txt
+++ b/metal/CMakeLists.txt
@@ -31,7 +31,7 @@ set(GGML_COMMON "${STRATA_ROOT}/third_party/ggml/ggml-common.h")
# ---- the kernel tree (mac-kern) ------------------------------------------------------------------------------------
add_library(mtl_rt STATIC src/mtl_rt.mm src/ref_quant.cpp src/gguf_lite.cpp)
-target_include_directories(mtl_rt PUBLIC src ${STRATA_ROOT}/third_party/ggml)
+target_include_directories(mtl_rt PUBLIC src ${STRATA_ROOT}/third_party/ggml ${STRATA_ROOT}/include)
target_compile_options(mtl_rt PRIVATE $<$<COMPILE_LANGUAGE:OBJCXX>:-fobjc-arc>)
target_compile_definitions(mtl_rt PRIVATE
STRATA_GGML_COMMON_DEFAULT="${GGML_COMMON}"
diff --git a/metal/kernels/moe.metal b/metal/kernels/moe.metal
index bb8fdbbcc..e3f0b1d68 100644
--- a/metal/kernels/moe.metal
+++ b/metal/kernels/moe.metal
@@ -334,3 +334,6 @@ DN_SET(T_Q5_1) DN_SET(T_Q8_0)
DN_SET(T_Q2_0)
DN_SET(T_Q2_0R)
DN_SET(T_IQ4_NL)
+// Gyro (agentionai's trellis experts): TQK6 / TQ2_T gate/up, TQK7 / TQ2_T down (the CUDA engine's STRATA_GU_FMTS / D_FMTS)
+GU_SET(T_TQK6) GU_SET(T_TQ2T)
+DN_SET(T_TQK7) DN_SET(T_TQ2T)
diff --git a/metal/kernels/moe_mm.metal b/metal/kernels/moe_mm.metal
index ff28897ac..fb2f6f056 100644
--- a/metal/kernels/moe_mm.metal
+++ b/metal/kernels/moe_mm.metal
@@ -393,7 +393,9 @@ MM_GU2_KQ(T_Q4_K) MM_GU2_KQ(T_Q5_K) MM_GU2_KQ(T_Q6_K) MM_GU2_U32(T_Q8_0) MM_DN2_
MMGU2(T_IQ4_XS, 16, 64, 32, 2) MMGU2(T_IQ4_XS, 16, 64, 64, 2) MMGU2(T_IQ4_XS, 32, 64, 32, 2) MMGU2(T_IQ4_XS, 16, 32, 32, 2)
MMGU2(T_IQ4_XS, 64, 64, 32, 4)
MM_GU2_SET(T_Q2_0) MM_GU2_SET(T_IQ3_XXS) MM_GU2_SET(T_IQ2_XXS) MM_GU2_SET(T_IQ2_XS) MM_GU2_SET(T_IQ2_S) MM_GU2_SET(T_IQ3_S)
+MM_GU2_SET(T_TQK6) MM_GU2_SET(T_TQ2T) // Gyro: the trellis gate/up (units of 16 weights: TK 16 or a multiple)
MM_DN2_SET(T_Q2_0)
+MM_DN2_SET(T_TQK7) MM_DN2_SET(T_TQ2T)
MMDN2(T_IQ4_NL, 16, 64, 32, 2) MMDN2(T_IQ4_NL, 16, 64, 64, 2) MMDN2(T_IQ4_NL, 16, 32, 32, 2) MMDN2(T_IQ4_NL, 32, 32, 32, 2) MMDN2(T_IQ4_NL, 16, 128, 32, 4)
// ---- v3: the weights decoded straight into matmul2d's right-input cooperative tensor (single-simdgroup scope),
diff --git a/metal/kernels/quant.metal b/metal/kernels/quant.metal
index 51fecca7c..14df14eef 100644
--- a/metal/kernels/quant.metal
+++ b/metal/kernels/quant.metal
@@ -29,6 +29,9 @@
#define T_IQ4_XS 23
#define T_Q2_0 42
#define T_Q2_0R 142 // Q2_0 read with raw activations (codes - 1 in the decoder; no prepared x needed)
+#define T_TQ2T 144 // agentionai's trellis types (Gyro): 128 values a block, 34 / 26 / 30 bytes, Hadamard-folded experts
+#define T_TQK6 145
+#define T_TQK7 146
template <int F> struct fmt;
// QK block elements, BB block bytes, UNIT weights per unit, TAB words of threadgroup table
@@ -51,6 +54,10 @@ template <> struct fmt<T_Q8_0> { enum { QK = 32, BB = 34, UNIT = 32, TAB =
template <> struct fmt<T_Q4_0> { enum { QK = 32, BB = 18, UNIT = 32, TAB = 1 }; };
template <> struct fmt<T_Q5_0> { enum { QK = 32, BB = 22, UNIT = 32, TAB = 1 }; };
template <> struct fmt<T_Q5_1> { enum { QK = 32, BB = 24, UNIT = 32, TAB = 1 }; };
+// trellis: a unit is one lane of 4 steps = 16 weights; TAB = the 2048-point codebook as half2 words
+template <> struct fmt<T_TQ2T> { enum { QK = 128, BB = 34, UNIT = 16, TAB = 2048 }; };
+template <> struct fmt<T_TQK6> { enum { QK = 128, BB = 26, UNIT = 16, TAB = 2048 }; };
+template <> struct fmt<T_TQK7> { enum { QK = 128, BB = 30, UNIT = 16, TAB = 2048 }; };
static inline uint rd32(device const uchar* p) {
device const ushort* s = (device const ushort*)p;
@@ -75,7 +82,277 @@ static inline uint g_iq2xxs(threadgroup const uint* tab, uint i, int w) { return
static inline uint g_iq2xs(threadgroup const uint* tab, uint i, int w) { return ctab ? uint(iq2xs_grid[i] >> (32 * w)) : tab[2 * i + w]; }
static inline uint g_iq2s(threadgroup const uint* tab, uint i, int w) { return ctab ? uint(iq2s_grid[i] >> (32 * w)) : tab[2 * i + w]; }
+// BEGIN tq_lut (generated by metal/tools/gen_tq_lut.py from third_party/ggml/ggml-common-tq.h; do not edit)
+constant uint tq_lut_u[2048] = {
+ 0xb8f7bd4d, 0x3cd5b6ce, 0x4368bd9e, 0xb8a63dce, 0xb430395e, 0x3dbc3bc1, 0x3c9cb92b, 0x3aaabca2,
+ 0xb8e13cdb, 0x3dfbbb33, 0xbb183a8d, 0x3da0bd64, 0x3b83bf1a, 0x3659b389, 0xabd7b40f, 0x35a240ee,
+ 0x3972c057, 0x3996be17, 0xb9b5b926, 0x3c75393b, 0xbdef4151, 0xbd09b5a3, 0x4012b408, 0x3ece3b7f,
+ 0x4011b688, 0xbe303b17, 0xb4bf3d9b, 0xb8c8b870, 0x38b2b537, 0x34b031be, 0x38713389, 0x38b8ba33,
+ 0x3b3dbce0, 0x3d3d3367, 0x3cbe3f50, 0xb99130b0, 0x39863733, 0xb6d9b70e, 0xbd98be8c, 0xbe04bcca,
+ 0x403c36a9, 0x3eaeb527, 0x39823c6a, 0xbd5bb655, 0x35963ebb, 0xbbbd417e, 0xb1d32fe0, 0x31c0b858,
+ 0xbc10b54e, 0xaf8b3cf0, 0x3f5f38e6, 0xb2f63b87, 0xb675b114, 0x329829bf, 0xb955bb7b, 0xa98434e9,
+ 0x378fc04a, 0x3c8c31a0, 0x3c682dea, 0x2d292fe1, 0xbd722c33, 0x3e3e379d, 0x3cf2b918, 0x38bfbcbe,
+ 0xbdd53284, 0xbd013707, 0x38c1bec1, 0x3838b960, 0x30103413, 0x3672376d, 0x3f42302f, 0xbe9a3a61,
+ 0xbc713c06, 0xbcf2bb57, 0x3c85a9dc, 0xb8b23873, 0x340d2d1e, 0xb73f3661, 0xb8b7bb91, 0x3943b956,
+ 0x39853958, 0xb5563b26, 0xc2b9ae17, 0xb9ea34e9, 0xb6f433d5, 0xb1af3826, 0xaf98b8a4, 0xbc693a2d,
+ 0x358e3a25, 0x32fd406d, 0xb11a38a4, 0xb02eb170, 0xb5cbaa0a, 0x3d453c04, 0xb55bb258, 0xbbf5b412,
+ 0x3f4b3d33, 0x38a9bc6e, 0xb715b8c0, 0xbdd92ec6, 0x2980beb3, 0xbb3c3c54, 0x2d683ba0, 0xb20942f9,
+ 0x3337b557, 0xbda73870, 0x35bd2d57, 0x385eb8cd, 0xb68e39fe, 0xbd743eda, 0xbd543643, 0x37d43a32,
+ 0x2f7f38f8, 0x380f37f3, 0xbc231b1e, 0x35c825c7, 0xb82a2e3e, 0x3897b85f, 0x3cb33c0c, 0x377bc13e,
+ 0x3f09b3a4, 0x3c73c006, 0x3ad1bda5, 0xa3cfb0c6, 0xb680b9d5, 0xb493384b, 0xa9853a3c, 0xba24bcb5,
+ 0x3c21bd0c, 0x34dc2f68, 0xbdb7bc5d, 0xbddbbbbf, 0xb95ab4c4, 0xbdaab59b, 0xbdce34cf, 0x3f9d3bad,
+ 0x3bd53a05, 0x2c64b69c, 0x3adb4003, 0x35e83d4c, 0xb5cc308d, 0xbb86bd2d, 0x3cd3b124, 0xb2ab3a0a,
+ 0x3d4bb507, 0x284cb190, 0xbe83384c, 0xb806bc94, 0xb14631e4, 0x3178bbe0, 0x3819bc9d, 0x2e8f3824,
+ 0x4150c246, 0x3a43bcf3, 0xbdd53da9, 0x3c47b229, 0xc001400e, 0xb0c33bb2, 0x3822bd46, 0xa4a9b92d,
+ 0x3cfa3c72, 0x42843550, 0xbc97b9e5, 0x357ab996, 0xaf70ba03, 0xbc0735e7, 0x397bb663, 0x29423c59,
+ 0xb689adf7, 0x3db8b875, 0xb63c35ea, 0x33493674, 0x3ae03926, 0xb56ebb20, 0xa9ae3f05, 0x3c383ce5,
+ 0xb1ef40b8, 0xba2bba73, 0x3d3fbdd7, 0x3c093822, 0xbce4bd87, 0x36beba38, 0xbe23b808, 0xbd77ba33,
+ 0x34663b90, 0xb805c1c5, 0x375b34c0, 0xa2dbb4be, 0x3495b96c, 0xb99ab38a, 0x31d5390a, 0x3e16a910,
+ 0x3dd62878, 0xbdc03964, 0xb3efb783, 0xb571ba12, 0xc0f2b8ce, 0x3771bd8f, 0x3eba3656, 0x3c39be0e,
+ 0x371b3ac5, 0x40713469, 0x2e19bd99, 0x3cb83ab2, 0xa92c36d9, 0xbc9bbcb4, 0xb4e2a0fa, 0x31fd34ed,
+ 0xb0a04033, 0x342ebac8, 0xbccd3df5, 0xbc5fbc5a, 0x3ada3f32, 0x39ec3878, 0x3ce53db1, 0xbe93b891,
+ 0xb75e3c11, 0xbb27b82c, 0xba6736d6, 0x352cb602, 0xbf3a3507, 0xbcf5b8e9, 0x3773b55e, 0xbd0ab832,
+ 0x3989bf01, 0x3d733edf, 0x3bc8bb3d, 0xb7e0bf82, 0x393f4053, 0x2f91b07d, 0xb6afbc8b, 0x2a463538,
+ 0xbbfd3c2f, 0x324ebfc9, 0xb7023a8f, 0xbda9b939, 0x3c3f3ac3, 0x37983875, 0x3e1b3dc4, 0x255eb820,
+ 0xbe92a7e2, 0xb7c7b63f, 0xbcb438bd, 0xb9583a2f, 0x34fcbdf2, 0xb5a83ba1, 0x30473975, 0x21daaecc,
+ 0x3bc4b7f7, 0x2eb639f8, 0xbb3abb3f, 0xb13dbd9b, 0xbab237fa, 0x30743794, 0xbad3bd7e, 0x346b3d65,
+ 0xba613cfd, 0xb8feadd5, 0xb6ecbe09, 0xb544c05b, 0x2fdb360c, 0x39e5b8ba, 0x3e96b8a8, 0x38cf40cf,
+ 0xac5aba8b, 0xb284b691, 0xb5d5c0b9, 0x3d003864, 0xa3823220, 0xbb8f3422, 0xb85eba85, 0x3eb33854,
+ 0xb885c110, 0x2e60bdf8, 0xb8ba1cb3, 0x3636b4d3, 0xaf93382a, 0x346abc81, 0x31483d66, 0x36fec0b3,
+ 0x4001af70, 0x3eac3caf, 0xb614bd48, 0x3de83d04, 0x31d03f2b, 0x30ceb955, 0x2d33b81b, 0xb657b328,
+ 0x3527b7c2, 0xc255b9f6, 0x3c90bc1a, 0x35f3be9c, 0x409ac11d, 0xbf2f30db, 0x3285be02, 0x2f14b607,
+ 0x35c23c56, 0x30ac3288, 0xc25c3c1c, 0xbbd63ca4, 0x29c83465, 0x412937a3, 0x32253151, 0xad412ef2,
+ 0xbfc1bbcb, 0xbe71b0c0, 0xbb59b17d, 0xbefb3cfe, 0x39f0bdb8, 0x28a74014, 0x396ab771, 0xc063bbdc,
+ 0x2898babe, 0x3de9b683, 0xaa78bd1f, 0x3ded4187, 0x3d7b3e18, 0xab0caf3c, 0xb52438a7, 0xc38fb964,
+ 0x3d02beda, 0xb25daa74, 0xbe9a2d90, 0xbbb0be23, 0x3a8cbf6a, 0x3d87bbfa, 0x3571b53c, 0xc027385c,
+ 0xb801b903, 0x3b7b3368, 0x3ab8b70c, 0xb62538b5, 0x3cb1b436, 0x3f2c8d2a, 0x2c53b88e, 0x393bb2c8,
+ 0x3f40bdbf, 0x39d9bc1b, 0x306dbc3a, 0xb8fcb3ba, 0x3c75c07b, 0xb421b655, 0xb7203d34, 0xb6a037f1,
+ 0xbec7b338, 0x334eba58, 0xb8103aab, 0xbe3db3d7, 0x40a2b984, 0xba54b5cd, 0xb63e3afa, 0x394dbd80,
+ 0x2ddaba54, 0xb8e4baec, 0xad382b2a, 0x3886bd82, 0xc015ac6b, 0x3c72368b, 0xbaa72fb3, 0xb2cb38fb,
+ 0xbfb5335d, 0xa535aba9, 0x38b7afb8, 0x35243d00, 0xaeac3b42, 0xb1a64144, 0xba3ab352, 0x39f32c72,
+ 0xb82c4079, 0xb45fb9e5, 0xb9f59d20, 0xbd36b2ee, 0x2f11bad5, 0x3e803450, 0xb684213f, 0xbd073200,
+ 0xbc5eb839, 0x3f5dc217, 0xbe943d9a, 0x33813947, 0xb9439081, 0x37953eac, 0x3915ad61, 0x3d5db810,
+ 0x33f239cb, 0x41edbfe8, 0xb4a23ee7, 0xb88ab9e5, 0xc0da36b3, 0x406cb57f, 0x3c5cba70, 0x3ae4b936,
+ 0xb7ff3e2c, 0xb08e35b6, 0x3c3ab8d1, 0xafed227e, 0x310a3a3d, 0xb9262d00, 0x3f76bbe1, 0x3e15b8aa,
+ 0xc074b31c, 0xbcb640ee, 0x3b033449, 0xbd0138f5, 0x386abc2d, 0xbcfc3f94, 0xb04933a2, 0xb86e3c77,
+ 0xb91c366c, 0x35c6b832, 0xbc4c36ab, 0x3a04b6b9, 0x3c84b2c9, 0xb7623006, 0x3580330e, 0x39263d32,
+ 0xbfb238f9, 0x3de6c00a, 0xafad3a35, 0x3d88be69, 0xbe12c30f, 0x3d26b13d, 0xb7a5382a, 0x3ac340fa,
+ 0xc028b8a9, 0x3e19b979, 0xbdd7bac1, 0xb802bd42, 0xbe1ab631, 0xac7ab8fb, 0x38193ff8, 0x42e83d3f,
+ 0xb1dcb312, 0xaa42c036, 0xc078bd28, 0x36ddb6f5, 0x31fdbad4, 0xb7e93933, 0x31d439b4, 0xb4aebd13,
+ 0x3c5d37d3, 0xbafd3925, 0x38753b52, 0xa5812a38, 0xb4503b51, 0x3cccc248, 0x3ad2b1aa, 0xbe9036eb,
+ 0x34e3423c, 0x3bc13ac2, 0x37493be5, 0x3ceebcd1, 0x40173ac7, 0xbfc43a67, 0xb9a8b7eb, 0xbf08bf94,
+ 0xa983b711, 0xb6093c0f, 0xb7f73ebd, 0xbca734f4, 0x38b1bac3, 0xb93fb5d0, 0x3f1d3e04, 0x305cb7fd,
+ 0xbcaaaae3, 0x3d8cbb0f, 0xbd3fbca4, 0xbc08bb3b, 0xb9de3a90, 0xad25377f, 0xbefeb988, 0xbc6cbdf7,
+ 0x35c3ba23, 0x370abb14, 0xb3a93c45, 0x3c72b889, 0x3e713a36, 0xb8e03a90, 0x34a634cb, 0x3c6c40be,
+ 0xbc3fbe95, 0x2284ba4d, 0xb73eb96e, 0xbc04384d, 0x3a03b452, 0xc10041a8, 0x2ed23c75, 0x3ab6bc47,
+ 0x3a633746, 0x3735b62f, 0x3c2fb9dd, 0xb08fbba8, 0x3289b270, 0x3c1e3972, 0xb5ae3844, 0x3d303cee,
+ 0x3fa2b98f, 0xb1503e16, 0x382c3c9c, 0xae2e3483, 0x3a843b71, 0x32f4bb53, 0x3ab23540, 0xba06b6e0,
+ 0x3701bcc9, 0x361ab89a, 0x3a74be12, 0x37ae3b56, 0x38eeb6ec, 0xb60ab47a, 0x374a3247, 0x3883394d,
+ 0x2c41b76f, 0xb862bc07, 0xbba5395e, 0x3865b71f, 0x2cbc3897, 0x35c9ac7a, 0x3ac5b9df, 0xb8b9b63b,
+ 0x3ff6c064, 0xb8293ba3, 0x3b7eb904, 0xb78eafdb, 0x3c22baec, 0x39a5b53d, 0xc0d4bab6, 0xbddbb47e,
+ 0xc01cba66, 0xbcb037ec, 0x3ac3b409, 0x3e472b2b, 0xb8e03b8a, 0x3a7738cd, 0xb7243cb0, 0x381fbaef,
+ 0xb0f9ba7a, 0xb1383940, 0xb8403855, 0x35f439af, 0x2c193b0f, 0x3b833760, 0x3801a813, 0x41d6b801,
+ 0x3ab33026, 0xa9f5bca3, 0x370ebf1e, 0x3467b82b, 0xbd57b4c3, 0xb4e9ad4c, 0xbad6b8c4, 0xbd66a8f7,
+ 0x3cd23e70, 0x3421236c, 0x3c22aa8a, 0x9661b9c2, 0xb6223988, 0xb8aab797, 0x305c254c, 0x271bbd53,
+ 0xbaa93c11, 0x3601b75b, 0xb4e936d1, 0xb15db856, 0xc0b2be96, 0x3e32b7c5, 0x3e22b069, 0x3b41bc18,
+ 0x31cac16e, 0x35f9365a, 0xb700b491, 0xb41b3a6a, 0x406a2f26, 0xbd54b8a6, 0x39e33b9b, 0xad0fbeae,
+ 0xb1663b1b, 0x3655afd4, 0xbaa7bde7, 0xbd6c37c1, 0x3cbfbc6d, 0x2dbd3584, 0xb9f93d69, 0xac53b992,
+ 0xb2a2bf6b, 0xc15735b5, 0x328e331a, 0xb7b3ba14, 0x38093c44, 0x33613c08, 0xbc1c3b93, 0xbd213b53,
+ 0x39b2b9d1, 0xbad9326b, 0x35e6b1c8, 0xaeff38ff, 0xb734b831, 0xbbe1ba96, 0x36fb33f8, 0xc15ab049,
+ 0xb7a33b22, 0xbc3db462, 0xb57cbf97, 0xbcaa3c86, 0x3c6ebcb9, 0xb0f539ca, 0xb5bf379f, 0xbcb6b629,
+ 0x3f0f3aab, 0x34fcc01a, 0xb6be3869, 0x32f4b953, 0xbdb7b85f, 0x411a39f4, 0x203cbce8, 0x2a593824,
+ 0xa60cb26f, 0x3d96b47f, 0xb53fb8d0, 0xb69f3706, 0x36cfb1c9, 0x37c1ad03, 0xbc903d80, 0xb98eba26,
+ 0x334b3c5b, 0xb051c196, 0xbd79bd24, 0x3e2f3b38, 0xb64dba82, 0x34ed2a56, 0x2a01b36f, 0xbfbc40e9,
+ 0xb600b7ce, 0x3a9c3cce, 0xb23c33c6, 0x36c83c33, 0x409ea776, 0x3c35b822, 0x30ef3e43, 0xb9f7b9b2,
+ 0xba813c79, 0xaedb3973, 0xba33bfe1, 0x3d8a348e, 0xbfa1b5ac, 0x31b7b8d9, 0x34813aee, 0xb56e3d3b,
+ 0x2c653d98, 0x3d7439e0, 0x3ae2b51a, 0x3bb9ae6c, 0x1d86bdc8, 0x3840b9d9, 0xc0f1bc60, 0x396d3e26,
+ 0xa4783dc5, 0xc01bbf35, 0x39c8b938, 0x3fd035d1, 0x3d41b3bc, 0x3bf9bca8, 0x32f5bd36, 0x34c83f59,
+ 0x38543e38, 0xb82e338e, 0xa3ce37b2, 0x37fdb683, 0x3f02c104, 0x38b3b92f, 0x3bcdb454, 0xba4231f4,
+ 0x3878bf4d, 0x3559bf7f, 0xbc53b8e0, 0x407ebd77, 0xb0c6b69e, 0xbabdacce, 0xb92f3c1b, 0x3a5fb8fa,
+ 0x3f13b628, 0xbacd3893, 0x3ec8c047, 0xba7fb70b, 0xb140b997, 0xb4662c0c, 0x3ddeba3f, 0x3ae1be75,
+ 0x3879b466, 0x3c553d68, 0x3c6eb4d9, 0xb803be71, 0x2a523e15, 0xbc373e1b, 0xb7473585, 0x3db4bf22,
+ 0x31a6bcdb, 0x2e23b415, 0xb884ab25, 0xb5ef329e, 0xb59c3525, 0x39c2ba76, 0xbf1eb6f5, 0x2dadbcdf,
+ 0x40193e8c, 0xbb4ab92c, 0x2ddb3494, 0x3afd406e, 0xb97a3b91, 0xbce13c1f, 0x3c343eae, 0x38ecbddd,
+ 0x3e4f390f, 0x3e04bc20, 0xba0b378c, 0xb8aa330d, 0x348e3e34, 0x352fbaa5, 0x3d2bbac3, 0xbe71b982,
+ 0xc1e1305c, 0xbe12be05, 0x2fc1b70b, 0xbe713c8b, 0x3dcc35ad, 0xb612b581, 0xb50cb42c, 0xb36cb1c9,
+ 0x3977b443, 0x3d11bd4c, 0x3b1739b8, 0xbd27269a, 0x3d6cbce0, 0xb611b86d, 0xb50c35e7, 0x359cb453,
+ 0x38bf38c6, 0x39a6b82a, 0x3bb1a8c2, 0x3c8bb020, 0x2dfa2c82, 0xb43ebb37, 0xbbbd2b23, 0xbd253c97,
+ 0xbb813d21, 0xbe12b8e5, 0x3d6a3845, 0xc0ecb571, 0x34ba3c69, 0x2818b96f, 0x382dbdde, 0x3f333f0c,
+ 0x3c30b44b, 0xbef5aebd, 0x26833a3b, 0xb7f8aaf0, 0xb4f5405e, 0xb1973c5b, 0xbada359f, 0xbb773227,
+ 0x3dec3877, 0xb7c634a2, 0xba0e3bde, 0x20ccb758, 0xb8e83760, 0x3cb8bdaf, 0x385d3cf3, 0x3078306f,
+ 0x34d1ac7a, 0x402fb9b8, 0xbc802d2e, 0xbeefb55d, 0x356a348e, 0x3b683511, 0x32ea35a4, 0x3a133e99,
+ 0xb8c94002, 0x355a3dc1, 0xbb783556, 0x37ee3560, 0xbc4aba6e, 0x33ebb14d, 0x3c4a22a9, 0x3a27bd5b,
+ 0xba1dc0ca, 0xbd314053, 0xb87b3c1a, 0x36e338de, 0x2b952663, 0x36853b4f, 0xc020353c, 0x4050be97,
+ 0xa53abc48, 0xbeea3e4e, 0xbb52ac24, 0x40d2b743, 0x3762b389, 0xbb02b493, 0x342f3cb4, 0x3aceba83,
+ 0x3923bc28, 0xc0afc004, 0xb842b898, 0x276038d0, 0x405eb83e, 0x4254affb, 0x3c6f3b69, 0x3bedba70,
+ 0xba70b0de, 0xb6353d9c, 0x3de8bde3, 0x38e53321, 0x40ae369f, 0xbaf236e3, 0xbb9baf7b, 0xc1f3b484,
+ 0x40973f5a, 0xaead3da5, 0x1e3a3934, 0x409fbc67, 0x3a55b2ed, 0x385a3009, 0xb6ac3166, 0xc3f83832,
+ 0xa95fb800, 0x2f61bc8a, 0xbeba353c, 0x3ce33284, 0xade83fad, 0xb7c13da0, 0x382db56e, 0x3855385f,
+ 0x2ff5bd34, 0xb290ba07, 0xb6f4bda4, 0x3ad2b89b, 0xbe273613, 0x35fc3838, 0x3c9628e1, 0xbd623c1e,
+ 0x356b315a, 0x3b163b05, 0x3ed5baf3, 0x3c65bf42, 0x378a3d3f, 0xbd43bf4e, 0x39643c0d, 0x3c0034c3,
+ 0xbafe345a, 0xba8bbf4f, 0xb95b4240, 0xc0a23b21, 0x331db6ed, 0x3111b57a, 0xb8ca3594, 0x3d96c16a,
+ 0x3c86b7a9, 0xb498b949, 0x3c1e2d89, 0x2eebad06, 0xba8abc5b, 0xb0b9be54, 0x3e71bc9a, 0x3ff43ca7,
+ 0xbbbcbf8b, 0x3fefb894, 0x40c93bda, 0xbb663bc7, 0x3cbbb87e, 0xb1f7bb1e, 0x38f63a3d, 0x3cd9ad22,
+ 0xbd81bdb4, 0xbd96b005, 0xb91c3d6e, 0xbd5e38f2, 0x2e27b513, 0xbe1c3e5b, 0x366e34b3, 0xb6cdbb24,
+ 0xbfb436f7, 0x34a3b00c, 0x37dfb1bf, 0x33d3b9d6, 0x3edf396f, 0xb6f7b5d1, 0x38b22aa1, 0x223d36b1,
+ 0x3d3ebc6a, 0xb7b5b777, 0xb91cb8ed, 0x35f53be6, 0xbd74c077, 0x3966bcdc, 0x3d0bbc12, 0x28cb3312,
+ 0x4128bd0d, 0xb80ebef1, 0x3eb4b758, 0x352fb6d3, 0x3ca3343d, 0xbaefb733, 0xbc1dbce4, 0x38fa3eac,
+ 0x3bceb9d1, 0x36ec3953, 0x3985bbb4, 0x3fe24005, 0xb9913f4e, 0xbe2c39d5, 0xb37fb44e, 0x30aa3acf,
+ 0x40c3b358, 0x3b9b3c1f, 0x39a739ec, 0x41113d01, 0xbb76367d, 0x3a483c75, 0x3c01b547, 0x3ae03d2f,
+ 0xb837c3b1, 0x36c12e1d, 0xbd8eb2d3, 0x3d80b23a, 0x3c9bbd29, 0x348c37a8, 0xb5aa3c9e, 0x40d33311,
+ 0x4028256d, 0xa66e2e47, 0xb8de390a, 0x3c033306, 0xbb3fbc58, 0xbc21bc05, 0x28e2376c, 0x3ddab536,
+ 0x3ebb2427, 0xbdbfb70a, 0xbb103cc4, 0x2d293d33, 0xbfd0c044, 0x3726bfd3, 0xad3432d9, 0x3d32c040,
+ 0x2b123f66, 0x3ae9377a, 0xc02d3088, 0x38e2384f, 0xacfb3c2b, 0xb9debbda, 0x39452d0a, 0x40ba38f1,
+ 0x1b913858, 0xba152f58, 0x3a7f3642, 0x387a39d4, 0x3efcb9af, 0xbcc6c007, 0x3938ba9b, 0xaa27b635,
+ 0x3509a2e3, 0xbb66b38b, 0xb775bbc3, 0x33f5bbc7, 0xb6bac014, 0x340ba91a, 0x3dadc0ad, 0xb70ea94d,
+ 0xbcf0b9b4, 0xb38a2f73, 0x304e34f1, 0xbfe63cdf, 0x34c3c329, 0xbdc03c80, 0x39ed37bd, 0xaaa3393d,
+ 0xbd493a94, 0xb4d2c288, 0xbfb62c3e, 0xb93dbca0, 0x3200b9d6, 0x3875ac36, 0x33f73072, 0xb8f73e47,
+ 0x3116ba54, 0xc0373a12, 0xb48c3ad7, 0xb798323b, 0xbf50c11c, 0xaf0baf13, 0x2efc3c1c, 0xbc683b04,
+ 0x38fab611, 0x3f8cb74f, 0xbf1a3b22, 0x3cefc394, 0x26143ca6, 0xbe0337c3, 0xc0723cd0, 0xc15f3f2e,
+ 0x34da3c11, 0x3cf5b4e7, 0x3030c030, 0x3837b32a, 0xbca2c1c0, 0x3ef133dd, 0xac37982a, 0xb7b33c5e,
+ 0xb9cdb5b6, 0x2c0cb9e9, 0xab2a3832, 0x419aacea, 0x39223458, 0x3c1fc18e, 0xb7622ae8, 0xbe8c393d,
+ 0xbd47b009, 0x3abd3d9b, 0x3c56b95a, 0x31073cfe, 0xbc5637d9, 0xb846312d, 0x3d2e3752, 0x403041d2,
+ 0x3bf0b2d1, 0xbc3930c5, 0xb8aab1ee, 0x3d48405a, 0x2a023c11, 0xc184b800, 0x308d3dd2, 0xbd48b975,
+ 0xb94232b5, 0xbe0e38c6, 0xc06f3688, 0xbc6133b3, 0xc0943910, 0x38f83cb1, 0xc2b8be18, 0xb3de37d6,
+ 0x3b2eab85, 0x299cb8f6, 0xbdb13676, 0x3f26b87d, 0xc1173068, 0x3a16b7ae, 0x3b4430ea, 0x30a9b347,
+ 0x39c331a2, 0x302136bf, 0xbc1338f6, 0x338db882, 0x3bf4b892, 0xba38385d, 0x305d3b4e, 0x387b43aa,
+ 0xb0c436a4, 0xb925b800, 0xbdefb9d7, 0x4244bba6, 0x345236da, 0xc1913b75, 0x318b381f, 0x405a38a3,
+ 0x3d74ba2a, 0x3666b9b5, 0x38023990, 0x37ccb8ed, 0x36e93697, 0xb3ef3d43, 0x3cbfbaa8, 0xadc4bdf3,
+ 0xb0b62c73, 0x386abbd5, 0xbecebaaf, 0x3d5835f2, 0x3ca3358e, 0x3d37b987, 0x3cba3770, 0x400f3d8a,
+ 0x391037a5, 0x31483bce, 0xb1f2b10b, 0x3a6d343f, 0xb733399b, 0x355038b6, 0xb90abe87, 0xbe4bc1ea,
+ 0xbe8fbe96, 0xb92a3999, 0xb1c83a77, 0x3508366c, 0xbcc4b7aa, 0xb591b03d, 0xb9fcbecf, 0x40f5bb2f,
+ 0xc0db4073, 0xbf44421c, 0x34cd339f, 0xb4dc37bd, 0xb44cb531, 0x3b383e9d, 0xa9ffba12, 0xb9ce3655,
+ 0x3441b32e, 0x3c083f6f, 0xb8603a43, 0x3dfebcf7, 0xbc86c07a, 0xbbf63745, 0x36a62987, 0x267f35e0,
+ 0xb9f1b4b8, 0x3a39b865, 0xbd262f53, 0x388c37b5, 0xb8893b0e, 0xa4223c18, 0x1666b894, 0xaf72b37f,
+ 0xb0813772, 0xb0eac090, 0x41d033ad, 0x345fb75b, 0xba823de3, 0xb979afd1, 0x36ec3f52, 0x42c7b928,
+ 0xb54ebc71, 0x356d3b45, 0x3d77b63d, 0x2873a722, 0xac223ab5, 0x3ccbbe38, 0xba9db82f, 0x34f9b88a,
+ 0xb9973dde, 0x399dc1b1, 0xc19a3d55, 0x3e72b380, 0xbd0ab701, 0xba7fc16c, 0x3df039a8, 0xb595bf04,
+ 0xb904ba46, 0x35c93ab7, 0xb9dab181, 0x3a40ba29, 0xb257bee1, 0xbabbc262, 0x39fa3d35, 0x39222661,
+ 0x3c35b5da, 0xbc02ac1e, 0x375bb9c2, 0xb5e8343c, 0xbd4abc03, 0x38402c70, 0x3a48b98c, 0xb987bd06,
+ 0x3a3fc008, 0xadf03676, 0x437f391e, 0x38df4179, 0x3dd23a92, 0xb870b42f, 0xb0b33e99, 0x3c3b3136,
+ 0x3b57b98b, 0x3d80af33, 0x3475b578, 0xb6203f58, 0x386c3a60, 0xbc68af9c, 0xb99a3826, 0xb414b8c8,
+ 0x374bb952, 0xba0c40e7, 0x40723a49, 0xbc2ab76d, 0xaebbbc67, 0xba0d3a0b, 0x3e433f56, 0xbc433d02,
+ 0x3e82b9a2, 0xb7963a1f, 0xbc5c355d, 0xba8a3a70, 0xbd61333c, 0xb9c73caf, 0x31f0bd93, 0xbce5bd05,
+ 0xba81b4b9, 0xbbf2bc7d, 0xb9763740, 0xbe7bb53f, 0xb266bced, 0x3bda30be, 0xbab03f03, 0xbfc73bdf,
+ 0xbf88adc6, 0x2c42b00b, 0xc2c93e4d, 0x4114be94, 0xbcba3b5a, 0xbd313d7c, 0x37ac3726, 0x3a3a397e,
+ 0xbcd02f06, 0x3cb93d0c, 0x3f3735a6, 0xb16141f6, 0x3e57ba84, 0x3e1eb423, 0x2e083eb8, 0xbc7fb1c0,
+ 0xc03a3bef, 0xbb16bcc4, 0x3c4a38a5, 0x3bf83b7f, 0x296439c9, 0xb944bddc, 0x3717406a, 0xaef2b80f,
+ 0xbf1438c2, 0x37eb3633, 0x3d192c27, 0x37d2ba23, 0x382331dd, 0x39a3bb12, 0xb57cb985, 0x36a5b7fa,
+ 0x3b5bb882, 0x34c03856, 0xb60d3a69, 0x3b6ec03c, 0x219c3b95, 0xbdd6ac3a, 0x356f4008, 0x33cbae4c,
+ 0x3c41bd84, 0x3b3c3ba0, 0x387ebb56, 0xba143b22, 0x3f8d33cb, 0x18e03508, 0xa89e3b2d, 0x3c7f3c83,
+ 0xb82a3cd9, 0x391239b5, 0xbb1c39d4, 0xb1deb8ed, 0xbb923b0b, 0x41783be0, 0x31e83716, 0xbc80b3ff,
+ 0x393eba07, 0xba8739a9, 0x4185bb2d, 0x32fdb616, 0xbbe7b9d6, 0x3cee3645, 0xbcbab4c6, 0xac743e24,
+ 0xb9ba2b37, 0x31423c42, 0x33e93505, 0xb53ebe07, 0x3b1db609, 0x36e4bd2b, 0x36fb3c8a, 0xb8c339e9,
+ 0x42953fba, 0xbf1a3766, 0x3ca339e3, 0xb4343fc7, 0xbb95b899, 0xb2e3398e, 0xb5c8bcd6, 0xb44238c5,
+ 0x407d3cf2, 0xb847b6e2, 0xbdcd3a6a, 0xb34234b0, 0x2d943a83, 0xb6443e96, 0x3e5dbead, 0x39cdb0bb,
+ 0xba58a92f, 0xbdf93c0b, 0xa45939ad, 0xbd5c3e21, 0x41c83d83, 0xbeb73299, 0xbc3ab260, 0xb98cb6c7,
+ 0x3d96321a, 0x3a6f3834, 0x3d353920, 0xbe2828a0, 0xb9dbbe36, 0xbbbab7d3, 0x357fbbd1, 0xc114bd91,
+ 0xae9bb5fd, 0xb85bc078, 0xb4c3c123, 0xb79638b3, 0x3ab8bb1b, 0x37e533b3, 0x3a311a62, 0x3a67b4f1,
+ 0x3e7bbbce, 0xbc7cb73c, 0x4098408e, 0x342bbe64, 0x39c73da6, 0x1dea4077, 0x3c46af4e, 0xbbe83ebd,
+ 0x3cd83936, 0x3e99402f, 0xb2d23c05, 0xb1ebbc89, 0x3d842dfa, 0xbb80bed9, 0xbb7238a5, 0xb173bc2c,
+ 0xb35db84d, 0x3e9fb0ff, 0x3b57bc79, 0x307dbb5b, 0x3703b498, 0x38bebe4a, 0xab2eb878, 0xbacc3b46,
+ 0x32d83ab4, 0x365bc20b, 0x36ed35ac, 0x2d1b40e5, 0xb0f7b594, 0xbe4cba7a, 0xb070a99c, 0x2dcab21f,
+ 0x3fd8bf8a, 0x391bb83e, 0x330fbc35, 0xb69834e7, 0x34c1b482, 0xb81836cd, 0xb4493e69, 0x31ab3646,
+ 0x341e38c2, 0xb9873925, 0x41b938e5, 0xb1acb79d, 0x38f936c6, 0xa977c103, 0xbce0bead, 0xb897bc63,
+ 0xb8423f5d, 0x38753d63, 0xad0bab3a, 0xbcd625c0, 0xb85138d6, 0xc077b9a9, 0xb4a13be3, 0x39c4c293,
+ 0x414b32dc, 0x38cc3ada, 0x3b60bad6, 0xb52e3a85, 0x3d6d3d6c, 0xb855b5a4, 0x27f62ccf, 0xbcf635b1,
+ 0xb4a034df, 0xb62abe85, 0xad0d39d4, 0xab88b542, 0x345a35dd, 0x3951353b, 0x2f10b8f8, 0x363fbaae,
+ 0x3ca23878, 0x38da35df, 0xbcfdb443, 0x2fdbb87e, 0x36abbc75, 0x3243389a, 0xba7029dc, 0x32502ee6,
+ 0x3d29b748, 0xbe2140a9, 0xb60dbba6, 0x3c40bbbc, 0xbb7bb5dd, 0x3bf32631, 0x2ad5bfbb, 0xb41735b7,
+ 0x3831afde, 0x3145b651, 0xb23f3cfa, 0xb56f39ff, 0x3f03bd12, 0x362333d8, 0x3e043401, 0x3f6bbe9f,
+ 0xa9d730db, 0xb34d31d9, 0x32529f5e, 0x3c75c0f6, 0xb8ec348d, 0x384138d0, 0x20a83abf, 0x2b97c090,
+ 0xac01bf35, 0x3722b854, 0x344dc074, 0x39723a91, 0x39b9be82, 0xc02ab62c, 0xba61bb3a, 0x3a292fcb,
+ 0xbb4eba51, 0x3d17b604, 0x3b16363e, 0x3ab93c14, 0x3a9cb80d, 0x3936b0df, 0xb8cab50c, 0x35f3bd85,
+ 0xb02b2ffc, 0x38fec0b9, 0xba48390d, 0xbc03408f, 0x3646b937, 0xc0c1c22b, 0xbad0bab4, 0x99a8b3f6,
+ 0xbf40bc8c, 0xbc802074, 0xb89bb927, 0xa9e533be, 0x3229417f, 0x353cb346, 0xb883395d, 0xbadd3d69,
+ 0xa7433c6c, 0x3466bd97, 0x3977bf85, 0x39763618, 0xb9f83e77, 0xc024be06, 0x325abc84, 0x3d18bb6c,
+ 0xbca2b930, 0xbfb2b806, 0xb2533578, 0x33f1b48c, 0xbb21b9ac, 0x374ab012, 0x3c0bb0cc, 0x3dc8aca9,
+ 0x322cb4d9, 0xb993ab2c, 0x3a913aa6, 0x3da642a1, 0x3e593151, 0x2d8e3751, 0x2d9a3365, 0xb77fbaaa,
+ 0x32eeb7f0, 0xbb783dd2, 0xbe1c3d0e, 0xbd05be19, 0xbebdbcfe, 0x36b0bba1, 0x40063962, 0x370d380e,
+ 0x3b9c3e08, 0xb2a82bfe, 0xc0383dd4, 0xb32cc00f, 0x39f333b5, 0xb8933691, 0xac1fbb79, 0xbd823d09,
+ 0xb867afcd, 0xbbb5c01c, 0xb323bbca, 0x3d5db8d8, 0x2db4bbda, 0x38f9b8ca, 0x3a87b615, 0xc099321c,
+ 0x3f8c3a26, 0x36f130a7, 0xc2a5386f, 0xb157b485, 0x3f94b4d2, 0xb1bb2056, 0x35dbbb33, 0xba5135b5,
+ 0x845ebe3f, 0xbd67b7cd, 0xb6702dae, 0xc11f399d, 0x378e2c99, 0xbe92b735, 0x3c614021, 0x34c2ba18,
+ 0xb927b993, 0xb82ab98a, 0xbf68ba9d, 0xb7f9b507, 0xb71840e7, 0x3cd31e42, 0x3a822b48, 0xbf983ef1,
+ 0x3a56b0ad, 0xbc1234a6, 0x31d63fe4, 0xa3153d63, 0xb81a22f5, 0x32d73b45, 0x36893e26, 0xbc0c3d81,
+ 0xbcd33d04, 0xb5c836b2, 0xb86b37bb, 0x3c81b658, 0xba20b81b, 0xadad356a, 0x3048be5d, 0x301db9d4,
+ 0x3d2dadb0, 0xb415aa8b, 0x413fb4b2, 0xb5163c3c, 0x3572aff4, 0xbbf6b682, 0x3bd3bbce, 0x3d0a3a19,
+ 0x35923729, 0x3a21bc5f, 0xb6673c5b, 0x38c73c4b, 0xaecd3180, 0xb9d9bd82, 0x36d1ac5a, 0x2c34c22c,
+ 0xb450bcae, 0x388f3f37, 0xb5fdb698, 0x3b4a23d2, 0xae67b721, 0xab023cbc, 0xbd7339d8, 0xbaf198eb,
+ 0x381e3ad1, 0xbc12b864, 0xc084c0e9, 0x39713831, 0xb2fa3ca9, 0x3708a167, 0xb9ca405f, 0x31ad3ca1,
+ 0x339d383f, 0x364f389a, 0x3db4b77d, 0xb2bcb56a, 0xb4ddba8f, 0x32b6bf46, 0xb90a381e, 0xbf8a3db5,
+ 0x37b4bc4e, 0xad30bc0e, 0x3d833c84, 0x3a043a52, 0x4027bc64, 0x3e6d3c18, 0xaf48bcce, 0x325cb052,
+ 0xbf1db876, 0xaf28bafa, 0x34983a5b, 0xbc54aab7, 0x3d7e3fdb, 0xbf2039e0, 0xaddcbfc8, 0xb83bb815,
+ 0x3798b793, 0xb6c53915, 0x395bbc75, 0xb960358f, 0xbcedbc4a, 0x34c2bb4c, 0x20223cfd, 0x30cf2cce,
+ 0xbc4db632, 0x33083a35, 0x38d22f07, 0x3c45bc58, 0xb9adbae1, 0x3910bb37, 0xb91ebf4b, 0xb27e374d,
+ 0x367bb59a, 0x39fb3af6, 0x3a2abb7c, 0x2d37bf2d, 0x3803b489, 0xbdc63fc6, 0x3fe4381d, 0xb50833f7,
+ 0xbeb94023, 0xb0c63d4c, 0x38c5b9aa, 0x360ab65d, 0x3ad7bd31, 0x39bf3f3e, 0xb920bc24, 0xb7a33774,
+ 0xbcb9afd2, 0x387ab627, 0x38cbb7cc, 0xb37eaf26, 0xbad1b2c5, 0xb982b881, 0xbf5fbe79, 0x3cb6b58c,
+ 0xc084b76a, 0x305f3888, 0xbb1c4386, 0x2990318f, 0xb415bc45, 0x3b9abd45, 0xb6c7400b, 0x3f4940d8,
+ 0x39533b3e, 0x41a54121, 0x3a863e15, 0xb50cb745, 0xbc62394d, 0x36fd3ce8, 0xb7d3b3fe, 0x3663320d,
+ 0x39e838fd, 0x38c0b2dd, 0xb5163992, 0xbc1fb019, 0xa8c635e3, 0x39753fda, 0xc0dd3dba, 0x3863166a,
+ 0x3ab2a3cd, 0xbcb9332f, 0xb88ebce3, 0x41f6bd5e, 0xbd0aba72, 0xc170bac9, 0xb615bc26, 0xbca7bab0,
+ 0xbb6cc082, 0x415e3f5b, 0xbe9b3be8, 0x3f323c55, 0x410aa521, 0xbe1ebf27, 0x39dfbca8, 0xbbf5b924,
+ 0x39c7b2aa, 0x424b3af7, 0xaed0b4c6, 0x3532358e, 0x39243116, 0xc199bee5, 0x3849ba6e, 0xb9203c77,
+ 0xb4813c82, 0x3c4e33de, 0x3775ba98, 0x96a7b594, 0x4000bd85, 0x3fd82fd0, 0x3bb73d71, 0x38083dc5,
+ 0x34a8bc1c, 0xb86c3d3d, 0x38413bf1, 0xb9d3399c, 0xb752bcee, 0xb00a3ab9, 0x3198b744, 0x398f2fcd,
+ 0x3d759a32, 0x3bbc360a, 0xbb5937e9, 0xb330b969, 0x3ae6c0a5, 0x40b03e0c, 0x343cbcda, 0xb86c34bc,
+ 0x35be3cb0, 0xac1bbd82, 0xa5a7bb08, 0xb0f534bf, 0xb4853087, 0xbb9ca123, 0xb45d3df9, 0x404c3c17,
+ 0x39a3347a, 0xb913b6e8, 0x38cebd20, 0x40b0c006, 0xbb4d402a, 0x3e493e7d, 0xb6c83b8b, 0x3b902d82,
+ 0xae29388d, 0x3ad7327b, 0xb9843d10, 0x34cc3933, 0xbbda3002, 0x3819bcf3, 0xbc312c89, 0x38e83dbd,
+ 0xbb87b4e4, 0x3886b172, 0xb82939b9, 0xb345bd54, 0xba7fb951, 0x36c13d9f, 0x3dd3b259, 0x36a43a49,
+ 0x3cf23b65, 0xbe2cacae, 0x3ba3bdae, 0x3603356a, 0x33703d0e, 0x2d88b96b, 0x36e3b8d3, 0x3b24b804,
+ 0xbdd0c000, 0xb8c7c006, 0x33673dc9, 0x344bb663, 0x341f3406, 0xc0cfac64, 0x3d0234c6, 0x39f83676,
+ 0x2e04a4b0, 0x3b5fb4ee, 0x37d4bb6d, 0xb491b0cc, 0xacc8b1aa, 0x3c6cbea5, 0xbc46b99e, 0x3b6cbe17,
+ 0xbf93bd5d, 0x38fab462, 0x331fc0e4, 0x31643419, 0x4066bb25, 0xbe28bd61, 0xbc663891, 0xbc0c32be,
+ 0x3dee3055, 0xb3ab20f7, 0x3a5031ff, 0xbad6be6a, 0x3cb5bb8f, 0x3e97be0a, 0xbc8b3110, 0x3c1b36cb,
+ 0xbe5fbb83, 0x310baecd, 0x2985b4b8, 0xb9d93c4a, 0xbc1d39c5, 0x3d9e390b, 0xbca43675, 0xb49c3ce1,
+ 0x3fffbb08, 0x35e33927, 0xbc60bd50, 0xbd1339b6, 0xc2a24123, 0x37d9beab, 0x3425b8ed, 0x3a36bace,
+ 0x4057b0c7, 0x3e0c3c69, 0x3a03b5b0, 0xbaa0ba05, 0x9c4c19c4, 0x3c37b72f, 0xbd023467, 0xb537b522,
+ 0x2c4d3673, 0xbdefb17b, 0x3daab95a, 0xb4e3bbe8, 0xbb3a3034, 0xbd9cc116, 0xc1d53891, 0xb874bd9f,
+ 0x357fbc4b, 0x3f79b09a, 0xc07e3f0c, 0xbca2bbfe, 0x2af5bb50, 0xbaf7b02f, 0xb509b629, 0x36fc39c8,
+ 0x395ca825, 0x3553b901, 0xbbebbd9d, 0xb45139f2, 0x3f3c37bc, 0x386635a3, 0x35fb3046, 0xbcb33999,
+ 0x3d1da417, 0x3becbc37, 0x3bfa3c7c, 0xc1d5bcb8, 0x3c403c1a, 0xb73bb23c, 0x3a093546, 0xbb7fbbf5,
+ 0xbe3f30e8, 0x3a8b39fe, 0x9b02341a, 0x2b91bc35, 0x3a514256, 0xba433fd1, 0x393ab55c, 0xae6a3c6f,
+ 0xb2663da8, 0xba78bd17, 0x3e3a35cf, 0x3e4db5dd, 0x32ff345d, 0x35a2bca2, 0x3c82b9f1, 0x38f1bbc5,
+ 0xbe7b3f36, 0xb44abe77, 0x3501bd3b, 0xbe42344c, 0x3dc3bc81, 0xbf152613, 0x3c71bb1c, 0xba6a345f,
+ 0x3c03b944, 0xbae1b5ea, 0x3e77ab67, 0xb9dbc04f, 0xb86ebe0c, 0xb9cdbc4f, 0xb07a3ca8, 0xb41f36be,
+ 0x3256ac27, 0xb3273870, 0x3b9db5f6, 0xb8de30ba, 0xb06a3c1b, 0x3d543ae1, 0x3d08ba28, 0xb93838a3,
+ 0xb96b3472, 0x3b803945, 0x3ab1adbe, 0x39b53ccf, 0x383634a8, 0x308cb49e, 0xb5ba3916, 0x3b1bbb8a,
+ 0xbe6fbc56, 0xb4d5bd9e, 0xa3433e83, 0xb2fc3adc, 0x3b55ba2f, 0xb6243cec, 0x37d7261f, 0x3eb1bf76,
+ 0xb9c838b1, 0x361ca60d, 0xc00cbca6, 0xbc6bbb55, 0x38ba34c5, 0x383cb802, 0x3e68bd59, 0xbbad3a2e,
+ 0x3b243c74, 0x3ebf3010, 0xbfa8b953, 0x3b7cb16b, 0x2c823952, 0x412cb8ef, 0x3cdbb7ec, 0x38e03bc9,
+ 0xbaa3bbfb, 0xb1df3f28, 0xbccf3a77, 0xb933b1a5, 0x37cb3018, 0x31bcbec0, 0x3997bd2e, 0x39063930,
+ 0x37fdb86f, 0x33e5323d, 0x38d7a60a, 0xb451b30a, 0x31233597, 0xbe8ac063, 0x37afbe42, 0x3b21aff0,
+ 0xb277365f, 0x2ad2b5b8, 0x3a45c11d, 0xb50031fd, 0xbc72bf3f, 0x24a33038, 0xb5952890, 0xc1c7c08f,
+ 0xbcaab86e, 0x38a63189, 0x325db408, 0x3b052ca7, 0xbf353c55, 0x3b7e387c, 0x3a0b3c20, 0x3db83741,
+ 0x3c4d3a12, 0x3d11b87c, 0xbed7bdc9, 0xbdbb2444, 0x3cc9b9c6, 0xba10ae72, 0x395338b7, 0xbbe2b1e6,
+ 0x30b1a918, 0x283dbc8f, 0xb160aeb3, 0x33363794, 0x401d3327, 0x3d15bf99, 0x3a32ad02, 0x3c573df6,
+ 0x34edbef4, 0x368cbc1f, 0xb95b3aed, 0x3bf338dd, 0x3c2f4185, 0x37b5bbfc, 0x355e37f3, 0x3c463559,
+ 0x38bcc003, 0xbda03b53, 0x2acbac8d, 0x396cb8ac, 0xbb1f2be0, 0x3b903ce8, 0x3ef8ae9f, 0x3cf4b33e,
+ 0xb8982c97, 0x3f56baa5, 0xbb163e58, 0xc004b33c, 0xbc68400e, 0xa93938bd, 0x3b353a55, 0x3db540d6,
+ 0xb8694176, 0xbd7f30ba, 0xb379ba99, 0xb9153ecf, 0x30f2b19f, 0xb6323e11, 0x3be5be94, 0xac5d3d28,
+ 0xbb64b6e7, 0x4380b282, 0x36b5bde1, 0x386736c2, 0x2e1c3cca, 0x2e2f31cc, 0xbcffb0d4, 0xbd67bb1c,
+ 0x3d2c309a, 0xbd03ab7d, 0xbefcbbe1, 0x3b98bfbe, 0xbc72b53c, 0x3cca2f15, 0xbc36c106, 0x3b57b702,
+ 0xb4da391a, 0x3afe3861, 0x335b3ea8, 0xbd6234e4, 0xbc863f1f, 0x35bcbcf4, 0xbf53b34a, 0xbd103834,
+ 0x39a1273d, 0xb4e4b839, 0x34da39b8, 0x3a76bed6, 0xc0fa3c50, 0xb061bd35, 0x34f0b1b6, 0xbc183ab1,
+ 0xb62cb91d, 0xb829bb36, 0xafabb940, 0x3e813d46, 0xb822b20b, 0x35fdbe39, 0x396632f1, 0xb5432e17,
+ 0x3a89bbed, 0xbcc5b2a3, 0xb42b3369, 0x1f58bbd4, 0x3ef4bc48, 0xac2b3bbe, 0xb2e4bdf5, 0x3f94bcae,
+ 0xc068a11a, 0xb82e35bb, 0xbbb93f77, 0x3b46b386, 0x22f3b675, 0xbcb23e8b, 0xba29b8ba, 0xbd084235,
+ 0x3f9f1038, 0x399fade8, 0xb9da3363, 0x37c83909, 0xbc4c3c82, 0xb788bc3a, 0x39daa897, 0x3beeb6d2,
+};
+// END tq_lut
+// the trellis codebook point i (two fp16 values in one word): from the constant copy, or the threadgroup one
+static inline uint g_tq(threadgroup const uint* tab, uint i) { return ctab ? tq_lut_u[i] : tab[i]; }
+
template <> inline void load_tab<T_Q2_0>(threadgroup uint*, ushort, ushort) {}
+template <int F> static inline void load_tab_tq(threadgroup uint* tab, ushort tid, ushort nth) {
+ if (ctab) return;
+ for (int i = tid; i < 2048; i += nth) tab[i] = tq_lut_u[i];
+}
+template <> inline void load_tab<T_TQ2T>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQ2T>(tab, tid, nth); }
+template <> inline void load_tab<T_TQK6>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQK6>(tab, tid, nth); }
+template <> inline void load_tab<T_TQK7>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQK7>(tab, tid, nth); }
template <> inline void load_tab<T_Q2_0R>(threadgroup uint*, ushort, ushort) {}
template <> inline void load_tab<T_Q8_0>(threadgroup uint*, ushort, ushort) {}
template <> inline void load_tab<T_Q4_0>(threadgroup uint*, ushort, ushort) {}
@@ -304,6 +581,47 @@ template <> inline float deq<T_Q5_1>(device const uchar* b, int, thread float* q
return rdh(b);
}
+// Trellis types (ggml-common-tq.h): a 128-value block is {fp16 d; qs}, 32 steps of K bits (K = 8 / 6 / 7). A unit is
+// lane ui = steps 4ui..4ui+3, 16 weights. Step t's 15-bit state: TQ2_T qs[(t + 31) % 32] << 8 | qs[t]; TQK the 16
+// stream bits from bit (31 - t) * K, circular over 32 K bits. x = state * 0x9e3779b1 selects two codebook points,
+// lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs); the four weights of the step are those four values times d.
+// Same arithmetic as src/kernels/cuda/iq_kernels.cu (tq_states4 / tq_step) and ref_quant.cpp (deq_tq).
+template <int KB>
+static inline float deq_tq(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) {
+ device const ushort* qs16 = (device const ushort*)(b + 2);
+ uint s[4];
+ if (KB == 8) {
+ const uint w01 = qs16[2 * ui], w23 = qs16[2 * ui + 1];
+ const uint prev = b[2 + ((4 * ui + 31) & 31)];
+ const uint b0 = w01 & 0xFFu, b1 = w01 >> 8, b2 = w23 & 0xFFu, b3 = w23 >> 8;
+ s[0] = (prev << 8) | b0;
+ s[1] = (b0 << 8) | b1;
+ s[2] = (b1 << 8) | b2;
+ s[3] = (b2 << 8) | b3;
+ } else {
+ constexpr int NW = 2 * KB;
+ const int off3 = (28 - 4 * ui) * KB, wb = off3 >> 4, r = off3 & 15;
+ const int wi1 = wb + 1 >= NW ? wb + 1 - NW : wb + 1;
+ const int wi2 = wb + 2 >= NW ? wb + 2 - NW : wb + 2;
+ const int wi3 = wb + 3 >= NW ? wb + 3 - NW : wb + 3;
+ const ulong v = ulong(qs16[wb]) | (ulong(qs16[wi1]) << 16) | (ulong(qs16[wi2]) << 32) | (ulong(qs16[wi3]) << 48);
+ for (int i = 0; i < 4; ++i) s[i] = uint(v >> (r + (3 - i) * KB)) & 0xFFFFu;
+ }
+ for (int i = 0; i < 4; ++i) {
+ const uint x = (s[i] & 0x7FFFu) * 0x9e3779b1u;
+ const float2 a = float2(as_type<half2>(g_tq(tab, x >> 21)));
+ const float2 c = float2(as_type<half2>(g_tq(tab, (x >> 10) & 2047u)));
+ q[4 * i] = a.x;
+ q[4 * i + 1] = a.y;
+ q[4 * i + 2] = c.x;
+ q[4 * i + 3] = c.y;
+ }
+ return rdh(b);
+}
+template <> inline float deq<T_TQ2T>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<8>(b, ui, q, tab); }
+template <> inline float deq<T_TQK6>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<6>(b, ui, q, tab); }
+template <> inline float deq<T_TQK7>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<7>(b, ui, q, tab); }
+
// sum over groups of LPR consecutive lanes (LPR = 1, 2, 4, 8, 16 or 32); the result is valid in every lane
template <int LPR>
static inline float lane_sum(float v) {
@@ -311,3 +629,66 @@ static inline float lane_sum(float v) {
for (ushort o = LPR / 2; o > 0; o >>= 1) v += simd_shuffle_xor(v, o);
return v;
}
+
+// ---- Gyro: the block Hadamard rotation of the activations ------------------------------------------------------------
+// Hadamard-folded experts (docs/GYRO.md, include/strata/kernels/cpu/hadamard.hpp) take y = H (s * x) per block of 128
+// values (the normalized Sylvester-Walsh-Hadamard matrix, s = a +/-1 sign vector of the width) in place of x: the gate/up
+// input and the down input. One thread per value, 128 threads a block, grid (width / 128, rows). The butterflies run in
+// the CPU's order (len = 1, 2, 4, ...; lower element a + c, upper a - c), so the rotated floats are the CPU's and the
+// CUDA kernel's: len 1..16 by SIMD shuffles, 32 and 64 through threadgroup memory.
+struct HadArgs {
+ uint width; // values per row (a multiple of 128)
+ uint x_stride; // row strides in elements
+ uint y_stride;
+ float scale; // 1 / sqrt(128), as the CPU computes it
+ uint has_signs;
+ uint pad;
+};
+template <typename T>
+static inline float had_ld(device const T* p);
+template <> inline float had_ld<float>(device const float* p) { return *p; }
+template <> inline float had_ld<half>(device const half* p) { return float(*p); }
+template <typename T>
+static inline void had_body(device const T* x, device T* y, device const float* signs, constant HadArgs& a, uint3 tg,
+ ushort tid, threadgroup float* buf) {
+ const uint col = tg.x * 128u + tid;
+ float v = had_ld<T>(x + ulong(tg.y) * a.x_stride + col);
+ if (a.has_signs) v *= signs[col];
+ for (ushort len = 1; len < 32; len <<= 1) {
+ const float o = simd_shuffle_xor(v, len);
+ v = (tid & len) ? o - v : v + o;
+ }
+ buf[tid] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ float o = buf[tid ^ 32];
+ v = (tid & 32) ? o - v : v + o;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ buf[tid] = v;
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ o = buf[tid ^ 64];
+ v = (tid & 64) ? o - v : v + o;
+ y[ulong(tg.y) * a.y_stride + col] = T(v * a.scale);
+}
+kernel void had128_f32(device const float* x [[buffer(0)]], device float* y [[buffer(1)]],
+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {
+ threadgroup float buf[128];
+ had_body<float>(x, y, signs, a, tg, tid, buf);
+}
+kernel void had128_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]],
+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {
+ threadgroup float buf[128];
+ had_body<half>(x, y, signs, a, tg, tid, buf);
+}
+// half rows in place, one threadgroup a row (grid.x = the row, so an indirect grid of the row count works): the prompt
+// path's h rows (the padded rows of the expert tiles), width values each
+kernel void had128_f16_rows(device half* x [[buffer(0)]], device half* y [[buffer(1)]],
+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],
+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {
+ threadgroup float buf[128];
+ for (uint b = 0; b < a.width / 128u; ++b) {
+ had_body<half>(x, y, signs, a, uint3(b, tg.x, 0), tid, buf);
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+}
diff --git a/metal/src/engine/iq_host.cpp b/metal/src/engine/iq_host.cpp
index 7c893382b..28dca0355 100644
--- a/metal/src/engine/iq_host.cpp
+++ b/metal/src/engine/iq_host.cpp
@@ -5,12 +5,15 @@
#define GGML_COMMON_DECL_CPP
#include "ggml-common.h"
+#include "ggml-common-tq.h" // block_tq2_t / block_tqk6 / block_tqk7
#include <cstddef>
#include <cstdint>
namespace strata::kernels {
namespace {
+// agentionai's trellis types (Gyro): TQ2_T (144), TQK6 (145), TQK7 (146); 128 weights a block
+bool is_tq(int t) { return t == 144 || t == 145 || t == 146; }
bool is_iq(int t) {
return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||
t == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8; // 14: Q6_K (Unsloth's Q5/Q6 gate/up)
@@ -20,6 +23,7 @@ int gu_qk(int t) {
switch (t) {
case 16: case 17: case 18: case 21: case 22: case 23: case 29: case 12: case 13: case 14: return 256;
case 42: return 64;
+ case 144: case 145: return 128; // TQ2_T / TQK6 gate/up (the CUDA engine's STRATA_GU_FMTS)
case 6: case 2: case 3: case 8: return 32;
default: return 0;
}
@@ -28,6 +32,7 @@ int d_qk(int t) {
switch (t) {
case 23: return 256;
case 42: return 64;
+ case 144: case 146: return 128; // TQ2_T / TQK7 down
case 20: case 7: case 6: case 2: case 3: case 8: return 32;
default: return 0;
}
@@ -57,6 +62,9 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
case 2: return (size_t) (n / 32) * sizeof(block_q4_0);
case 3: return (size_t) (n / 32) * sizeof(block_q4_1);
case 8: return (size_t) (n / 32) * sizeof(block_q8_0);
+ case 144: return (size_t) (n / 128) * sizeof(block_tq2_t);
+ case 145: return (size_t) (n / 128) * sizeof(block_tqk6);
+ case 146: return (size_t) (n / 128) * sizeof(block_tqk7);
case 30: return (size_t) n * 2;
default: return 0;
}
@@ -64,8 +72,8 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {
bool native_expert_supported(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) noexcept {
const int qg = gu_qk(gu_type), qd = d_qk(d_type);
- return qg > 0 && qd > 0 && is_iq(gu_type) && is_iq(d_type) && n_embd % qg == 0 && n_ff % qd == 0 &&
- n_embd % 256 == 0 && (n_ff * n_embd) % 256 == 0;
+ return qg > 0 && qd > 0 && (is_iq(gu_type) || is_tq(gu_type)) && (is_iq(d_type) || is_tq(d_type)) && n_embd % qg == 0 && n_ff % qd == 0 &&
+ n_embd % 256 == 0 && (n_ff * n_embd) % 256 == 0; // (the trellis rows: 128-value blocks, 2560 = 20, 640 = 5)
}
NativeExpertLayout native_expert_layout(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) {
diff --git a/metal/src/engine/prompt_chain.inc b/metal/src/engine/prompt_chain.inc
index 39efd5e60..f9b24133c 100644
--- a/metal/src/engine/prompt_chain.inc
+++ b/metal/src/engine/prompt_chain.inc
@@ -58,6 +58,7 @@ bool Window::pc_setup(int C) {
b.k4 = coop_k4();
}
b = PromptBufs{b.lib, b.lgdn, b.lqsa, b.k4}; // (re)allocated below for the larger chunk
+ b.v3e = b.k4 && T("blk.0.ffn_gate_exps.weight").type < 144; // the trellis experts have the v2 kernels only
b.C = C;
const size_t MT = (size_t) C;
const int MB = (int) max_cells_ / 4 + 2;
@@ -88,13 +89,14 @@ bool Window::pc_setup(int C) {
// the PLE block's rows and projections
b.pemb = F32(MT * N); b.pembh = F16(MT * N); b.pkey = F32(MT * 10240); b.pval = F32(MT * N); b.keyn = F32(MT * 10240);
b.qn = F32(MT * 10240); b.gated = F32(MT * 10240); b.normW = F32(MT * 10240);
+ if (had_on_) b.mixr = F32(MT * N);
b.Wpk = F16((size_t) 10240 * N); b.Wpv = F16((size_t) N * N);
// the expert GEMM tiles (prompt_chain's defaults): v3 gate/up 16,32,32,4 and down 16,64,32,2 with the M5 layout,
// else v2
b.gu3[0] = 16; b.gu3[1] = 32; b.gu3[2] = 32; b.gu3[3] = 4;
b.dn3[0] = 16; b.dn3[1] = 64; b.dn3[2] = 32; b.dn3[3] = 2;
- b.meta.as<uint32_t>()[2] = b.k4 ? NFF / (b.gu3[1] * b.gu3[3]) : NFF / 64;
- b.meta.as<uint32_t>()[5] = b.k4 ? N / (b.dn3[1] * b.dn3[3]) : N / 64;
+ b.meta.as<uint32_t>()[2] = b.v3e ? NFF / (b.gu3[1] * b.gu3[3]) : NFF / 64;
+ b.meta.as<uint32_t>()[5] = b.v3e ? N / (b.dn3[1] * b.dn3[3]) : N / 64;
std::fprintf(stderr, "strata metal: prompt kernels (mac-kern's prompt chain) for chunks of %d rows, %s, ready in %.0f ms\n",
C, b.k4 ? "matrix-unit register operands (COOP_K4)" : "the older expert / attention kernels (no M5 layout)",
now_ms() - t0);
@@ -290,16 +292,18 @@ bool Window::pc_chunk(const int64_t* tokens, int nT, int64_t pos0, std::string&
ma.d_stride = (uint64_t) ma.d_row * N;
const size_t ebase = moe_rebase(ma.gate_off, ma.up_off, ma.down_off);
const uint32_t ne = N;
+ const bool hg = had_gu(l), hd = had_dn(l);
+ if (hg) had_rows(ds, b.lib, "pc:", b.mixed, 0, b.mixr, 0, nT, N, hsx_.handle ? &hsx_ : nullptr, false);
mtl::Dispatch g;
g.p = P_(b.lib, "mm_gather");
- g.args = {mtl::Arg::B(b.mixed), mtl::Arg::B(b.stok), mtl::Arg::B(b.XS), mtl::Arg::V(ne)};
+ g.args = {mtl::Arg::B(hg ? b.mixr : b.mixed), mtl::Arg::B(b.stok), mtl::Arg::B(b.XS), mtl::Arg::V(ne)};
g.tg[0] = 256;
g.indirect = &b.meta;
g.ind_off = 8 * 4;
ds.push_back(g);
auto tl3 = [](const int* t) { return std::to_string(t[0]) + "_" + std::to_string(t[1]) + "_" + std::to_string(t[2]) + "_" + std::to_string(t[3]); };
mtl::Dispatch gu;
- if (b.k4) {
+ if (b.v3e) {
const std::string fn = "mm_gu3_" + std::to_string(eg.type) + "_" + tl3(b.gu3);
gu.p = P_(b.lib, fn, mtl::Constants().b(7, true), fn + "#k4");
} else {
@@ -307,19 +311,20 @@ bool Window::pc_chunk(const int64_t* tokens, int nT, int64_t pos0, std::string&
gu.p = P_(b.lib, fn, mtl::Constants().i(5, 0), fn + "#p0");
}
gu.args = {mtl::Arg::B(W_, ebase), mtl::Arg::B(b.XS), mtl::Arg::B(b.HS), mtl::Arg::B(b.tiles), mtl::Arg::V(ma)};
- gu.tg[0] = b.k4 ? 32 * b.gu3[3] : 64;
+ gu.tg[0] = b.v3e ? 32 * b.gu3[3] : 64;
gu.indirect = &b.meta;
gu.ind_off = 2 * 4;
ds.push_back(gu);
+ if (hd) had_rows_ind(ds, b.lib, "pc:", b.HS, b.meta);
mtl::Dispatch dn;
- if (b.k4) {
+ if (b.v3e) {
const std::string fn = "mm_down3_" + std::to_string(ed.type) + "_" + tl3(b.dn3);
dn.p = P_(b.lib, fn, mtl::Constants().b(7, true), fn + "#k4");
} else {
dn.p = P_(b.lib, "mm_down2_" + std::to_string(ed.type) + dn2_tile(ed.type));
}
dn.args = {mtl::Arg::B(W_, ebase), mtl::Arg::B(b.HS), mtl::Arg::B(b.YE), mtl::Arg::B(b.tiles), mtl::Arg::V(ma)};
- dn.tg[0] = b.k4 ? 32 * b.dn3[3] : 64;
+ dn.tg[0] = b.v3e ? 32 * b.dn3[3] : 64;
dn.indirect = &b.meta;
dn.ind_off = 5 * 4;
ds.push_back(dn);
diff --git a/metal/src/engine/window.cpp b/metal/src/engine/window.cpp
index 042562afa..a792aa5f8 100644
--- a/metal/src/engine/window.cpp
+++ b/metal/src/engine/window.cpp
@@ -23,6 +23,8 @@
#include "strata/core/conversation_cache.hpp"
#include "strata/core/session.hpp"
#include "strata/kernels/ngram.hpp"
+#include "strata/kernels/cpu/hadamard.hpp"
+#include "strata/kernels/cpu/expert_layout.hpp"
#include <algorithm>
#include <chrono>
@@ -97,13 +99,14 @@ double now_ms() { return std::chrono::duration<double, std::milli>(std::chrono::
struct PromptBufs {
int lib = -1, lgdn = -1, lqsa = -1;
bool k4 = false;
+ bool v3e = false; // the M5's expert kernels (v3): not for the trellis types, which have only v2
int C = 0, maxrows = 0;
double gpu_ms = 0, host_ms = 0, mtp_ms = 0;
int gu3[4] = {}, dn3[4] = {};
mtl::Buffer R, bo, inj[2], xn, xnh, V, loh, mixed, mixh, qkv, z, al, be, yg, ygh, gqn, gkn, gvc, ggb, gkq, go, qfull,
kcur, vcur, iqr, ikr, qcur, qcurh, qidx, scores, qids, part, XQ, xqh, logits, ids, wts, bcnt, boff, eoff, tiles, meta,
stok, sw, pos, XS, HS, YE, yr, gsh, ush, hh, ys, sgv, WD, WU, Wqkv, Wz, Wal, Wbe, Wout, Wq, Wk, Wv, Wiq, Wik, Wr,
- Wsg, Wshg, Wshu, Wshd, pemb, pembh, pkey, pval, keyn, qn, gated, normW, Wpk, Wpv;
+ Wsg, Wshg, Wshu, Wshd, pemb, pembh, pkey, pval, keyn, qn, gated, normW, Wpk, Wpv, mixr;
};
// The vision path (--vision, GENI requests), from the shared host code: the image rows of the prompt being read
@@ -331,6 +334,19 @@ public:
void prof_add(bool draft, double ms) { Prof& x = draft ? dprof_ : cprof_; ++x.n; x.all += ms; }
private:
// the Q8_0 copies of the hyper-connection matrices (the default; STRATA_HC_Q8=0: the BF16 originals)
+ // Gyro (agentionai's Hadamard-folded trellis experts, native_experts.txt v5 + hadamard.txt): the gate/up input is
+ // H (s * x) and the down input H h, per block of 128; XR_ holds the rotated rows, hsx_/hsh_ the +/-1 signs
+ bool had_on_ = false;
+ int had_block_ = 128;
+ bool had_gu(int l) const { return had_on_ && strata::kernels::cpu::hadamard_spec().gu(l); }
+ bool had_dn(int l) const { return had_on_ && strata::kernels::cpu::hadamard_spec().down(l); }
+ mtl::Buffer XR_, hsx_, hsh_;
+ // one rotation dispatch: rows of `width` values (a multiple of 128) in place or from x to y; f16 = half rows
+ void had_rows(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& x, size_t x_off,
+ const mtl::Buffer& y, size_t y_off, int rows, int width, const mtl::Buffer* signs, bool f16);
+ // the prompt's h rows (half, padded rows of the expert tiles, their count in meta[8]): rotated in place, one
+ // threadgroup a row through the indirect grid
+ void had_rows_ind(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& h, const mtl::Buffer& meta);
bool hc_q8_ = false;
mtl::Buffer Q8_;
std::map<std::string, size_t> q8off_;
@@ -487,6 +503,29 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionSta
pleHist_ = A(4 * 9 * 10240);
auto act = [&](int n) { Act a; a.n = n; a.b = A(4 * ((size_t) 4 * MAXT * n + (size_t) MAXT * n / 16)); return a; };
XA_ = act(N); XF_ = act(N); XY_ = act(6144); XQ_ = act(6144); XH_ = act(NFF); XE_ = act(N);
+ { // Gyro (native_experts.txt v5 + hadamard.txt): the rotation's signs and the rotated gate/up input
+ const auto& hs = strata::kernels::cpu::hadamard_spec();
+ had_on_ = hs.any();
+ if (had_on_) {
+ had_block_ = hs.block;
+ if (had_block_ != 128) {
+ err = "metal window: the pack's Hadamard block is " + std::to_string(had_block_) + "; the Metal rotation takes 128";
+ return false;
+ }
+ XR_ = A(4 * MAXT * N);
+ auto upload = [&](int w, mtl::Buffer& b) {
+ if (const float* sg = strata::kernels::cpu::hadamard_signs(w)) {
+ b = A(4 * (size_t) w);
+ std::memcpy(b.ptr, sg, 4 * (size_t) w);
+ }
+ };
+ upload(N, hsx_);
+ upload(NFF, hsh_);
+ std::fprintf(stderr, "strata metal: Hadamard-folded experts (block %d): the gate/up and down inputs are rotated on the GPU
+",
+ had_block_);
+ }
+ }
// RoPE: cos/sin [pos][32] for the first 64 dims, NEOX pairs (i, i + 32), in double as build_rope_table
const double base = f_.meta_num.count("qwen4exp.rope.freq_base") ? f_.meta_num["qwen4exp.rope.freq_base"] : 1e7;
for (int p = 0; p < MAXC + 16; ++p)
@@ -518,13 +557,16 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionSta
const char* e = std::getenv("STRATA_HC_Q8");
hc_q8_ = !(e && std::string(e) == "0");
}
- if (hc_q8_) {
+ {
const double tq = now_ms();
size_t tot = 0;
std::vector<std::string> names;
for (const auto& kv : f_.tensors) {
const gguf::Tensor& x = kv.second;
- if (x.type != ref::BF16 || x.name.find("hc_") == std::string::npos || x.name.find("_norm") != std::string::npos) continue;
+ // BF16 (when the Q8_0 read is on) and, whatever the setting, the Gyro file's Q6_K / IQ4_NL ones: the decode window's
+ // hyper-connection kernels read BF16 or Q8_0 only
+ const bool conv = (x.type == ref::BF16 && hc_q8_) || x.type == ref::Q6_K || x.type == ref::IQ4_NL;
+ if (!conv || x.name.find("hc_") == std::string::npos || x.name.find("_norm") != std::string::npos) continue;
uint64_t ne = 1;
for (uint64_t d : x.shape) ne *= d;
q8off_[x.name] = tot;
@@ -549,7 +591,7 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionSta
if (!st_->read(raw.data(), x.bytes, x.offset)) { std::fprintf(stderr, "strata metal: read failed\n"); std::exit(6); }
src = raw.data();
}
- ref::dequant(ref::BF16, src, tmp.data(), ne);
+ ref::dequant(x.type, src, tmp.data(), ne);
ref::quantize_q8_0(tmp.data(), ne, Q8_.as<uint8_t>() + q8off_[names[i]]);
}
});
@@ -676,6 +718,35 @@ void Window::gr_read(std::vector<mtl::Dispatch>& ds, const std::string& pre, boo
cur_inj_ = 1 - cur_inj_;
}
+namespace {
+struct HadArgs { uint32_t width, x_stride, y_stride; float scale; uint32_t has_signs, pad; }; // quant.metal
+}
+// y = H (s * x) per block of 128 values of `rows` rows of `width` (x_off / y_off: byte offsets; f16: half rows, no signs)
+void Window::had_rows(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& x, size_t x_off,
+ const mtl::Buffer& y, size_t y_off, int rows, int width, const mtl::Buffer* signs, bool f16) {
+ HadArgs a{(uint32_t) width, (uint32_t) width, (uint32_t) width, 1.0f / std::sqrt((float) had_block_), signs ? 1u : 0u, 0u};
+ const char* fn = f16 ? "had128_f16" : "had128_f32";
+ mtl::Dispatch d;
+ d.p = P(lib, fn, {}, pfx + fn);
+ d.args = {mtl::Arg::B(x, x_off), mtl::Arg::B(y, y_off), mtl::Arg::B(signs ? *signs : x), mtl::Arg::V(a)};
+ d.grid[0] = width / 128;
+ d.grid[1] = rows;
+ d.tg[0] = 128;
+ push(ds, d);
+}
+
+void Window::had_rows_ind(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& h,
+ const mtl::Buffer& meta) {
+ HadArgs a{(uint32_t) NFF, (uint32_t) NFF, (uint32_t) NFF, 1.0f / std::sqrt((float) had_block_), hsh_.handle ? 1u : 0u, 0u};
+ mtl::Dispatch d;
+ d.p = P(lib, "had128_f16_rows", {}, pfx + "had128_f16_rows");
+ d.args = {mtl::Arg::B(h), mtl::Arg::B(h), mtl::Arg::B(hsh_.handle ? hsh_ : h), mtl::Arg::V(a)};
+ d.tg[0] = 128;
+ d.indirect = &meta;
+ d.ind_off = 8 * 4; // (rows, 1, 1): the padded rows of the tiles
+ push(ds, d);
+}
+
void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0) {
const std::string b = "blk." + std::to_string(l) + ".";
// ---- the mixer: layer 1's read after the PLE is not applied again (its pending write was applied before it)
@@ -874,8 +945,10 @@ void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0
ac.b(23, ct);
const std::string ak = std::string(stream_ ? "#arena" : "") + (ct ? "#ct" : "");
g.p = P(lib_, gun, ac, gun + ak);
- g.args = {mtl::Arg::B(EW, ebase), mtl::Arg::B(XF_.b, XF_.plane_bytes() * xkind(tg_.type)), mtl::Arg::B(h_), mtl::Arg::B(plan_),
- mtl::Arg::V(a), mtl::Arg::B(XF_.b, XF_.plane_bytes() * 4)};
+ const bool hg = had_gu(l), hd = had_dn(l);
+ if (hg) had_rows(ds, lib_, "d:", XF_.b, 0, XR_, 0, ntok, N, hsx_.handle ? &hsx_ : nullptr, false); // before the host's turn
+ g.args = {mtl::Arg::B(EW, ebase), hg ? mtl::Arg::B(XR_) : mtl::Arg::B(XF_.b, XF_.plane_bytes() * xkind(tg_.type)), mtl::Arg::B(h_),
+ mtl::Arg::B(plan_), mtl::Arg::V(a), mtl::Arg::B(XF_.b, XF_.plane_bytes() * 4)};
if (stream_)
for (int c = -1; c < 3; ++c)
g.args.push_back(c < 0 ? mtl::Arg::B(st_->smap(), (size_t) l * 512 * 4) : mtl::Arg::B(st_->chunk(c + 1)));
@@ -886,6 +959,7 @@ void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0
push(ds, g);
nb_ = true;
gemv(ds, T(b + "ffn_down_shexp.weight"), XH_, ysh_, N, ntok);
+ if (hd) had_rows(ds, lib_, "d:", h_, 0, h_, 0, ntok * TOPK, NFF, hsh_.handle ? &hsh_ : nullptr, false); // H h, in place
mtl::Dispatch dd;
dd.p = P(lib_, dnn, ac, dnn + ak);
dd.args = {mtl::Arg::B(EW, ebase), mtl::Arg::B(h_), mtl::Arg::B(ye_), mtl::Arg::B(plan_), mtl::Arg::V(a), mtl::Arg::B(h_)};
@@ -2023,6 +2097,7 @@ struct Window::PromptGemm {
iqr, ikr, qcur, qcurh, qidx, scores, qids, part, XQ, xqh, logits, ids, wts, bcnt, boff, eoff, tiles, meta, stok, sw, pos,
XS, HS, YE, yr, gsh, ush, hh, ys, sgv, emb, embh, pkey, pval, keyn, qn, gated, normW;
mtl::Buffer WD, WU, Wqkv, Wz, Wal, Wbe, Wout, Wq, Wk, Wv, Wiq, Wik, Wr, Wsg, Wshg, Wshu, Wshd, Wpk, Wpv;
+ mtl::Buffer mixr; // Gyro: the rotated gate/up input rows [MT][N]
};
int Window::prompt_chunk_rows() const { return pg_ ? pg_->MT : 0; }
@@ -2063,6 +2138,7 @@ bool Window::prompt_gemm_init(std::string& err) {
p.k4 = ::coop_k4(*cx_);
const char* eg = std::getenv("STRATA_METAL_PROMPT_EXPERTS"); // v2 | v3 (default: v3 with the M5's layout)
p.gu_v3 = p.dn_v3 = eg ? std::string(eg) == "v3" : p.k4;
+ if (T("blk.0.ffn_gate_exps.weight").type >= 144) p.gu_v3 = p.dn_v3 = false; // the trellis experts have the v2 kernels only
const char* eq = std::getenv("STRATA_METAL_PROMPT_QSA"); // v1 | flash (default: flash with the M5's layout)
p.flash = eq ? std::string(eq) == "flash" : p.k4;
const size_t MT = (size_t) C;
@@ -2102,6 +2178,7 @@ bool Window::prompt_gemm_init(std::string& err) {
p.stok = AB(4 * mr); p.sw = F32(mr); p.pos = AB(4 * MT * TOPK); p.XS = F16(mr * N); p.HS = F16(mr * NFF);
p.YE = F32(mr * N); p.yr = F32(MT * N); p.gsh = F32(MT * NFF); p.ush = F32(MT * NFF); p.hh = F16(MT * NFF);
p.ys = F32(MT * N); p.sgv = F32(MT);
+ if (had_on_) p.mixr = F32(MT * N);
p.WD = F16((size_t) (LR + HC) * HC * N); p.WU = F16((size_t) HC * N * LR); p.Wqkv = F16((size_t) 10240 * N);
p.Wz = F16((size_t) 6144 * N); p.Wal = F16((size_t) 48 * N); p.Wbe = F16((size_t) 48 * N); p.Wout = F16((size_t) N * 6144);
p.Wq = F16((size_t) 12288 * N); p.Wk = F16((size_t) 512 * N); p.Wv = F16((size_t) 512 * N); p.Wiq = F16((size_t) 512 * N);
@@ -2339,10 +2416,12 @@ bool Window::prompt_chunk(const int32_t* tokens, int Tn, int64_t pos0, std::stri
if (stream_) { ma.gate_off = 0; ma.up_off = st_->up_in_slot(l); ma.down_off = st_->down_in_slot(l); }
const size_t ebase = moe_rebase(ma.gate_off, ma.up_off, ma.down_off);
const uint32_t ne_ = N;
+ const bool hg = had_gu(l), hd = had_dn(l);
+ if (hg) had_rows(ds, p.lib, "pg:", p.mixed, 0, p.mixr, 0, T, N, hsx_.handle ? &hsx_ : nullptr, false);
{
mtl::Dispatch g;
g.p = PP(p.lib, "mm_gather");
- g.args = {mtl::Arg::B(p.mixed), mtl::Arg::B(p.stok), mtl::Arg::B(p.XS), mtl::Arg::V(ne_)};
+ g.args = {mtl::Arg::B(hg ? p.mixr : p.mixed), mtl::Arg::B(p.stok), mtl::Arg::B(p.XS), mtl::Arg::V(ne_)};
g.tg[0] = 256;
g.indirect = &p.meta;
g.ind_off = 8 * 4;
@@ -2375,6 +2454,7 @@ bool Window::prompt_chunk(const int32_t* tokens, int Tn, int64_t pos0, std::stri
gu.ind_off = 2 * 4;
if (stream_) cuts.push_back((int) ds.size()); // the host's turn: this layer's experts put in place
push(ds, gu);
+ if (hd) had_rows_ind(ds, p.lib, "pg:", p.HS, p.meta);
mtl::Dispatch dn;
if (p.dn_v3) {
const std::string fn = "mm_down3_" + std::to_string(ed.type) + "_16_64_32_2";
diff --git a/metal/src/ref_quant.cpp b/metal/src/ref_quant.cpp
index ede5e5b3d..51bae0f90 100644
--- a/metal/src/ref_quant.cpp
+++ b/metal/src/ref_quant.cpp
@@ -12,6 +12,8 @@
#define GGML_COMMON_DECL_CPP
#define GGML_COMMON_IMPL_CPP
#include "ggml-common.h"
+#include "ggml-common-tq.h" // the trellis block structs and the codebook tq_lut_f16 (Gyro)
+#include "../../src/kernels/cpu/tq_states.hpp" // the CPU trellis state extraction (the reference decoder)
namespace ref {
@@ -23,6 +25,7 @@ const char* type_name(int t) {
case IQ2_XXS: return "IQ2_XXS"; case IQ2_XS: return "IQ2_XS"; case IQ2_S: return "IQ2_S";
case IQ3_XXS: return "IQ3_XXS"; case IQ3_S: return "IQ3_S"; case IQ4_NL: return "IQ4_NL";
case IQ4_XS: return "IQ4_XS"; case Q2_0: return "Q2_0";
+ case TQ2_T: return "TQ2_T"; case TQK6: return "TQK6"; case TQK7: return "TQK7";
}
return "?";
}
@@ -32,6 +35,7 @@ int block_elems(int t) {
case F32: case F16: case BF16: return 1;
case Q4_0: case Q5_0: case Q5_1: case Q8_0: case IQ4_NL: return 32;
case Q2_0: return 64;
+ case TQ2_T: case TQK6: case TQK7: return 128;
default: return 256;
}
}
@@ -43,6 +47,7 @@ int block_bytes(int t) {
case Q3_K: return 110; case Q4_K: return 144; case Q5_K: return 176; case Q6_K: return 210;
case IQ2_XXS: return 66; case IQ2_XS: return 74; case IQ2_S: return 82; case IQ3_XXS: return 98;
case IQ3_S: return 110; case IQ4_XS: return 136;
+ case TQ2_T: return 34; case TQK6: return 26; case TQK7: return 30;
}
std::fprintf(stderr, "ref: unknown type %d\n", t);
std::exit(2);
@@ -102,8 +107,30 @@ static void get_scale_min_k4(int j, const uint8_t* q, uint8_t& d, uint8_t& m) {
}
}
+// The trellis types (ggml 144 TQ2_T, 145 TQK6, 146 TQK7): 128 weights = 32 steps of 4; step t's state comes from
+// the CPU engine's own extraction (src/kernels/cpu/tq_states.hpp), x = state * 0x9e3779b1, the two codebook points
+// lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs) give weights 4t..4t+3, times the fp16 scale.
+static void deq_tq(int kb, const uint8_t* b, float* y) {
+ uint32_t st[32];
+ if (kb == 8) strata::kernels::cpu::tq_detail::tq_states<8>(b + 2, st);
+ else if (kb == 6) strata::kernels::cpu::tq_detail::tq_states<6>(b + 2, st);
+ else strata::kernels::cpu::tq_detail::tq_states<7>(b + 2, st);
+ const float d = f16_to_f32(rd16(b));
+ for (int t = 0; t < 32; ++t) {
+ const uint32_t x = st[t] * 0x9e3779b1u;
+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;
+ y[4 * t + 0] = f16_to_f32(tq_lut_f16[2 * pa]) * d;
+ y[4 * t + 1] = f16_to_f32(tq_lut_f16[2 * pa + 1]) * d;
+ y[4 * t + 2] = f16_to_f32(tq_lut_f16[2 * pb]) * d;
+ y[4 * t + 3] = f16_to_f32(tq_lut_f16[2 * pb + 1]) * d;
+ }
+}
+
static void deq_block(int t, const uint8_t* b, float* y) {
switch (t) {
+ case TQ2_T: deq_tq(8, b, y); return;
+ case TQK6: deq_tq(6, b, y); return;
+ case TQK7: deq_tq(7, b, y); return;
case Q2_0: {
const float d = f16_to_f32(rd16(b));
for (int j = 0; j < 64; ++j) y[j] = (float)(((b[2 + j / 4] >> ((j % 4) * 2)) & 3) - 1) * d;
@@ -437,6 +464,7 @@ void random_blocks(int t, uint8_t* dst, int rows, int cols, uint64_t seed, float
case IQ2_XXS: case IQ2_XS: case IQ2_S: put16(b, f32_to_f16(scale * u / 4)); break;
case IQ3_XXS: put16(b, f32_to_f16(scale * u / 8)); break;
case IQ3_S: put16(b, f32_to_f16(scale * u / 64)); break;
+ case TQ2_T: case TQK6: case TQK7: put16(b, f32_to_f16(scale * u / 1.5f)); break; // codebook |point| ~ 1
default: break;
}
}
diff --git a/metal/src/ref_quant.h b/metal/src/ref_quant.h
index f20aeee99..47b51cf94 100644
--- a/metal/src/ref_quant.h
+++ b/metal/src/ref_quant.h
@@ -13,6 +13,7 @@ namespace ref {
enum : int {
F32 = 0, F16 = 1, Q4_0 = 2, Q5_0 = 6, Q5_1 = 7, Q8_0 = 8, Q3_K = 11, Q4_K = 12, Q5_K = 13, Q6_K = 14,
IQ2_XXS = 16, IQ2_XS = 17, IQ3_XXS = 18, IQ3_S = 21, IQ2_S = 22, IQ4_NL = 20, IQ4_XS = 23, BF16 = 30, Q2_0 = 42,
+ TQ2_T = 144, TQK6 = 145, TQK7 = 146, // agentionai's trellis types (Gyro): 128 values a block, 34 / 26 / 30 bytes
};
const char* type_name(int t);
diff --git a/metal/tests/expert_parity.cpp b/metal/tests/expert_parity.cpp
index 3bd15ba35..7c101c0f1 100644
--- a/metal/tests/expert_parity.cpp
+++ b/metal/tests/expert_parity.cpp
@@ -4,6 +4,13 @@
// expert_parity [--gu Q2_0] [--dn Q2_0] [--ntok 1..8] [--gucfg L,R,S] [--dncfg L,R,S] parity (synthetic)
// expert_parity --bench [--gu ..] [--dn ..] [--ntok N] [--layers 8] [--reps 96] [--sweep] speed
//
+// Gyro (agentionai's trellis experts, TQ2_T / TQK6 / TQK7):
+// expert_parity --gu TQK6 --dn TQK7 [--had-pack <pack dir>] synthetic blocks; with a pack, the Hadamard rotation of
+// x (CPU) and of h (the Metal had128 kernel) is part of it
+// expert_parity --gguf <Gyro file> --layer L [--had-pack <pack dir>] [--ntok N] [--seed S] REAL experts of layer L (64 of
+// the 512, random) instead of random blocks
+// expert_parity --had-test [--had-pack <pack dir>] the Metal had128 kernels (f32, f16, f16 rows) against the CPU rotation, bitwise
+//
// Parity: random valid blocks for 64 experts, random activations x ~ U(-1,1), random distinct top-10 per token,
// weights summing to 1. Checked: the plan (groups by expert id, entries in (token, k) order), h = silu(gate) * up
// per entry and y = sum_k w * down . h, each against double; reported as max |err| / max |ref| and relative L2.
@@ -11,6 +18,9 @@
// a different random routing per rep; GB/s = bytes of the distinct experts the kernel must read / GPU time.
#include "mtl_rt.h"
#include "ref_quant.h"
+#include "gguf_lite.h"
+// Gyro: the engine's CPU Hadamard rotation (the reference of the Metal one) and hadamard.txt's parser
+#include "../../src/kernels/cpu/hadamard.cpp"
#include <algorithm>
#include <cmath>
@@ -36,7 +46,7 @@ struct MoeArgs {
};
int type_of(const std::string& s) {
- for (int t : {ref::Q2_0, ref::IQ3_XXS, ref::IQ2_XXS, ref::IQ2_XS, ref::IQ2_S, ref::IQ3_S, ref::IQ4_NL, ref::IQ4_XS, ref::Q4_K, ref::Q5_K, ref::Q6_K, ref::Q8_0, ref::Q5_1})
+ for (int t : {ref::TQ2_T, ref::TQK6, ref::TQK7, ref::Q2_0, ref::IQ3_XXS, ref::IQ2_XXS, ref::IQ2_XS, ref::IQ2_S, ref::IQ3_S, ref::IQ4_NL, ref::IQ4_XS, ref::Q4_K, ref::Q5_K, ref::Q6_K, ref::Q8_0, ref::Q5_1})
if (s == ref::type_name(t)) return t;
std::fprintf(stderr, "unknown type %s\n", s.c_str());
std::exit(2);
@@ -118,6 +128,11 @@ mtl::Dispatch prep_dispatch(const mtl::Pipeline& p, const mtl::Buffer& x, const
}
int g_gu_type = 0, g_dn_type = 0;
+// Gyro: the rotation of h between the gate/up and the down dispatch (the Metal had128_f32 kernel, signs of width n_ff)
+struct HadArgs { uint32_t width, x_stride, y_stride; float scale; uint32_t has_signs, pad; };
+const mtl::Pipeline* g_had32 = nullptr;
+const mtl::Buffer* g_hsh = nullptr;
+bool g_had_h = false;
bool g_dn_prep = false; // Q2_0 down: prepared h (an xprep dispatch) instead of the raw-x decoder
const mtl::Buffer* g_xpx = nullptr;
const mtl::Buffer* g_xph = nullptr;
@@ -144,6 +159,17 @@ void encode_layer(std::vector<mtl::Dispatch>& ds, const Kernels& K, const mtl::B
d.indirect = &plan;
d.ind_off = plan_off + P_IGU * 4;
ds.push_back(d);
+ if (g_had_h) { // H h in place, one row per entry (the entries are the window's token x k)
+ static HadArgs ha;
+ ha = HadArgs{(uint32_t)N_FF, (uint32_t)N_FF, (uint32_t)N_FF, 1.0f / std::sqrt(128.0f), g_hsh ? 1u : 0u, 0u};
+ mtl::Dispatch hd;
+ hd.p = g_had32;
+ hd.args = {mtl::Arg::B(h), mtl::Arg::B(h), mtl::Arg::B(g_hsh ? *g_hsh : h), mtl::Arg::V(ha)};
+ hd.grid[0] = N_FF / 128;
+ hd.grid[1] = a.n_tok * TOPK;
+ hd.tg[0] = 128;
+ ds.push_back(hd);
+ }
}
if (with_dn) {
const int dk = g_dn_prep ? xkind(g_dn_type) : 0;
@@ -190,8 +216,11 @@ double median(std::vector<double> v) {
int main(int argc, char** argv) {
int gu = ref::Q2_0, dn = ref::Q2_0, ntok = 1, layers = 8, reps = 96;
Cfg cgu{32, 2, 4}, cdn{8, 2, 4};
- bool bench = false, sweep = false;
+ bool bench = false, sweep = false, had_test = false;
double share = 0.0;
+ std::string gguf_path, had_pack;
+ int real_layer = 0;
+ uint64_t seed = 1;
bool cfg_gu_set = false, cfg_dn_set = false;
for (int i = 1; i < argc; ++i) {
std::string a = argv[i];
@@ -207,14 +236,123 @@ int main(int argc, char** argv) {
else if (a == "--reps") reps = std::atoi(nx());
else if (a == "--share") share = std::atof(nx());
else if (a == "--dn-prep") g_dn_prep = true;
+ else if (a == "--gguf") gguf_path = nx();
+ else if (a == "--layer") real_layer = std::atoi(nx());
+ else if (a == "--had-pack") had_pack = nx();
+ else if (a == "--had-test") had_test = true;
+ else if (a == "--seed") seed = std::strtoull(nx(), nullptr, 10);
else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; }
}
(void)cfg_gu_set;
(void)cfg_dn_set;
if (dn == ref::IQ4_NL && cdn.L == 8 && !cfg_dn_set) cdn = {4, 2, 4};
+ gguf::File gf;
+ const gguf::Tensor *tg_ = nullptr, *tu_ = nullptr, *td_ = nullptr;
+ if (!gguf_path.empty()) { // real experts: the types come from the file
+ std::string err;
+ if (!gf.open(gguf_path, err)) { std::fprintf(stderr, "%s\n", err.c_str()); return 2; }
+ const std::string b = "blk." + std::to_string(real_layer) + ".";
+ tg_ = gf.find(b + "ffn_gate_exps.weight");
+ tu_ = gf.find(b + "ffn_up_exps.weight");
+ td_ = gf.find(b + "ffn_down_exps.weight");
+ if (!tg_ || !tu_ || !td_) { std::fprintf(stderr, "no routed experts in layer %d of %s\n", real_layer, gguf_path.c_str()); return 2; }
+ gu = tg_->type;
+ dn = td_->type;
+ }
mtl::Context cx;
const int lib = cx.compile({"quant.metal", "units.metal", "moe.metal"});
const Layout L = layout(gu, dn);
+ // Gyro: the pack's hadamard.txt (the signs of the widths 2560 and 640) -> the engine's CPU spec (the reference)
+ mtl::Buffer hsx, hsh;
+ mtl::Pipeline had32 = cx.pipeline(lib, "had128_f32"), had16 = cx.pipeline(lib, "had128_f16"), had16r = cx.pipeline(lib, "had128_f16_rows");
+ const float* sgx = nullptr;
+ const float* sgh = nullptr;
+ if (!had_pack.empty()) {
+ strata::kernels::cpu::HadamardSpec spec;
+ std::string err;
+ if (!strata::kernels::cpu::hadamard_parse(had_pack + "/hadamard.txt", 48, spec, err)) {
+ std::fprintf(stderr, "%s\n", err.c_str());
+ return 2;
+ }
+ strata::kernels::cpu::hadamard_set(std::move(spec));
+ sgx = strata::kernels::cpu::hadamard_signs(N_EMBD);
+ sgh = strata::kernels::cpu::hadamard_signs(N_FF);
+ if (sgx) { hsx = cx.alloc(4 * N_EMBD); std::memcpy(hsx.ptr, sgx, 4 * N_EMBD); }
+ if (sgh) { hsh = cx.alloc(4 * N_FF); std::memcpy(hsh.ptr, sgh, 4 * N_FF); }
+ g_had32 = &had32;
+ g_hsh = sgh ? &hsh : nullptr;
+ g_had_h = !had_test;
+ std::printf("Hadamard: block %d, signs for %s%s\n", strata::kernels::cpu::hadamard_spec().block, sgx ? "2560 " : "", sgh ? "640" : "(none)");
+ }
+ if (had_test) {
+ // the Metal rotation kernels against the CPU's (hadamard_rotate), bit for bit
+ int bad = 0;
+ ref::Rng r2(77 + seed);
+ for (int width : {N_FF, N_EMBD}) {
+ const int rows = 37;
+ const float* sg = width == N_EMBD ? sgx : sgh;
+ const mtl::Buffer& sb = width == N_EMBD ? hsx : hsh;
+ const float scale = 1.0f / std::sqrt(128.0f);
+ std::vector<float> in((size_t)rows * width);
+ for (auto& v : in) v = r2.sym() * 3.0f;
+ mtl::Buffer bx = cx.alloc(4 * in.size()), by = cx.alloc(4 * in.size());
+ std::memcpy(bx.ptr, in.data(), 4 * in.size());
+ HadArgs ha{(uint32_t)width, (uint32_t)width, (uint32_t)width, scale, sg ? 1u : 0u, 0u};
+ auto run32 = [&](const mtl::Buffer& src, const mtl::Buffer& dst) {
+ mtl::Dispatch d;
+ d.p = &had32;
+ d.args = {mtl::Arg::B(src), mtl::Arg::B(dst), mtl::Arg::B(sg ? sb : src), mtl::Arg::V(ha)};
+ d.grid[0] = width / 128; d.grid[1] = rows; d.tg[0] = 128;
+ cx.run({d});
+ };
+ run32(bx, by); // out of place
+ std::vector<float> want = in;
+ for (int r = 0; r < rows; ++r) strata::kernels::cpu::hadamard_rotate(want.data() + (size_t)r * width, width, 128, sg);
+ int b32 = 0;
+ for (size_t i = 0; i < in.size(); ++i) b32 += std::memcmp(&want[i], by.as<float>() + i, 4) != 0;
+ run32(bx, bx); // in place
+ int b32i = 0;
+ for (size_t i = 0; i < in.size(); ++i) b32i += std::memcmp(&want[i], bx.as<float>() + i, 4) != 0;
+ // f16 (no signs): out of place, and the rows kernel in place; the CPU's float result rounded to half
+ std::vector<_Float16> hin(in.size());
+ for (size_t i = 0; i < in.size(); ++i) hin[i] = (_Float16)in[i];
+ std::vector<float> hw(in.size());
+ for (size_t i = 0; i < in.size(); ++i) hw[i] = (float)hin[i];
+ for (int r = 0; r < rows; ++r) strata::kernels::cpu::hadamard_rotate(hw.data() + (size_t)r * width, width, 128, nullptr);
+ mtl::Buffer hx = cx.alloc(2 * in.size()), hy = cx.alloc(2 * in.size());
+ std::memcpy(hx.ptr, hin.data(), 2 * in.size());
+ HadArgs hb{(uint32_t)width, (uint32_t)width, (uint32_t)width, scale, 0u, 0u};
+ {
+ mtl::Dispatch d;
+ d.p = &had16;
+ d.args = {mtl::Arg::B(hx), mtl::Arg::B(hy), mtl::Arg::B(hx), mtl::Arg::V(hb)};
+ d.grid[0] = width / 128; d.grid[1] = rows; d.tg[0] = 128;
+ cx.run({d});
+ }
+ int b16 = 0;
+ for (size_t i = 0; i < in.size(); ++i) {
+ const _Float16 w = (_Float16)hw[i];
+ b16 += std::memcmp(&w, hy.as<_Float16>() + i, 2) != 0;
+ }
+ {
+ mtl::Dispatch d;
+ d.p = &had16r;
+ d.args = {mtl::Arg::B(hx), mtl::Arg::B(hx), mtl::Arg::B(hx), mtl::Arg::V(hb)};
+ d.grid[0] = rows; d.tg[0] = 128;
+ cx.run({d});
+ }
+ int b16r = 0;
+ for (size_t i = 0; i < in.size(); ++i) {
+ const _Float16 w = (_Float16)hw[i];
+ b16r += std::memcmp(&w, hx.as<_Float16>() + i, 2) != 0;
+ }
+ std::printf("had-test width %d, %d rows%s: f32 %d / f32 in place %d / f16 %d / f16 rows %d of %zu values differ from the CPU\n",
+ width, rows, sg ? " (with signs)" : " (no signs)", b32, b32i, b16, b16r, in.size());
+ bad += b32 + b32i + b16 + b16r;
+ }
+ std::printf("had-test %s\n", bad == 0 ? "PASS (bitwise)" : "FAIL");
+ return bad == 0 ? 0 : 1;
+ }
std::printf("device %s; gate/up %s, down %s, blob %zu bytes; %d token(s)\n", cx.device_name().c_str(),
ref::type_name(gu), ref::type_name(dn), L.blob, ntok);
@@ -255,8 +393,21 @@ int main(int argc, char** argv) {
g_xph = &XPH;
g_gu_type = gu;
g_dn_type = dn;
- ref::Rng rng(12345);
+ ref::Rng rng(12345 + (seed - 1));
for (int i = 0; i < 8 * N_EMBD; ++i) x.as<float>()[i] = rng.sym();
+ if (g_had_h) { // H (s * x) per token on the GPU, checked against the CPU's; x then holds the rotated rows
+ std::vector<float> want(x.as<float>(), x.as<float>() + 8 * N_EMBD);
+ for (int t = 0; t < 8; ++t) strata::kernels::cpu::hadamard_rotate(want.data() + t * N_EMBD, N_EMBD, 128, sgx);
+ HadArgs ha{(uint32_t)N_EMBD, (uint32_t)N_EMBD, (uint32_t)N_EMBD, 1.0f / std::sqrt(128.0f), sgx ? 1u : 0u, 0u};
+ mtl::Dispatch d;
+ d.p = &had32;
+ d.args = {mtl::Arg::B(x), mtl::Arg::B(x), mtl::Arg::B(sgx ? hsx : x), mtl::Arg::V(ha)};
+ d.grid[0] = N_EMBD / 128; d.grid[1] = 8; d.tg[0] = 128;
+ cx.run({d});
+ int bad = 0;
+ for (int i = 0; i < 8 * N_EMBD; ++i) bad += std::memcmp(&want[i], x.as<float>() + i, 4) != 0;
+ std::printf("x rotation on the GPU vs the CPU: %d of %d values differ\n", bad, 8 * N_EMBD);
+ }
{
mtl::Pipeline pp = cx.pipeline(lib, "xprep");
cx.run({prep_dispatch(pp, x, XPX, N_EMBD, 8)});
@@ -266,10 +417,20 @@ int main(int argc, char** argv) {
// ---------------- parity ----------------
const int NE = 64;
mtl::Buffer W = cx.alloc(L.blob * NE);
+ std::vector<int> pick;
for (int e = 0; e < NE; ++e) {
uint8_t* b = W.as<uint8_t>() + e * L.blob;
- ref::random_blocks(gu, b, 2 * N_FF, N_EMBD, 1000 + e);
- ref::random_blocks(dn, b + L.down_off, N_EMBD, N_FF, 5000 + e);
+ if (tg_) { // real experts of the file: expert `ex` of the layer's three tensors
+ int ex;
+ do ex = (int)(rng.next() % 512); while (std::find(pick.begin(), pick.end(), ex) != pick.end());
+ pick.push_back(ex);
+ std::memcpy(b, (const uint8_t*)gf.map + tg_->offset + (size_t)ex * L.gu_row * N_FF, L.gu_row * N_FF);
+ std::memcpy(b + L.up_off, (const uint8_t*)gf.map + tu_->offset + (size_t)ex * L.gu_row * N_FF, L.gu_row * N_FF);
+ std::memcpy(b + L.down_off, (const uint8_t*)gf.map + td_->offset + (size_t)ex * L.d_row * N_EMBD, L.d_row * N_EMBD);
+ } else {
+ ref::random_blocks(gu, b, 2 * N_FF, N_EMBD, 1000 + e + 100 * (seed - 1));
+ ref::random_blocks(dn, b + L.down_off, N_EMBD, N_FF, 5000 + e + 100 * (seed - 1));
+ }
}
std::vector<int> ids;
std::vector<float> w;
@@ -323,8 +484,14 @@ int main(int argc, char** argv) {
u += (double)wu[r * N_EMBD + k] * xt[k];
}
hr[r] = g / (1.0 + std::exp(-g)) * u;
- eh.add(h.as<float>()[ent * N_FF + r], hr[r]);
}
+ if (g_had_h) { // the reference of the down input: H h, in float as the engine's CPU path does
+ std::vector<float> hf(N_FF);
+ for (int r = 0; r < N_FF; ++r) hf[r] = (float)hr[r];
+ strata::kernels::cpu::hadamard_rotate(hf.data(), N_FF, 128, sgh);
+ for (int r = 0; r < N_FF; ++r) hr[r] = hf[r];
+ }
+ for (int r = 0; r < N_FF; ++r) eh.add(h.as<float>()[ent * N_FF + r], hr[r]);
for (int r = 0; r < N_EMBD; ++r) {
double s = 0;
// the reference down uses the GPU's h (float) so that the y check isolates the down kernel ...
diff --git a/metal/tools/gen_tq_lut.py b/metal/tools/gen_tq_lut.py
new file mode 100644
index 000000000..c94d23c05
--- /dev/null
+++ b/metal/tools/gen_tq_lut.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""metal/tools/gen_tq_lut.py - writes the trellis codebook into metal/kernels/quant.metal.
+
+third_party/ggml/ggml-common-tq.h holds tq_lut_f16[4096]: 2048 codebook points of two fp16 values each. The Metal
+kernels read a point as one 32-bit word (low half = the first value), so this script rewrites the table between the
+"BEGIN tq_lut" and "END tq_lut" markers of quant.metal with tq_lut_u[2048] = lut[2i] | lut[2i+1] << 16.
+
+ python metal/tools/gen_tq_lut.py rewrite quant.metal
+ python metal/tools/gen_tq_lut.py --check exit 1 when quant.metal differs from what would be written
+"""
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+SRC = ROOT / "third_party" / "ggml" / "ggml-common-tq.h"
+DST = ROOT / "metal" / "kernels" / "quant.metal"
+BEGIN, END = "// BEGIN tq_lut", "// END tq_lut"
+
+
+def table():
+ text = SRC.read_text()
+ m = re.search(r"tq_lut_f16\[4096\]\s*=\s*\{(.*?)\};", text, re.S)
+ vals = [int(v, 16) for v in re.findall(r"0x[0-9a-fA-F]+", m.group(1))]
+ assert len(vals) == 4096, len(vals)
+ return [vals[2 * i] | (vals[2 * i + 1] << 16) for i in range(2048)]
+
+
+def block():
+ words = table()
+ lines = [BEGIN + " (generated by metal/tools/gen_tq_lut.py from third_party/ggml/ggml-common-tq.h; do not edit)",
+ "constant uint tq_lut_u[2048] = {"]
+ for i in range(0, 2048, 8):
+ lines.append(" " + ", ".join("0x%08x" % w for w in words[i:i + 8]) + ",")
+ lines.append("};")
+ lines.append(END)
+ return "\n".join(lines)
+
+
+def main():
+ src = DST.read_text()
+ i, j = src.index(BEGIN), src.index(END) + len(END)
+ new = src[:i] + block() + src[j:]
+ if "--check" in sys.argv:
+ sys.exit(0 if new == src else 1)
+ DST.write_text(new)
+
+
+if __name__ == "__main__":
+ main()