Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Metal: trellis expert kernels (TQ2_T, TQK6, TQK7) and the block Hadamard rotation for Gyro
Niko1221672fd99ccce0
diff --git a/metal/CMakeLists.txt b/metal/CMakeLists.txtindex 7a2e60b9a..220164d45 100644--- a/metal/CMakeLists.txt+++ b/metal/CMakeLists.txt@@ -31,7 +31,7 @@ set(GGML_COMMON "${STRATA_ROOT}/third_party/ggml/ggml-common.h")# ---- the kernel tree (mac-kern) ------------------------------------------------------------------------------------add_library(mtl_rt STATIC src/mtl_rt.mm src/ref_quant.cpp src/gguf_lite.cpp)-target_include_directories(mtl_rt PUBLIC src ${STRATA_ROOT}/third_party/ggml)+target_include_directories(mtl_rt PUBLIC src ${STRATA_ROOT}/third_party/ggml ${STRATA_ROOT}/include)target_compile_options(mtl_rt PRIVATE $<$<COMPILE_LANGUAGE:OBJCXX>:-fobjc-arc>)target_compile_definitions(mtl_rt PRIVATESTRATA_GGML_COMMON_DEFAULT="${GGML_COMMON}"diff --git a/metal/kernels/moe.metal b/metal/kernels/moe.metalindex bb8fdbbcc..e3f0b1d68 100644--- a/metal/kernels/moe.metal+++ b/metal/kernels/moe.metal@@ -334,3 +334,6 @@ DN_SET(T_Q5_1) DN_SET(T_Q8_0)DN_SET(T_Q2_0)DN_SET(T_Q2_0R)DN_SET(T_IQ4_NL)+// Gyro (agentionai's trellis experts): TQK6 / TQ2_T gate/up, TQK7 / TQ2_T down (the CUDA engine's STRATA_GU_FMTS / D_FMTS)+GU_SET(T_TQK6) GU_SET(T_TQ2T)+DN_SET(T_TQK7) DN_SET(T_TQ2T)diff --git a/metal/kernels/moe_mm.metal b/metal/kernels/moe_mm.metalindex ff28897ac..fb2f6f056 100644--- a/metal/kernels/moe_mm.metal+++ b/metal/kernels/moe_mm.metal@@ -393,7 +393,9 @@ MM_GU2_KQ(T_Q4_K) MM_GU2_KQ(T_Q5_K) MM_GU2_KQ(T_Q6_K) MM_GU2_U32(T_Q8_0) MM_DN2_MMGU2(T_IQ4_XS, 16, 64, 32, 2) MMGU2(T_IQ4_XS, 16, 64, 64, 2) MMGU2(T_IQ4_XS, 32, 64, 32, 2) MMGU2(T_IQ4_XS, 16, 32, 32, 2)MMGU2(T_IQ4_XS, 64, 64, 32, 4)MM_GU2_SET(T_Q2_0) MM_GU2_SET(T_IQ3_XXS) MM_GU2_SET(T_IQ2_XXS) MM_GU2_SET(T_IQ2_XS) MM_GU2_SET(T_IQ2_S) MM_GU2_SET(T_IQ3_S)+MM_GU2_SET(T_TQK6) MM_GU2_SET(T_TQ2T) // Gyro: the trellis gate/up (units of 16 weights: TK 16 or a multiple)MM_DN2_SET(T_Q2_0)+MM_DN2_SET(T_TQK7) MM_DN2_SET(T_TQ2T)MMDN2(T_IQ4_NL, 16, 64, 32, 2) MMDN2(T_IQ4_NL, 16, 64, 64, 2) MMDN2(T_IQ4_NL, 16, 32, 32, 2) MMDN2(T_IQ4_NL, 32, 32, 32, 2) MMDN2(T_IQ4_NL, 16, 128, 32, 4)// ---- v3: the weights decoded straight into matmul2d's right-input cooperative tensor (single-simdgroup scope),diff --git a/metal/kernels/quant.metal b/metal/kernels/quant.metalindex 51fecca7c..14df14eef 100644--- a/metal/kernels/quant.metal+++ b/metal/kernels/quant.metal@@ -29,6 +29,9 @@#define T_IQ4_XS 23#define T_Q2_0 42#define T_Q2_0R 142 // Q2_0 read with raw activations (codes - 1 in the decoder; no prepared x needed)+#define T_TQ2T 144 // agentionai's trellis types (Gyro): 128 values a block, 34 / 26 / 30 bytes, Hadamard-folded experts+#define T_TQK6 145+#define T_TQK7 146template <int F> struct fmt;// QK block elements, BB block bytes, UNIT weights per unit, TAB words of threadgroup table@@ -51,6 +54,10 @@ template <> struct fmt<T_Q8_0> { enum { QK = 32, BB = 34, UNIT = 32, TAB =template <> struct fmt<T_Q4_0> { enum { QK = 32, BB = 18, UNIT = 32, TAB = 1 }; };template <> struct fmt<T_Q5_0> { enum { QK = 32, BB = 22, UNIT = 32, TAB = 1 }; };template <> struct fmt<T_Q5_1> { enum { QK = 32, BB = 24, UNIT = 32, TAB = 1 }; };+// trellis: a unit is one lane of 4 steps = 16 weights; TAB = the 2048-point codebook as half2 words+template <> struct fmt<T_TQ2T> { enum { QK = 128, BB = 34, UNIT = 16, TAB = 2048 }; };+template <> struct fmt<T_TQK6> { enum { QK = 128, BB = 26, UNIT = 16, TAB = 2048 }; };+template <> struct fmt<T_TQK7> { enum { QK = 128, BB = 30, UNIT = 16, TAB = 2048 }; };static inline uint rd32(device const uchar* p) {device const ushort* s = (device const ushort*)p;@@ -75,7 +82,277 @@ static inline uint g_iq2xxs(threadgroup const uint* tab, uint i, int w) { returnstatic inline uint g_iq2xs(threadgroup const uint* tab, uint i, int w) { return ctab ? uint(iq2xs_grid[i] >> (32 * w)) : tab[2 * i + w]; }static inline uint g_iq2s(threadgroup const uint* tab, uint i, int w) { return ctab ? uint(iq2s_grid[i] >> (32 * w)) : tab[2 * i + w]; }+// BEGIN tq_lut (generated by metal/tools/gen_tq_lut.py from third_party/ggml/ggml-common-tq.h; do not edit)+constant uint tq_lut_u[2048] = {+ 0xb8f7bd4d, 0x3cd5b6ce, 0x4368bd9e, 0xb8a63dce, 0xb430395e, 0x3dbc3bc1, 0x3c9cb92b, 0x3aaabca2,+ 0xb8e13cdb, 0x3dfbbb33, 0xbb183a8d, 0x3da0bd64, 0x3b83bf1a, 0x3659b389, 0xabd7b40f, 0x35a240ee,+ 0x3972c057, 0x3996be17, 0xb9b5b926, 0x3c75393b, 0xbdef4151, 0xbd09b5a3, 0x4012b408, 0x3ece3b7f,+ 0x4011b688, 0xbe303b17, 0xb4bf3d9b, 0xb8c8b870, 0x38b2b537, 0x34b031be, 0x38713389, 0x38b8ba33,+ 0x3b3dbce0, 0x3d3d3367, 0x3cbe3f50, 0xb99130b0, 0x39863733, 0xb6d9b70e, 0xbd98be8c, 0xbe04bcca,+ 0x403c36a9, 0x3eaeb527, 0x39823c6a, 0xbd5bb655, 0x35963ebb, 0xbbbd417e, 0xb1d32fe0, 0x31c0b858,+ 0xbc10b54e, 0xaf8b3cf0, 0x3f5f38e6, 0xb2f63b87, 0xb675b114, 0x329829bf, 0xb955bb7b, 0xa98434e9,+ 0x378fc04a, 0x3c8c31a0, 0x3c682dea, 0x2d292fe1, 0xbd722c33, 0x3e3e379d, 0x3cf2b918, 0x38bfbcbe,+ 0xbdd53284, 0xbd013707, 0x38c1bec1, 0x3838b960, 0x30103413, 0x3672376d, 0x3f42302f, 0xbe9a3a61,+ 0xbc713c06, 0xbcf2bb57, 0x3c85a9dc, 0xb8b23873, 0x340d2d1e, 0xb73f3661, 0xb8b7bb91, 0x3943b956,+ 0x39853958, 0xb5563b26, 0xc2b9ae17, 0xb9ea34e9, 0xb6f433d5, 0xb1af3826, 0xaf98b8a4, 0xbc693a2d,+ 0x358e3a25, 0x32fd406d, 0xb11a38a4, 0xb02eb170, 0xb5cbaa0a, 0x3d453c04, 0xb55bb258, 0xbbf5b412,+ 0x3f4b3d33, 0x38a9bc6e, 0xb715b8c0, 0xbdd92ec6, 0x2980beb3, 0xbb3c3c54, 0x2d683ba0, 0xb20942f9,+ 0x3337b557, 0xbda73870, 0x35bd2d57, 0x385eb8cd, 0xb68e39fe, 0xbd743eda, 0xbd543643, 0x37d43a32,+ 0x2f7f38f8, 0x380f37f3, 0xbc231b1e, 0x35c825c7, 0xb82a2e3e, 0x3897b85f, 0x3cb33c0c, 0x377bc13e,+ 0x3f09b3a4, 0x3c73c006, 0x3ad1bda5, 0xa3cfb0c6, 0xb680b9d5, 0xb493384b, 0xa9853a3c, 0xba24bcb5,+ 0x3c21bd0c, 0x34dc2f68, 0xbdb7bc5d, 0xbddbbbbf, 0xb95ab4c4, 0xbdaab59b, 0xbdce34cf, 0x3f9d3bad,+ 0x3bd53a05, 0x2c64b69c, 0x3adb4003, 0x35e83d4c, 0xb5cc308d, 0xbb86bd2d, 0x3cd3b124, 0xb2ab3a0a,+ 0x3d4bb507, 0x284cb190, 0xbe83384c, 0xb806bc94, 0xb14631e4, 0x3178bbe0, 0x3819bc9d, 0x2e8f3824,+ 0x4150c246, 0x3a43bcf3, 0xbdd53da9, 0x3c47b229, 0xc001400e, 0xb0c33bb2, 0x3822bd46, 0xa4a9b92d,+ 0x3cfa3c72, 0x42843550, 0xbc97b9e5, 0x357ab996, 0xaf70ba03, 0xbc0735e7, 0x397bb663, 0x29423c59,+ 0xb689adf7, 0x3db8b875, 0xb63c35ea, 0x33493674, 0x3ae03926, 0xb56ebb20, 0xa9ae3f05, 0x3c383ce5,+ 0xb1ef40b8, 0xba2bba73, 0x3d3fbdd7, 0x3c093822, 0xbce4bd87, 0x36beba38, 0xbe23b808, 0xbd77ba33,+ 0x34663b90, 0xb805c1c5, 0x375b34c0, 0xa2dbb4be, 0x3495b96c, 0xb99ab38a, 0x31d5390a, 0x3e16a910,+ 0x3dd62878, 0xbdc03964, 0xb3efb783, 0xb571ba12, 0xc0f2b8ce, 0x3771bd8f, 0x3eba3656, 0x3c39be0e,+ 0x371b3ac5, 0x40713469, 0x2e19bd99, 0x3cb83ab2, 0xa92c36d9, 0xbc9bbcb4, 0xb4e2a0fa, 0x31fd34ed,+ 0xb0a04033, 0x342ebac8, 0xbccd3df5, 0xbc5fbc5a, 0x3ada3f32, 0x39ec3878, 0x3ce53db1, 0xbe93b891,+ 0xb75e3c11, 0xbb27b82c, 0xba6736d6, 0x352cb602, 0xbf3a3507, 0xbcf5b8e9, 0x3773b55e, 0xbd0ab832,+ 0x3989bf01, 0x3d733edf, 0x3bc8bb3d, 0xb7e0bf82, 0x393f4053, 0x2f91b07d, 0xb6afbc8b, 0x2a463538,+ 0xbbfd3c2f, 0x324ebfc9, 0xb7023a8f, 0xbda9b939, 0x3c3f3ac3, 0x37983875, 0x3e1b3dc4, 0x255eb820,+ 0xbe92a7e2, 0xb7c7b63f, 0xbcb438bd, 0xb9583a2f, 0x34fcbdf2, 0xb5a83ba1, 0x30473975, 0x21daaecc,+ 0x3bc4b7f7, 0x2eb639f8, 0xbb3abb3f, 0xb13dbd9b, 0xbab237fa, 0x30743794, 0xbad3bd7e, 0x346b3d65,+ 0xba613cfd, 0xb8feadd5, 0xb6ecbe09, 0xb544c05b, 0x2fdb360c, 0x39e5b8ba, 0x3e96b8a8, 0x38cf40cf,+ 0xac5aba8b, 0xb284b691, 0xb5d5c0b9, 0x3d003864, 0xa3823220, 0xbb8f3422, 0xb85eba85, 0x3eb33854,+ 0xb885c110, 0x2e60bdf8, 0xb8ba1cb3, 0x3636b4d3, 0xaf93382a, 0x346abc81, 0x31483d66, 0x36fec0b3,+ 0x4001af70, 0x3eac3caf, 0xb614bd48, 0x3de83d04, 0x31d03f2b, 0x30ceb955, 0x2d33b81b, 0xb657b328,+ 0x3527b7c2, 0xc255b9f6, 0x3c90bc1a, 0x35f3be9c, 0x409ac11d, 0xbf2f30db, 0x3285be02, 0x2f14b607,+ 0x35c23c56, 0x30ac3288, 0xc25c3c1c, 0xbbd63ca4, 0x29c83465, 0x412937a3, 0x32253151, 0xad412ef2,+ 0xbfc1bbcb, 0xbe71b0c0, 0xbb59b17d, 0xbefb3cfe, 0x39f0bdb8, 0x28a74014, 0x396ab771, 0xc063bbdc,+ 0x2898babe, 0x3de9b683, 0xaa78bd1f, 0x3ded4187, 0x3d7b3e18, 0xab0caf3c, 0xb52438a7, 0xc38fb964,+ 0x3d02beda, 0xb25daa74, 0xbe9a2d90, 0xbbb0be23, 0x3a8cbf6a, 0x3d87bbfa, 0x3571b53c, 0xc027385c,+ 0xb801b903, 0x3b7b3368, 0x3ab8b70c, 0xb62538b5, 0x3cb1b436, 0x3f2c8d2a, 0x2c53b88e, 0x393bb2c8,+ 0x3f40bdbf, 0x39d9bc1b, 0x306dbc3a, 0xb8fcb3ba, 0x3c75c07b, 0xb421b655, 0xb7203d34, 0xb6a037f1,+ 0xbec7b338, 0x334eba58, 0xb8103aab, 0xbe3db3d7, 0x40a2b984, 0xba54b5cd, 0xb63e3afa, 0x394dbd80,+ 0x2ddaba54, 0xb8e4baec, 0xad382b2a, 0x3886bd82, 0xc015ac6b, 0x3c72368b, 0xbaa72fb3, 0xb2cb38fb,+ 0xbfb5335d, 0xa535aba9, 0x38b7afb8, 0x35243d00, 0xaeac3b42, 0xb1a64144, 0xba3ab352, 0x39f32c72,+ 0xb82c4079, 0xb45fb9e5, 0xb9f59d20, 0xbd36b2ee, 0x2f11bad5, 0x3e803450, 0xb684213f, 0xbd073200,+ 0xbc5eb839, 0x3f5dc217, 0xbe943d9a, 0x33813947, 0xb9439081, 0x37953eac, 0x3915ad61, 0x3d5db810,+ 0x33f239cb, 0x41edbfe8, 0xb4a23ee7, 0xb88ab9e5, 0xc0da36b3, 0x406cb57f, 0x3c5cba70, 0x3ae4b936,+ 0xb7ff3e2c, 0xb08e35b6, 0x3c3ab8d1, 0xafed227e, 0x310a3a3d, 0xb9262d00, 0x3f76bbe1, 0x3e15b8aa,+ 0xc074b31c, 0xbcb640ee, 0x3b033449, 0xbd0138f5, 0x386abc2d, 0xbcfc3f94, 0xb04933a2, 0xb86e3c77,+ 0xb91c366c, 0x35c6b832, 0xbc4c36ab, 0x3a04b6b9, 0x3c84b2c9, 0xb7623006, 0x3580330e, 0x39263d32,+ 0xbfb238f9, 0x3de6c00a, 0xafad3a35, 0x3d88be69, 0xbe12c30f, 0x3d26b13d, 0xb7a5382a, 0x3ac340fa,+ 0xc028b8a9, 0x3e19b979, 0xbdd7bac1, 0xb802bd42, 0xbe1ab631, 0xac7ab8fb, 0x38193ff8, 0x42e83d3f,+ 0xb1dcb312, 0xaa42c036, 0xc078bd28, 0x36ddb6f5, 0x31fdbad4, 0xb7e93933, 0x31d439b4, 0xb4aebd13,+ 0x3c5d37d3, 0xbafd3925, 0x38753b52, 0xa5812a38, 0xb4503b51, 0x3cccc248, 0x3ad2b1aa, 0xbe9036eb,+ 0x34e3423c, 0x3bc13ac2, 0x37493be5, 0x3ceebcd1, 0x40173ac7, 0xbfc43a67, 0xb9a8b7eb, 0xbf08bf94,+ 0xa983b711, 0xb6093c0f, 0xb7f73ebd, 0xbca734f4, 0x38b1bac3, 0xb93fb5d0, 0x3f1d3e04, 0x305cb7fd,+ 0xbcaaaae3, 0x3d8cbb0f, 0xbd3fbca4, 0xbc08bb3b, 0xb9de3a90, 0xad25377f, 0xbefeb988, 0xbc6cbdf7,+ 0x35c3ba23, 0x370abb14, 0xb3a93c45, 0x3c72b889, 0x3e713a36, 0xb8e03a90, 0x34a634cb, 0x3c6c40be,+ 0xbc3fbe95, 0x2284ba4d, 0xb73eb96e, 0xbc04384d, 0x3a03b452, 0xc10041a8, 0x2ed23c75, 0x3ab6bc47,+ 0x3a633746, 0x3735b62f, 0x3c2fb9dd, 0xb08fbba8, 0x3289b270, 0x3c1e3972, 0xb5ae3844, 0x3d303cee,+ 0x3fa2b98f, 0xb1503e16, 0x382c3c9c, 0xae2e3483, 0x3a843b71, 0x32f4bb53, 0x3ab23540, 0xba06b6e0,+ 0x3701bcc9, 0x361ab89a, 0x3a74be12, 0x37ae3b56, 0x38eeb6ec, 0xb60ab47a, 0x374a3247, 0x3883394d,+ 0x2c41b76f, 0xb862bc07, 0xbba5395e, 0x3865b71f, 0x2cbc3897, 0x35c9ac7a, 0x3ac5b9df, 0xb8b9b63b,+ 0x3ff6c064, 0xb8293ba3, 0x3b7eb904, 0xb78eafdb, 0x3c22baec, 0x39a5b53d, 0xc0d4bab6, 0xbddbb47e,+ 0xc01cba66, 0xbcb037ec, 0x3ac3b409, 0x3e472b2b, 0xb8e03b8a, 0x3a7738cd, 0xb7243cb0, 0x381fbaef,+ 0xb0f9ba7a, 0xb1383940, 0xb8403855, 0x35f439af, 0x2c193b0f, 0x3b833760, 0x3801a813, 0x41d6b801,+ 0x3ab33026, 0xa9f5bca3, 0x370ebf1e, 0x3467b82b, 0xbd57b4c3, 0xb4e9ad4c, 0xbad6b8c4, 0xbd66a8f7,+ 0x3cd23e70, 0x3421236c, 0x3c22aa8a, 0x9661b9c2, 0xb6223988, 0xb8aab797, 0x305c254c, 0x271bbd53,+ 0xbaa93c11, 0x3601b75b, 0xb4e936d1, 0xb15db856, 0xc0b2be96, 0x3e32b7c5, 0x3e22b069, 0x3b41bc18,+ 0x31cac16e, 0x35f9365a, 0xb700b491, 0xb41b3a6a, 0x406a2f26, 0xbd54b8a6, 0x39e33b9b, 0xad0fbeae,+ 0xb1663b1b, 0x3655afd4, 0xbaa7bde7, 0xbd6c37c1, 0x3cbfbc6d, 0x2dbd3584, 0xb9f93d69, 0xac53b992,+ 0xb2a2bf6b, 0xc15735b5, 0x328e331a, 0xb7b3ba14, 0x38093c44, 0x33613c08, 0xbc1c3b93, 0xbd213b53,+ 0x39b2b9d1, 0xbad9326b, 0x35e6b1c8, 0xaeff38ff, 0xb734b831, 0xbbe1ba96, 0x36fb33f8, 0xc15ab049,+ 0xb7a33b22, 0xbc3db462, 0xb57cbf97, 0xbcaa3c86, 0x3c6ebcb9, 0xb0f539ca, 0xb5bf379f, 0xbcb6b629,+ 0x3f0f3aab, 0x34fcc01a, 0xb6be3869, 0x32f4b953, 0xbdb7b85f, 0x411a39f4, 0x203cbce8, 0x2a593824,+ 0xa60cb26f, 0x3d96b47f, 0xb53fb8d0, 0xb69f3706, 0x36cfb1c9, 0x37c1ad03, 0xbc903d80, 0xb98eba26,+ 0x334b3c5b, 0xb051c196, 0xbd79bd24, 0x3e2f3b38, 0xb64dba82, 0x34ed2a56, 0x2a01b36f, 0xbfbc40e9,+ 0xb600b7ce, 0x3a9c3cce, 0xb23c33c6, 0x36c83c33, 0x409ea776, 0x3c35b822, 0x30ef3e43, 0xb9f7b9b2,+ 0xba813c79, 0xaedb3973, 0xba33bfe1, 0x3d8a348e, 0xbfa1b5ac, 0x31b7b8d9, 0x34813aee, 0xb56e3d3b,+ 0x2c653d98, 0x3d7439e0, 0x3ae2b51a, 0x3bb9ae6c, 0x1d86bdc8, 0x3840b9d9, 0xc0f1bc60, 0x396d3e26,+ 0xa4783dc5, 0xc01bbf35, 0x39c8b938, 0x3fd035d1, 0x3d41b3bc, 0x3bf9bca8, 0x32f5bd36, 0x34c83f59,+ 0x38543e38, 0xb82e338e, 0xa3ce37b2, 0x37fdb683, 0x3f02c104, 0x38b3b92f, 0x3bcdb454, 0xba4231f4,+ 0x3878bf4d, 0x3559bf7f, 0xbc53b8e0, 0x407ebd77, 0xb0c6b69e, 0xbabdacce, 0xb92f3c1b, 0x3a5fb8fa,+ 0x3f13b628, 0xbacd3893, 0x3ec8c047, 0xba7fb70b, 0xb140b997, 0xb4662c0c, 0x3ddeba3f, 0x3ae1be75,+ 0x3879b466, 0x3c553d68, 0x3c6eb4d9, 0xb803be71, 0x2a523e15, 0xbc373e1b, 0xb7473585, 0x3db4bf22,+ 0x31a6bcdb, 0x2e23b415, 0xb884ab25, 0xb5ef329e, 0xb59c3525, 0x39c2ba76, 0xbf1eb6f5, 0x2dadbcdf,+ 0x40193e8c, 0xbb4ab92c, 0x2ddb3494, 0x3afd406e, 0xb97a3b91, 0xbce13c1f, 0x3c343eae, 0x38ecbddd,+ 0x3e4f390f, 0x3e04bc20, 0xba0b378c, 0xb8aa330d, 0x348e3e34, 0x352fbaa5, 0x3d2bbac3, 0xbe71b982,+ 0xc1e1305c, 0xbe12be05, 0x2fc1b70b, 0xbe713c8b, 0x3dcc35ad, 0xb612b581, 0xb50cb42c, 0xb36cb1c9,+ 0x3977b443, 0x3d11bd4c, 0x3b1739b8, 0xbd27269a, 0x3d6cbce0, 0xb611b86d, 0xb50c35e7, 0x359cb453,+ 0x38bf38c6, 0x39a6b82a, 0x3bb1a8c2, 0x3c8bb020, 0x2dfa2c82, 0xb43ebb37, 0xbbbd2b23, 0xbd253c97,+ 0xbb813d21, 0xbe12b8e5, 0x3d6a3845, 0xc0ecb571, 0x34ba3c69, 0x2818b96f, 0x382dbdde, 0x3f333f0c,+ 0x3c30b44b, 0xbef5aebd, 0x26833a3b, 0xb7f8aaf0, 0xb4f5405e, 0xb1973c5b, 0xbada359f, 0xbb773227,+ 0x3dec3877, 0xb7c634a2, 0xba0e3bde, 0x20ccb758, 0xb8e83760, 0x3cb8bdaf, 0x385d3cf3, 0x3078306f,+ 0x34d1ac7a, 0x402fb9b8, 0xbc802d2e, 0xbeefb55d, 0x356a348e, 0x3b683511, 0x32ea35a4, 0x3a133e99,+ 0xb8c94002, 0x355a3dc1, 0xbb783556, 0x37ee3560, 0xbc4aba6e, 0x33ebb14d, 0x3c4a22a9, 0x3a27bd5b,+ 0xba1dc0ca, 0xbd314053, 0xb87b3c1a, 0x36e338de, 0x2b952663, 0x36853b4f, 0xc020353c, 0x4050be97,+ 0xa53abc48, 0xbeea3e4e, 0xbb52ac24, 0x40d2b743, 0x3762b389, 0xbb02b493, 0x342f3cb4, 0x3aceba83,+ 0x3923bc28, 0xc0afc004, 0xb842b898, 0x276038d0, 0x405eb83e, 0x4254affb, 0x3c6f3b69, 0x3bedba70,+ 0xba70b0de, 0xb6353d9c, 0x3de8bde3, 0x38e53321, 0x40ae369f, 0xbaf236e3, 0xbb9baf7b, 0xc1f3b484,+ 0x40973f5a, 0xaead3da5, 0x1e3a3934, 0x409fbc67, 0x3a55b2ed, 0x385a3009, 0xb6ac3166, 0xc3f83832,+ 0xa95fb800, 0x2f61bc8a, 0xbeba353c, 0x3ce33284, 0xade83fad, 0xb7c13da0, 0x382db56e, 0x3855385f,+ 0x2ff5bd34, 0xb290ba07, 0xb6f4bda4, 0x3ad2b89b, 0xbe273613, 0x35fc3838, 0x3c9628e1, 0xbd623c1e,+ 0x356b315a, 0x3b163b05, 0x3ed5baf3, 0x3c65bf42, 0x378a3d3f, 0xbd43bf4e, 0x39643c0d, 0x3c0034c3,+ 0xbafe345a, 0xba8bbf4f, 0xb95b4240, 0xc0a23b21, 0x331db6ed, 0x3111b57a, 0xb8ca3594, 0x3d96c16a,+ 0x3c86b7a9, 0xb498b949, 0x3c1e2d89, 0x2eebad06, 0xba8abc5b, 0xb0b9be54, 0x3e71bc9a, 0x3ff43ca7,+ 0xbbbcbf8b, 0x3fefb894, 0x40c93bda, 0xbb663bc7, 0x3cbbb87e, 0xb1f7bb1e, 0x38f63a3d, 0x3cd9ad22,+ 0xbd81bdb4, 0xbd96b005, 0xb91c3d6e, 0xbd5e38f2, 0x2e27b513, 0xbe1c3e5b, 0x366e34b3, 0xb6cdbb24,+ 0xbfb436f7, 0x34a3b00c, 0x37dfb1bf, 0x33d3b9d6, 0x3edf396f, 0xb6f7b5d1, 0x38b22aa1, 0x223d36b1,+ 0x3d3ebc6a, 0xb7b5b777, 0xb91cb8ed, 0x35f53be6, 0xbd74c077, 0x3966bcdc, 0x3d0bbc12, 0x28cb3312,+ 0x4128bd0d, 0xb80ebef1, 0x3eb4b758, 0x352fb6d3, 0x3ca3343d, 0xbaefb733, 0xbc1dbce4, 0x38fa3eac,+ 0x3bceb9d1, 0x36ec3953, 0x3985bbb4, 0x3fe24005, 0xb9913f4e, 0xbe2c39d5, 0xb37fb44e, 0x30aa3acf,+ 0x40c3b358, 0x3b9b3c1f, 0x39a739ec, 0x41113d01, 0xbb76367d, 0x3a483c75, 0x3c01b547, 0x3ae03d2f,+ 0xb837c3b1, 0x36c12e1d, 0xbd8eb2d3, 0x3d80b23a, 0x3c9bbd29, 0x348c37a8, 0xb5aa3c9e, 0x40d33311,+ 0x4028256d, 0xa66e2e47, 0xb8de390a, 0x3c033306, 0xbb3fbc58, 0xbc21bc05, 0x28e2376c, 0x3ddab536,+ 0x3ebb2427, 0xbdbfb70a, 0xbb103cc4, 0x2d293d33, 0xbfd0c044, 0x3726bfd3, 0xad3432d9, 0x3d32c040,+ 0x2b123f66, 0x3ae9377a, 0xc02d3088, 0x38e2384f, 0xacfb3c2b, 0xb9debbda, 0x39452d0a, 0x40ba38f1,+ 0x1b913858, 0xba152f58, 0x3a7f3642, 0x387a39d4, 0x3efcb9af, 0xbcc6c007, 0x3938ba9b, 0xaa27b635,+ 0x3509a2e3, 0xbb66b38b, 0xb775bbc3, 0x33f5bbc7, 0xb6bac014, 0x340ba91a, 0x3dadc0ad, 0xb70ea94d,+ 0xbcf0b9b4, 0xb38a2f73, 0x304e34f1, 0xbfe63cdf, 0x34c3c329, 0xbdc03c80, 0x39ed37bd, 0xaaa3393d,+ 0xbd493a94, 0xb4d2c288, 0xbfb62c3e, 0xb93dbca0, 0x3200b9d6, 0x3875ac36, 0x33f73072, 0xb8f73e47,+ 0x3116ba54, 0xc0373a12, 0xb48c3ad7, 0xb798323b, 0xbf50c11c, 0xaf0baf13, 0x2efc3c1c, 0xbc683b04,+ 0x38fab611, 0x3f8cb74f, 0xbf1a3b22, 0x3cefc394, 0x26143ca6, 0xbe0337c3, 0xc0723cd0, 0xc15f3f2e,+ 0x34da3c11, 0x3cf5b4e7, 0x3030c030, 0x3837b32a, 0xbca2c1c0, 0x3ef133dd, 0xac37982a, 0xb7b33c5e,+ 0xb9cdb5b6, 0x2c0cb9e9, 0xab2a3832, 0x419aacea, 0x39223458, 0x3c1fc18e, 0xb7622ae8, 0xbe8c393d,+ 0xbd47b009, 0x3abd3d9b, 0x3c56b95a, 0x31073cfe, 0xbc5637d9, 0xb846312d, 0x3d2e3752, 0x403041d2,+ 0x3bf0b2d1, 0xbc3930c5, 0xb8aab1ee, 0x3d48405a, 0x2a023c11, 0xc184b800, 0x308d3dd2, 0xbd48b975,+ 0xb94232b5, 0xbe0e38c6, 0xc06f3688, 0xbc6133b3, 0xc0943910, 0x38f83cb1, 0xc2b8be18, 0xb3de37d6,+ 0x3b2eab85, 0x299cb8f6, 0xbdb13676, 0x3f26b87d, 0xc1173068, 0x3a16b7ae, 0x3b4430ea, 0x30a9b347,+ 0x39c331a2, 0x302136bf, 0xbc1338f6, 0x338db882, 0x3bf4b892, 0xba38385d, 0x305d3b4e, 0x387b43aa,+ 0xb0c436a4, 0xb925b800, 0xbdefb9d7, 0x4244bba6, 0x345236da, 0xc1913b75, 0x318b381f, 0x405a38a3,+ 0x3d74ba2a, 0x3666b9b5, 0x38023990, 0x37ccb8ed, 0x36e93697, 0xb3ef3d43, 0x3cbfbaa8, 0xadc4bdf3,+ 0xb0b62c73, 0x386abbd5, 0xbecebaaf, 0x3d5835f2, 0x3ca3358e, 0x3d37b987, 0x3cba3770, 0x400f3d8a,+ 0x391037a5, 0x31483bce, 0xb1f2b10b, 0x3a6d343f, 0xb733399b, 0x355038b6, 0xb90abe87, 0xbe4bc1ea,+ 0xbe8fbe96, 0xb92a3999, 0xb1c83a77, 0x3508366c, 0xbcc4b7aa, 0xb591b03d, 0xb9fcbecf, 0x40f5bb2f,+ 0xc0db4073, 0xbf44421c, 0x34cd339f, 0xb4dc37bd, 0xb44cb531, 0x3b383e9d, 0xa9ffba12, 0xb9ce3655,+ 0x3441b32e, 0x3c083f6f, 0xb8603a43, 0x3dfebcf7, 0xbc86c07a, 0xbbf63745, 0x36a62987, 0x267f35e0,+ 0xb9f1b4b8, 0x3a39b865, 0xbd262f53, 0x388c37b5, 0xb8893b0e, 0xa4223c18, 0x1666b894, 0xaf72b37f,+ 0xb0813772, 0xb0eac090, 0x41d033ad, 0x345fb75b, 0xba823de3, 0xb979afd1, 0x36ec3f52, 0x42c7b928,+ 0xb54ebc71, 0x356d3b45, 0x3d77b63d, 0x2873a722, 0xac223ab5, 0x3ccbbe38, 0xba9db82f, 0x34f9b88a,+ 0xb9973dde, 0x399dc1b1, 0xc19a3d55, 0x3e72b380, 0xbd0ab701, 0xba7fc16c, 0x3df039a8, 0xb595bf04,+ 0xb904ba46, 0x35c93ab7, 0xb9dab181, 0x3a40ba29, 0xb257bee1, 0xbabbc262, 0x39fa3d35, 0x39222661,+ 0x3c35b5da, 0xbc02ac1e, 0x375bb9c2, 0xb5e8343c, 0xbd4abc03, 0x38402c70, 0x3a48b98c, 0xb987bd06,+ 0x3a3fc008, 0xadf03676, 0x437f391e, 0x38df4179, 0x3dd23a92, 0xb870b42f, 0xb0b33e99, 0x3c3b3136,+ 0x3b57b98b, 0x3d80af33, 0x3475b578, 0xb6203f58, 0x386c3a60, 0xbc68af9c, 0xb99a3826, 0xb414b8c8,+ 0x374bb952, 0xba0c40e7, 0x40723a49, 0xbc2ab76d, 0xaebbbc67, 0xba0d3a0b, 0x3e433f56, 0xbc433d02,+ 0x3e82b9a2, 0xb7963a1f, 0xbc5c355d, 0xba8a3a70, 0xbd61333c, 0xb9c73caf, 0x31f0bd93, 0xbce5bd05,+ 0xba81b4b9, 0xbbf2bc7d, 0xb9763740, 0xbe7bb53f, 0xb266bced, 0x3bda30be, 0xbab03f03, 0xbfc73bdf,+ 0xbf88adc6, 0x2c42b00b, 0xc2c93e4d, 0x4114be94, 0xbcba3b5a, 0xbd313d7c, 0x37ac3726, 0x3a3a397e,+ 0xbcd02f06, 0x3cb93d0c, 0x3f3735a6, 0xb16141f6, 0x3e57ba84, 0x3e1eb423, 0x2e083eb8, 0xbc7fb1c0,+ 0xc03a3bef, 0xbb16bcc4, 0x3c4a38a5, 0x3bf83b7f, 0x296439c9, 0xb944bddc, 0x3717406a, 0xaef2b80f,+ 0xbf1438c2, 0x37eb3633, 0x3d192c27, 0x37d2ba23, 0x382331dd, 0x39a3bb12, 0xb57cb985, 0x36a5b7fa,+ 0x3b5bb882, 0x34c03856, 0xb60d3a69, 0x3b6ec03c, 0x219c3b95, 0xbdd6ac3a, 0x356f4008, 0x33cbae4c,+ 0x3c41bd84, 0x3b3c3ba0, 0x387ebb56, 0xba143b22, 0x3f8d33cb, 0x18e03508, 0xa89e3b2d, 0x3c7f3c83,+ 0xb82a3cd9, 0x391239b5, 0xbb1c39d4, 0xb1deb8ed, 0xbb923b0b, 0x41783be0, 0x31e83716, 0xbc80b3ff,+ 0x393eba07, 0xba8739a9, 0x4185bb2d, 0x32fdb616, 0xbbe7b9d6, 0x3cee3645, 0xbcbab4c6, 0xac743e24,+ 0xb9ba2b37, 0x31423c42, 0x33e93505, 0xb53ebe07, 0x3b1db609, 0x36e4bd2b, 0x36fb3c8a, 0xb8c339e9,+ 0x42953fba, 0xbf1a3766, 0x3ca339e3, 0xb4343fc7, 0xbb95b899, 0xb2e3398e, 0xb5c8bcd6, 0xb44238c5,+ 0x407d3cf2, 0xb847b6e2, 0xbdcd3a6a, 0xb34234b0, 0x2d943a83, 0xb6443e96, 0x3e5dbead, 0x39cdb0bb,+ 0xba58a92f, 0xbdf93c0b, 0xa45939ad, 0xbd5c3e21, 0x41c83d83, 0xbeb73299, 0xbc3ab260, 0xb98cb6c7,+ 0x3d96321a, 0x3a6f3834, 0x3d353920, 0xbe2828a0, 0xb9dbbe36, 0xbbbab7d3, 0x357fbbd1, 0xc114bd91,+ 0xae9bb5fd, 0xb85bc078, 0xb4c3c123, 0xb79638b3, 0x3ab8bb1b, 0x37e533b3, 0x3a311a62, 0x3a67b4f1,+ 0x3e7bbbce, 0xbc7cb73c, 0x4098408e, 0x342bbe64, 0x39c73da6, 0x1dea4077, 0x3c46af4e, 0xbbe83ebd,+ 0x3cd83936, 0x3e99402f, 0xb2d23c05, 0xb1ebbc89, 0x3d842dfa, 0xbb80bed9, 0xbb7238a5, 0xb173bc2c,+ 0xb35db84d, 0x3e9fb0ff, 0x3b57bc79, 0x307dbb5b, 0x3703b498, 0x38bebe4a, 0xab2eb878, 0xbacc3b46,+ 0x32d83ab4, 0x365bc20b, 0x36ed35ac, 0x2d1b40e5, 0xb0f7b594, 0xbe4cba7a, 0xb070a99c, 0x2dcab21f,+ 0x3fd8bf8a, 0x391bb83e, 0x330fbc35, 0xb69834e7, 0x34c1b482, 0xb81836cd, 0xb4493e69, 0x31ab3646,+ 0x341e38c2, 0xb9873925, 0x41b938e5, 0xb1acb79d, 0x38f936c6, 0xa977c103, 0xbce0bead, 0xb897bc63,+ 0xb8423f5d, 0x38753d63, 0xad0bab3a, 0xbcd625c0, 0xb85138d6, 0xc077b9a9, 0xb4a13be3, 0x39c4c293,+ 0x414b32dc, 0x38cc3ada, 0x3b60bad6, 0xb52e3a85, 0x3d6d3d6c, 0xb855b5a4, 0x27f62ccf, 0xbcf635b1,+ 0xb4a034df, 0xb62abe85, 0xad0d39d4, 0xab88b542, 0x345a35dd, 0x3951353b, 0x2f10b8f8, 0x363fbaae,+ 0x3ca23878, 0x38da35df, 0xbcfdb443, 0x2fdbb87e, 0x36abbc75, 0x3243389a, 0xba7029dc, 0x32502ee6,+ 0x3d29b748, 0xbe2140a9, 0xb60dbba6, 0x3c40bbbc, 0xbb7bb5dd, 0x3bf32631, 0x2ad5bfbb, 0xb41735b7,+ 0x3831afde, 0x3145b651, 0xb23f3cfa, 0xb56f39ff, 0x3f03bd12, 0x362333d8, 0x3e043401, 0x3f6bbe9f,+ 0xa9d730db, 0xb34d31d9, 0x32529f5e, 0x3c75c0f6, 0xb8ec348d, 0x384138d0, 0x20a83abf, 0x2b97c090,+ 0xac01bf35, 0x3722b854, 0x344dc074, 0x39723a91, 0x39b9be82, 0xc02ab62c, 0xba61bb3a, 0x3a292fcb,+ 0xbb4eba51, 0x3d17b604, 0x3b16363e, 0x3ab93c14, 0x3a9cb80d, 0x3936b0df, 0xb8cab50c, 0x35f3bd85,+ 0xb02b2ffc, 0x38fec0b9, 0xba48390d, 0xbc03408f, 0x3646b937, 0xc0c1c22b, 0xbad0bab4, 0x99a8b3f6,+ 0xbf40bc8c, 0xbc802074, 0xb89bb927, 0xa9e533be, 0x3229417f, 0x353cb346, 0xb883395d, 0xbadd3d69,+ 0xa7433c6c, 0x3466bd97, 0x3977bf85, 0x39763618, 0xb9f83e77, 0xc024be06, 0x325abc84, 0x3d18bb6c,+ 0xbca2b930, 0xbfb2b806, 0xb2533578, 0x33f1b48c, 0xbb21b9ac, 0x374ab012, 0x3c0bb0cc, 0x3dc8aca9,+ 0x322cb4d9, 0xb993ab2c, 0x3a913aa6, 0x3da642a1, 0x3e593151, 0x2d8e3751, 0x2d9a3365, 0xb77fbaaa,+ 0x32eeb7f0, 0xbb783dd2, 0xbe1c3d0e, 0xbd05be19, 0xbebdbcfe, 0x36b0bba1, 0x40063962, 0x370d380e,+ 0x3b9c3e08, 0xb2a82bfe, 0xc0383dd4, 0xb32cc00f, 0x39f333b5, 0xb8933691, 0xac1fbb79, 0xbd823d09,+ 0xb867afcd, 0xbbb5c01c, 0xb323bbca, 0x3d5db8d8, 0x2db4bbda, 0x38f9b8ca, 0x3a87b615, 0xc099321c,+ 0x3f8c3a26, 0x36f130a7, 0xc2a5386f, 0xb157b485, 0x3f94b4d2, 0xb1bb2056, 0x35dbbb33, 0xba5135b5,+ 0x845ebe3f, 0xbd67b7cd, 0xb6702dae, 0xc11f399d, 0x378e2c99, 0xbe92b735, 0x3c614021, 0x34c2ba18,+ 0xb927b993, 0xb82ab98a, 0xbf68ba9d, 0xb7f9b507, 0xb71840e7, 0x3cd31e42, 0x3a822b48, 0xbf983ef1,+ 0x3a56b0ad, 0xbc1234a6, 0x31d63fe4, 0xa3153d63, 0xb81a22f5, 0x32d73b45, 0x36893e26, 0xbc0c3d81,+ 0xbcd33d04, 0xb5c836b2, 0xb86b37bb, 0x3c81b658, 0xba20b81b, 0xadad356a, 0x3048be5d, 0x301db9d4,+ 0x3d2dadb0, 0xb415aa8b, 0x413fb4b2, 0xb5163c3c, 0x3572aff4, 0xbbf6b682, 0x3bd3bbce, 0x3d0a3a19,+ 0x35923729, 0x3a21bc5f, 0xb6673c5b, 0x38c73c4b, 0xaecd3180, 0xb9d9bd82, 0x36d1ac5a, 0x2c34c22c,+ 0xb450bcae, 0x388f3f37, 0xb5fdb698, 0x3b4a23d2, 0xae67b721, 0xab023cbc, 0xbd7339d8, 0xbaf198eb,+ 0x381e3ad1, 0xbc12b864, 0xc084c0e9, 0x39713831, 0xb2fa3ca9, 0x3708a167, 0xb9ca405f, 0x31ad3ca1,+ 0x339d383f, 0x364f389a, 0x3db4b77d, 0xb2bcb56a, 0xb4ddba8f, 0x32b6bf46, 0xb90a381e, 0xbf8a3db5,+ 0x37b4bc4e, 0xad30bc0e, 0x3d833c84, 0x3a043a52, 0x4027bc64, 0x3e6d3c18, 0xaf48bcce, 0x325cb052,+ 0xbf1db876, 0xaf28bafa, 0x34983a5b, 0xbc54aab7, 0x3d7e3fdb, 0xbf2039e0, 0xaddcbfc8, 0xb83bb815,+ 0x3798b793, 0xb6c53915, 0x395bbc75, 0xb960358f, 0xbcedbc4a, 0x34c2bb4c, 0x20223cfd, 0x30cf2cce,+ 0xbc4db632, 0x33083a35, 0x38d22f07, 0x3c45bc58, 0xb9adbae1, 0x3910bb37, 0xb91ebf4b, 0xb27e374d,+ 0x367bb59a, 0x39fb3af6, 0x3a2abb7c, 0x2d37bf2d, 0x3803b489, 0xbdc63fc6, 0x3fe4381d, 0xb50833f7,+ 0xbeb94023, 0xb0c63d4c, 0x38c5b9aa, 0x360ab65d, 0x3ad7bd31, 0x39bf3f3e, 0xb920bc24, 0xb7a33774,+ 0xbcb9afd2, 0x387ab627, 0x38cbb7cc, 0xb37eaf26, 0xbad1b2c5, 0xb982b881, 0xbf5fbe79, 0x3cb6b58c,+ 0xc084b76a, 0x305f3888, 0xbb1c4386, 0x2990318f, 0xb415bc45, 0x3b9abd45, 0xb6c7400b, 0x3f4940d8,+ 0x39533b3e, 0x41a54121, 0x3a863e15, 0xb50cb745, 0xbc62394d, 0x36fd3ce8, 0xb7d3b3fe, 0x3663320d,+ 0x39e838fd, 0x38c0b2dd, 0xb5163992, 0xbc1fb019, 0xa8c635e3, 0x39753fda, 0xc0dd3dba, 0x3863166a,+ 0x3ab2a3cd, 0xbcb9332f, 0xb88ebce3, 0x41f6bd5e, 0xbd0aba72, 0xc170bac9, 0xb615bc26, 0xbca7bab0,+ 0xbb6cc082, 0x415e3f5b, 0xbe9b3be8, 0x3f323c55, 0x410aa521, 0xbe1ebf27, 0x39dfbca8, 0xbbf5b924,+ 0x39c7b2aa, 0x424b3af7, 0xaed0b4c6, 0x3532358e, 0x39243116, 0xc199bee5, 0x3849ba6e, 0xb9203c77,+ 0xb4813c82, 0x3c4e33de, 0x3775ba98, 0x96a7b594, 0x4000bd85, 0x3fd82fd0, 0x3bb73d71, 0x38083dc5,+ 0x34a8bc1c, 0xb86c3d3d, 0x38413bf1, 0xb9d3399c, 0xb752bcee, 0xb00a3ab9, 0x3198b744, 0x398f2fcd,+ 0x3d759a32, 0x3bbc360a, 0xbb5937e9, 0xb330b969, 0x3ae6c0a5, 0x40b03e0c, 0x343cbcda, 0xb86c34bc,+ 0x35be3cb0, 0xac1bbd82, 0xa5a7bb08, 0xb0f534bf, 0xb4853087, 0xbb9ca123, 0xb45d3df9, 0x404c3c17,+ 0x39a3347a, 0xb913b6e8, 0x38cebd20, 0x40b0c006, 0xbb4d402a, 0x3e493e7d, 0xb6c83b8b, 0x3b902d82,+ 0xae29388d, 0x3ad7327b, 0xb9843d10, 0x34cc3933, 0xbbda3002, 0x3819bcf3, 0xbc312c89, 0x38e83dbd,+ 0xbb87b4e4, 0x3886b172, 0xb82939b9, 0xb345bd54, 0xba7fb951, 0x36c13d9f, 0x3dd3b259, 0x36a43a49,+ 0x3cf23b65, 0xbe2cacae, 0x3ba3bdae, 0x3603356a, 0x33703d0e, 0x2d88b96b, 0x36e3b8d3, 0x3b24b804,+ 0xbdd0c000, 0xb8c7c006, 0x33673dc9, 0x344bb663, 0x341f3406, 0xc0cfac64, 0x3d0234c6, 0x39f83676,+ 0x2e04a4b0, 0x3b5fb4ee, 0x37d4bb6d, 0xb491b0cc, 0xacc8b1aa, 0x3c6cbea5, 0xbc46b99e, 0x3b6cbe17,+ 0xbf93bd5d, 0x38fab462, 0x331fc0e4, 0x31643419, 0x4066bb25, 0xbe28bd61, 0xbc663891, 0xbc0c32be,+ 0x3dee3055, 0xb3ab20f7, 0x3a5031ff, 0xbad6be6a, 0x3cb5bb8f, 0x3e97be0a, 0xbc8b3110, 0x3c1b36cb,+ 0xbe5fbb83, 0x310baecd, 0x2985b4b8, 0xb9d93c4a, 0xbc1d39c5, 0x3d9e390b, 0xbca43675, 0xb49c3ce1,+ 0x3fffbb08, 0x35e33927, 0xbc60bd50, 0xbd1339b6, 0xc2a24123, 0x37d9beab, 0x3425b8ed, 0x3a36bace,+ 0x4057b0c7, 0x3e0c3c69, 0x3a03b5b0, 0xbaa0ba05, 0x9c4c19c4, 0x3c37b72f, 0xbd023467, 0xb537b522,+ 0x2c4d3673, 0xbdefb17b, 0x3daab95a, 0xb4e3bbe8, 0xbb3a3034, 0xbd9cc116, 0xc1d53891, 0xb874bd9f,+ 0x357fbc4b, 0x3f79b09a, 0xc07e3f0c, 0xbca2bbfe, 0x2af5bb50, 0xbaf7b02f, 0xb509b629, 0x36fc39c8,+ 0x395ca825, 0x3553b901, 0xbbebbd9d, 0xb45139f2, 0x3f3c37bc, 0x386635a3, 0x35fb3046, 0xbcb33999,+ 0x3d1da417, 0x3becbc37, 0x3bfa3c7c, 0xc1d5bcb8, 0x3c403c1a, 0xb73bb23c, 0x3a093546, 0xbb7fbbf5,+ 0xbe3f30e8, 0x3a8b39fe, 0x9b02341a, 0x2b91bc35, 0x3a514256, 0xba433fd1, 0x393ab55c, 0xae6a3c6f,+ 0xb2663da8, 0xba78bd17, 0x3e3a35cf, 0x3e4db5dd, 0x32ff345d, 0x35a2bca2, 0x3c82b9f1, 0x38f1bbc5,+ 0xbe7b3f36, 0xb44abe77, 0x3501bd3b, 0xbe42344c, 0x3dc3bc81, 0xbf152613, 0x3c71bb1c, 0xba6a345f,+ 0x3c03b944, 0xbae1b5ea, 0x3e77ab67, 0xb9dbc04f, 0xb86ebe0c, 0xb9cdbc4f, 0xb07a3ca8, 0xb41f36be,+ 0x3256ac27, 0xb3273870, 0x3b9db5f6, 0xb8de30ba, 0xb06a3c1b, 0x3d543ae1, 0x3d08ba28, 0xb93838a3,+ 0xb96b3472, 0x3b803945, 0x3ab1adbe, 0x39b53ccf, 0x383634a8, 0x308cb49e, 0xb5ba3916, 0x3b1bbb8a,+ 0xbe6fbc56, 0xb4d5bd9e, 0xa3433e83, 0xb2fc3adc, 0x3b55ba2f, 0xb6243cec, 0x37d7261f, 0x3eb1bf76,+ 0xb9c838b1, 0x361ca60d, 0xc00cbca6, 0xbc6bbb55, 0x38ba34c5, 0x383cb802, 0x3e68bd59, 0xbbad3a2e,+ 0x3b243c74, 0x3ebf3010, 0xbfa8b953, 0x3b7cb16b, 0x2c823952, 0x412cb8ef, 0x3cdbb7ec, 0x38e03bc9,+ 0xbaa3bbfb, 0xb1df3f28, 0xbccf3a77, 0xb933b1a5, 0x37cb3018, 0x31bcbec0, 0x3997bd2e, 0x39063930,+ 0x37fdb86f, 0x33e5323d, 0x38d7a60a, 0xb451b30a, 0x31233597, 0xbe8ac063, 0x37afbe42, 0x3b21aff0,+ 0xb277365f, 0x2ad2b5b8, 0x3a45c11d, 0xb50031fd, 0xbc72bf3f, 0x24a33038, 0xb5952890, 0xc1c7c08f,+ 0xbcaab86e, 0x38a63189, 0x325db408, 0x3b052ca7, 0xbf353c55, 0x3b7e387c, 0x3a0b3c20, 0x3db83741,+ 0x3c4d3a12, 0x3d11b87c, 0xbed7bdc9, 0xbdbb2444, 0x3cc9b9c6, 0xba10ae72, 0x395338b7, 0xbbe2b1e6,+ 0x30b1a918, 0x283dbc8f, 0xb160aeb3, 0x33363794, 0x401d3327, 0x3d15bf99, 0x3a32ad02, 0x3c573df6,+ 0x34edbef4, 0x368cbc1f, 0xb95b3aed, 0x3bf338dd, 0x3c2f4185, 0x37b5bbfc, 0x355e37f3, 0x3c463559,+ 0x38bcc003, 0xbda03b53, 0x2acbac8d, 0x396cb8ac, 0xbb1f2be0, 0x3b903ce8, 0x3ef8ae9f, 0x3cf4b33e,+ 0xb8982c97, 0x3f56baa5, 0xbb163e58, 0xc004b33c, 0xbc68400e, 0xa93938bd, 0x3b353a55, 0x3db540d6,+ 0xb8694176, 0xbd7f30ba, 0xb379ba99, 0xb9153ecf, 0x30f2b19f, 0xb6323e11, 0x3be5be94, 0xac5d3d28,+ 0xbb64b6e7, 0x4380b282, 0x36b5bde1, 0x386736c2, 0x2e1c3cca, 0x2e2f31cc, 0xbcffb0d4, 0xbd67bb1c,+ 0x3d2c309a, 0xbd03ab7d, 0xbefcbbe1, 0x3b98bfbe, 0xbc72b53c, 0x3cca2f15, 0xbc36c106, 0x3b57b702,+ 0xb4da391a, 0x3afe3861, 0x335b3ea8, 0xbd6234e4, 0xbc863f1f, 0x35bcbcf4, 0xbf53b34a, 0xbd103834,+ 0x39a1273d, 0xb4e4b839, 0x34da39b8, 0x3a76bed6, 0xc0fa3c50, 0xb061bd35, 0x34f0b1b6, 0xbc183ab1,+ 0xb62cb91d, 0xb829bb36, 0xafabb940, 0x3e813d46, 0xb822b20b, 0x35fdbe39, 0x396632f1, 0xb5432e17,+ 0x3a89bbed, 0xbcc5b2a3, 0xb42b3369, 0x1f58bbd4, 0x3ef4bc48, 0xac2b3bbe, 0xb2e4bdf5, 0x3f94bcae,+ 0xc068a11a, 0xb82e35bb, 0xbbb93f77, 0x3b46b386, 0x22f3b675, 0xbcb23e8b, 0xba29b8ba, 0xbd084235,+ 0x3f9f1038, 0x399fade8, 0xb9da3363, 0x37c83909, 0xbc4c3c82, 0xb788bc3a, 0x39daa897, 0x3beeb6d2,+};+// END tq_lut+// the trellis codebook point i (two fp16 values in one word): from the constant copy, or the threadgroup one+static inline uint g_tq(threadgroup const uint* tab, uint i) { return ctab ? tq_lut_u[i] : tab[i]; }+template <> inline void load_tab<T_Q2_0>(threadgroup uint*, ushort, ushort) {}+template <int F> static inline void load_tab_tq(threadgroup uint* tab, ushort tid, ushort nth) {+ if (ctab) return;+ for (int i = tid; i < 2048; i += nth) tab[i] = tq_lut_u[i];+}+template <> inline void load_tab<T_TQ2T>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQ2T>(tab, tid, nth); }+template <> inline void load_tab<T_TQK6>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQK6>(tab, tid, nth); }+template <> inline void load_tab<T_TQK7>(threadgroup uint* tab, ushort tid, ushort nth) { load_tab_tq<T_TQK7>(tab, tid, nth); }template <> inline void load_tab<T_Q2_0R>(threadgroup uint*, ushort, ushort) {}template <> inline void load_tab<T_Q8_0>(threadgroup uint*, ushort, ushort) {}template <> inline void load_tab<T_Q4_0>(threadgroup uint*, ushort, ushort) {}@@ -304,6 +581,47 @@ template <> inline float deq<T_Q5_1>(device const uchar* b, int, thread float* qreturn rdh(b);}+// Trellis types (ggml-common-tq.h): a 128-value block is {fp16 d; qs}, 32 steps of K bits (K = 8 / 6 / 7). A unit is+// lane ui = steps 4ui..4ui+3, 16 weights. Step t's 15-bit state: TQ2_T qs[(t + 31) % 32] << 8 | qs[t]; TQK the 16+// stream bits from bit (31 - t) * K, circular over 32 K bits. x = state * 0x9e3779b1 selects two codebook points,+// lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs); the four weights of the step are those four values times d.+// Same arithmetic as src/kernels/cuda/iq_kernels.cu (tq_states4 / tq_step) and ref_quant.cpp (deq_tq).+template <int KB>+static inline float deq_tq(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) {+ device const ushort* qs16 = (device const ushort*)(b + 2);+ uint s[4];+ if (KB == 8) {+ const uint w01 = qs16[2 * ui], w23 = qs16[2 * ui + 1];+ const uint prev = b[2 + ((4 * ui + 31) & 31)];+ const uint b0 = w01 & 0xFFu, b1 = w01 >> 8, b2 = w23 & 0xFFu, b3 = w23 >> 8;+ s[0] = (prev << 8) | b0;+ s[1] = (b0 << 8) | b1;+ s[2] = (b1 << 8) | b2;+ s[3] = (b2 << 8) | b3;+ } else {+ constexpr int NW = 2 * KB;+ const int off3 = (28 - 4 * ui) * KB, wb = off3 >> 4, r = off3 & 15;+ const int wi1 = wb + 1 >= NW ? wb + 1 - NW : wb + 1;+ const int wi2 = wb + 2 >= NW ? wb + 2 - NW : wb + 2;+ const int wi3 = wb + 3 >= NW ? wb + 3 - NW : wb + 3;+ const ulong v = ulong(qs16[wb]) | (ulong(qs16[wi1]) << 16) | (ulong(qs16[wi2]) << 32) | (ulong(qs16[wi3]) << 48);+ for (int i = 0; i < 4; ++i) s[i] = uint(v >> (r + (3 - i) * KB)) & 0xFFFFu;+ }+ for (int i = 0; i < 4; ++i) {+ const uint x = (s[i] & 0x7FFFu) * 0x9e3779b1u;+ const float2 a = float2(as_type<half2>(g_tq(tab, x >> 21)));+ const float2 c = float2(as_type<half2>(g_tq(tab, (x >> 10) & 2047u)));+ q[4 * i] = a.x;+ q[4 * i + 1] = a.y;+ q[4 * i + 2] = c.x;+ q[4 * i + 3] = c.y;+ }+ return rdh(b);+}+template <> inline float deq<T_TQ2T>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<8>(b, ui, q, tab); }+template <> inline float deq<T_TQK6>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<6>(b, ui, q, tab); }+template <> inline float deq<T_TQK7>(device const uchar* b, int ui, thread float* q, threadgroup const uint* tab) { return deq_tq<7>(b, ui, q, tab); }+// sum over groups of LPR consecutive lanes (LPR = 1, 2, 4, 8, 16 or 32); the result is valid in every lanetemplate <int LPR>static inline float lane_sum(float v) {@@ -311,3 +629,66 @@ static inline float lane_sum(float v) {for (ushort o = LPR / 2; o > 0; o >>= 1) v += simd_shuffle_xor(v, o);return v;}++// ---- Gyro: the block Hadamard rotation of the activations ------------------------------------------------------------+// Hadamard-folded experts (docs/GYRO.md, include/strata/kernels/cpu/hadamard.hpp) take y = H (s * x) per block of 128+// values (the normalized Sylvester-Walsh-Hadamard matrix, s = a +/-1 sign vector of the width) in place of x: the gate/up+// input and the down input. One thread per value, 128 threads a block, grid (width / 128, rows). The butterflies run in+// the CPU's order (len = 1, 2, 4, ...; lower element a + c, upper a - c), so the rotated floats are the CPU's and the+// CUDA kernel's: len 1..16 by SIMD shuffles, 32 and 64 through threadgroup memory.+struct HadArgs {+ uint width; // values per row (a multiple of 128)+ uint x_stride; // row strides in elements+ uint y_stride;+ float scale; // 1 / sqrt(128), as the CPU computes it+ uint has_signs;+ uint pad;+};+template <typename T>+static inline float had_ld(device const T* p);+template <> inline float had_ld<float>(device const float* p) { return *p; }+template <> inline float had_ld<half>(device const half* p) { return float(*p); }+template <typename T>+static inline void had_body(device const T* x, device T* y, device const float* signs, constant HadArgs& a, uint3 tg,+ ushort tid, threadgroup float* buf) {+ const uint col = tg.x * 128u + tid;+ float v = had_ld<T>(x + ulong(tg.y) * a.x_stride + col);+ if (a.has_signs) v *= signs[col];+ for (ushort len = 1; len < 32; len <<= 1) {+ const float o = simd_shuffle_xor(v, len);+ v = (tid & len) ? o - v : v + o;+ }+ buf[tid] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ float o = buf[tid ^ 32];+ v = (tid & 32) ? o - v : v + o;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ buf[tid] = v;+ threadgroup_barrier(mem_flags::mem_threadgroup);+ o = buf[tid ^ 64];+ v = (tid & 64) ? o - v : v + o;+ y[ulong(tg.y) * a.y_stride + col] = T(v * a.scale);+}+kernel void had128_f32(device const float* x [[buffer(0)]], device float* y [[buffer(1)]],+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {+ threadgroup float buf[128];+ had_body<float>(x, y, signs, a, tg, tid, buf);+}+kernel void had128_f16(device const half* x [[buffer(0)]], device half* y [[buffer(1)]],+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {+ threadgroup float buf[128];+ had_body<half>(x, y, signs, a, tg, tid, buf);+}+// half rows in place, one threadgroup a row (grid.x = the row, so an indirect grid of the row count works): the prompt+// path's h rows (the padded rows of the expert tiles), width values each+kernel void had128_f16_rows(device half* x [[buffer(0)]], device half* y [[buffer(1)]],+ device const float* signs [[buffer(2)]], constant HadArgs& a [[buffer(3)]],+ uint3 tg [[threadgroup_position_in_grid]], ushort tid [[thread_index_in_threadgroup]]) {+ threadgroup float buf[128];+ for (uint b = 0; b < a.width / 128u; ++b) {+ had_body<half>(x, y, signs, a, uint3(b, tg.x, 0), tid, buf);+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+}diff --git a/metal/src/engine/iq_host.cpp b/metal/src/engine/iq_host.cppindex 7c893382b..28dca0355 100644--- a/metal/src/engine/iq_host.cpp+++ b/metal/src/engine/iq_host.cpp@@ -5,12 +5,15 @@#define GGML_COMMON_DECL_CPP#include "ggml-common.h"+#include "ggml-common-tq.h" // block_tq2_t / block_tqk6 / block_tqk7#include <cstddef>#include <cstdint>namespace strata::kernels {namespace {+// agentionai's trellis types (Gyro): TQ2_T (144), TQK6 (145), TQK7 (146); 128 weights a block+bool is_tq(int t) { return t == 144 || t == 145 || t == 146; }bool is_iq(int t) {return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||t == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8; // 14: Q6_K (Unsloth's Q5/Q6 gate/up)@@ -20,6 +23,7 @@ int gu_qk(int t) {switch (t) {case 16: case 17: case 18: case 21: case 22: case 23: case 29: case 12: case 13: case 14: return 256;case 42: return 64;+ case 144: case 145: return 128; // TQ2_T / TQK6 gate/up (the CUDA engine's STRATA_GU_FMTS)case 6: case 2: case 3: case 8: return 32;default: return 0;}@@ -28,6 +32,7 @@ int d_qk(int t) {switch (t) {case 23: return 256;case 42: return 64;+ case 144: case 146: return 128; // TQ2_T / TQK7 downcase 20: case 7: case 6: case 2: case 3: case 8: return 32;default: return 0;}@@ -57,6 +62,9 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {case 2: return (size_t) (n / 32) * sizeof(block_q4_0);case 3: return (size_t) (n / 32) * sizeof(block_q4_1);case 8: return (size_t) (n / 32) * sizeof(block_q8_0);+ case 144: return (size_t) (n / 128) * sizeof(block_tq2_t);+ case 145: return (size_t) (n / 128) * sizeof(block_tqk6);+ case 146: return (size_t) (n / 128) * sizeof(block_tqk7);case 30: return (size_t) n * 2;default: return 0;}@@ -64,8 +72,8 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {bool native_expert_supported(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) noexcept {const int qg = gu_qk(gu_type), qd = d_qk(d_type);- return qg > 0 && qd > 0 && is_iq(gu_type) && is_iq(d_type) && n_embd % qg == 0 && n_ff % qd == 0 &&- n_embd % 256 == 0 && (n_ff * n_embd) % 256 == 0;+ return qg > 0 && qd > 0 && (is_iq(gu_type) || is_tq(gu_type)) && (is_iq(d_type) || is_tq(d_type)) && n_embd % qg == 0 && n_ff % qd == 0 &&+ n_embd % 256 == 0 && (n_ff * n_embd) % 256 == 0; // (the trellis rows: 128-value blocks, 2560 = 20, 640 = 5)}NativeExpertLayout native_expert_layout(int gu_type, int d_type, int64_t n_embd, int64_t n_ff) {diff --git a/metal/src/engine/prompt_chain.inc b/metal/src/engine/prompt_chain.incindex 39efd5e60..f9b24133c 100644--- a/metal/src/engine/prompt_chain.inc+++ b/metal/src/engine/prompt_chain.inc@@ -58,6 +58,7 @@ bool Window::pc_setup(int C) {b.k4 = coop_k4();}b = PromptBufs{b.lib, b.lgdn, b.lqsa, b.k4}; // (re)allocated below for the larger chunk+ b.v3e = b.k4 && T("blk.0.ffn_gate_exps.weight").type < 144; // the trellis experts have the v2 kernels onlyb.C = C;const size_t MT = (size_t) C;const int MB = (int) max_cells_ / 4 + 2;@@ -88,13 +89,14 @@ bool Window::pc_setup(int C) {// the PLE block's rows and projectionsb.pemb = F32(MT * N); b.pembh = F16(MT * N); b.pkey = F32(MT * 10240); b.pval = F32(MT * N); b.keyn = F32(MT * 10240);b.qn = F32(MT * 10240); b.gated = F32(MT * 10240); b.normW = F32(MT * 10240);+ if (had_on_) b.mixr = F32(MT * N);b.Wpk = F16((size_t) 10240 * N); b.Wpv = F16((size_t) N * N);// the expert GEMM tiles (prompt_chain's defaults): v3 gate/up 16,32,32,4 and down 16,64,32,2 with the M5 layout,// else v2b.gu3[0] = 16; b.gu3[1] = 32; b.gu3[2] = 32; b.gu3[3] = 4;b.dn3[0] = 16; b.dn3[1] = 64; b.dn3[2] = 32; b.dn3[3] = 2;- b.meta.as<uint32_t>()[2] = b.k4 ? NFF / (b.gu3[1] * b.gu3[3]) : NFF / 64;- b.meta.as<uint32_t>()[5] = b.k4 ? N / (b.dn3[1] * b.dn3[3]) : N / 64;+ b.meta.as<uint32_t>()[2] = b.v3e ? NFF / (b.gu3[1] * b.gu3[3]) : NFF / 64;+ b.meta.as<uint32_t>()[5] = b.v3e ? N / (b.dn3[1] * b.dn3[3]) : N / 64;std::fprintf(stderr, "strata metal: prompt kernels (mac-kern's prompt chain) for chunks of %d rows, %s, ready in %.0f ms\n",C, b.k4 ? "matrix-unit register operands (COOP_K4)" : "the older expert / attention kernels (no M5 layout)",now_ms() - t0);@@ -290,16 +292,18 @@ bool Window::pc_chunk(const int64_t* tokens, int nT, int64_t pos0, std::string&ma.d_stride = (uint64_t) ma.d_row * N;const size_t ebase = moe_rebase(ma.gate_off, ma.up_off, ma.down_off);const uint32_t ne = N;+ const bool hg = had_gu(l), hd = had_dn(l);+ if (hg) had_rows(ds, b.lib, "pc:", b.mixed, 0, b.mixr, 0, nT, N, hsx_.handle ? &hsx_ : nullptr, false);mtl::Dispatch g;g.p = P_(b.lib, "mm_gather");- g.args = {mtl::Arg::B(b.mixed), mtl::Arg::B(b.stok), mtl::Arg::B(b.XS), mtl::Arg::V(ne)};+ g.args = {mtl::Arg::B(hg ? b.mixr : b.mixed), mtl::Arg::B(b.stok), mtl::Arg::B(b.XS), mtl::Arg::V(ne)};g.tg[0] = 256;g.indirect = &b.meta;g.ind_off = 8 * 4;ds.push_back(g);auto tl3 = [](const int* t) { return std::to_string(t[0]) + "_" + std::to_string(t[1]) + "_" + std::to_string(t[2]) + "_" + std::to_string(t[3]); };mtl::Dispatch gu;- if (b.k4) {+ if (b.v3e) {const std::string fn = "mm_gu3_" + std::to_string(eg.type) + "_" + tl3(b.gu3);gu.p = P_(b.lib, fn, mtl::Constants().b(7, true), fn + "#k4");} else {@@ -307,19 +311,20 @@ bool Window::pc_chunk(const int64_t* tokens, int nT, int64_t pos0, std::string&gu.p = P_(b.lib, fn, mtl::Constants().i(5, 0), fn + "#p0");}gu.args = {mtl::Arg::B(W_, ebase), mtl::Arg::B(b.XS), mtl::Arg::B(b.HS), mtl::Arg::B(b.tiles), mtl::Arg::V(ma)};- gu.tg[0] = b.k4 ? 32 * b.gu3[3] : 64;+ gu.tg[0] = b.v3e ? 32 * b.gu3[3] : 64;gu.indirect = &b.meta;gu.ind_off = 2 * 4;ds.push_back(gu);+ if (hd) had_rows_ind(ds, b.lib, "pc:", b.HS, b.meta);mtl::Dispatch dn;- if (b.k4) {+ if (b.v3e) {const std::string fn = "mm_down3_" + std::to_string(ed.type) + "_" + tl3(b.dn3);dn.p = P_(b.lib, fn, mtl::Constants().b(7, true), fn + "#k4");} else {dn.p = P_(b.lib, "mm_down2_" + std::to_string(ed.type) + dn2_tile(ed.type));}dn.args = {mtl::Arg::B(W_, ebase), mtl::Arg::B(b.HS), mtl::Arg::B(b.YE), mtl::Arg::B(b.tiles), mtl::Arg::V(ma)};- dn.tg[0] = b.k4 ? 32 * b.dn3[3] : 64;+ dn.tg[0] = b.v3e ? 32 * b.dn3[3] : 64;dn.indirect = &b.meta;dn.ind_off = 5 * 4;ds.push_back(dn);diff --git a/metal/src/engine/window.cpp b/metal/src/engine/window.cppindex 042562afa..a792aa5f8 100644--- a/metal/src/engine/window.cpp+++ b/metal/src/engine/window.cpp@@ -23,6 +23,8 @@#include "strata/core/conversation_cache.hpp"#include "strata/core/session.hpp"#include "strata/kernels/ngram.hpp"+#include "strata/kernels/cpu/hadamard.hpp"+#include "strata/kernels/cpu/expert_layout.hpp"#include <algorithm>#include <chrono>@@ -97,13 +99,14 @@ double now_ms() { return std::chrono::duration<double, std::milli>(std::chrono::struct PromptBufs {int lib = -1, lgdn = -1, lqsa = -1;bool k4 = false;+ bool v3e = false; // the M5's expert kernels (v3): not for the trellis types, which have only v2int C = 0, maxrows = 0;double gpu_ms = 0, host_ms = 0, mtp_ms = 0;int gu3[4] = {}, dn3[4] = {};mtl::Buffer R, bo, inj[2], xn, xnh, V, loh, mixed, mixh, qkv, z, al, be, yg, ygh, gqn, gkn, gvc, ggb, gkq, go, qfull,kcur, vcur, iqr, ikr, qcur, qcurh, qidx, scores, qids, part, XQ, xqh, logits, ids, wts, bcnt, boff, eoff, tiles, meta,stok, sw, pos, XS, HS, YE, yr, gsh, ush, hh, ys, sgv, WD, WU, Wqkv, Wz, Wal, Wbe, Wout, Wq, Wk, Wv, Wiq, Wik, Wr,- Wsg, Wshg, Wshu, Wshd, pemb, pembh, pkey, pval, keyn, qn, gated, normW, Wpk, Wpv;+ Wsg, Wshg, Wshu, Wshd, pemb, pembh, pkey, pval, keyn, qn, gated, normW, Wpk, Wpv, mixr;};// The vision path (--vision, GENI requests), from the shared host code: the image rows of the prompt being read@@ -331,6 +334,19 @@ public:void prof_add(bool draft, double ms) { Prof& x = draft ? dprof_ : cprof_; ++x.n; x.all += ms; }private:// the Q8_0 copies of the hyper-connection matrices (the default; STRATA_HC_Q8=0: the BF16 originals)+ // Gyro (agentionai's Hadamard-folded trellis experts, native_experts.txt v5 + hadamard.txt): the gate/up input is+ // H (s * x) and the down input H h, per block of 128; XR_ holds the rotated rows, hsx_/hsh_ the +/-1 signs+ bool had_on_ = false;+ int had_block_ = 128;+ bool had_gu(int l) const { return had_on_ && strata::kernels::cpu::hadamard_spec().gu(l); }+ bool had_dn(int l) const { return had_on_ && strata::kernels::cpu::hadamard_spec().down(l); }+ mtl::Buffer XR_, hsx_, hsh_;+ // one rotation dispatch: rows of `width` values (a multiple of 128) in place or from x to y; f16 = half rows+ void had_rows(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& x, size_t x_off,+ const mtl::Buffer& y, size_t y_off, int rows, int width, const mtl::Buffer* signs, bool f16);+ // the prompt's h rows (half, padded rows of the expert tiles, their count in meta[8]): rotated in place, one+ // threadgroup a row through the indirect grid+ void had_rows_ind(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& h, const mtl::Buffer& meta);bool hc_q8_ = false;mtl::Buffer Q8_;std::map<std::string, size_t> q8off_;@@ -487,6 +503,29 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionStapleHist_ = A(4 * 9 * 10240);auto act = [&](int n) { Act a; a.n = n; a.b = A(4 * ((size_t) 4 * MAXT * n + (size_t) MAXT * n / 16)); return a; };XA_ = act(N); XF_ = act(N); XY_ = act(6144); XQ_ = act(6144); XH_ = act(NFF); XE_ = act(N);+ { // Gyro (native_experts.txt v5 + hadamard.txt): the rotation's signs and the rotated gate/up input+ const auto& hs = strata::kernels::cpu::hadamard_spec();+ had_on_ = hs.any();+ if (had_on_) {+ had_block_ = hs.block;+ if (had_block_ != 128) {+ err = "metal window: the pack's Hadamard block is " + std::to_string(had_block_) + "; the Metal rotation takes 128";+ return false;+ }+ XR_ = A(4 * MAXT * N);+ auto upload = [&](int w, mtl::Buffer& b) {+ if (const float* sg = strata::kernels::cpu::hadamard_signs(w)) {+ b = A(4 * (size_t) w);+ std::memcpy(b.ptr, sg, 4 * (size_t) w);+ }+ };+ upload(N, hsx_);+ upload(NFF, hsh_);+ std::fprintf(stderr, "strata metal: Hadamard-folded experts (block %d): the gate/up and down inputs are rotated on the GPU+",+ had_block_);+ }+ }// RoPE: cos/sin [pos][32] for the first 64 dims, NEOX pairs (i, i + 32), in double as build_rope_tableconst double base = f_.meta_num.count("qwen4exp.rope.freq_base") ? f_.meta_num["qwen4exp.rope.freq_base"] : 1e7;for (int p = 0; p < MAXC + 16; ++p)@@ -518,13 +557,16 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionStaconst char* e = std::getenv("STRATA_HC_Q8");hc_q8_ = !(e && std::string(e) == "0");}- if (hc_q8_) {+ {const double tq = now_ms();size_t tot = 0;std::vector<std::string> names;for (const auto& kv : f_.tensors) {const gguf::Tensor& x = kv.second;- if (x.type != ref::BF16 || x.name.find("hc_") == std::string::npos || x.name.find("_norm") != std::string::npos) continue;+ // BF16 (when the Q8_0 read is on) and, whatever the setting, the Gyro file's Q6_K / IQ4_NL ones: the decode window's+ // hyper-connection kernels read BF16 or Q8_0 only+ const bool conv = (x.type == ref::BF16 && hc_q8_) || x.type == ref::Q6_K || x.type == ref::IQ4_NL;+ if (!conv || x.name.find("hc_") == std::string::npos || x.name.find("_norm") != std::string::npos) continue;uint64_t ne = 1;for (uint64_t d : x.shape) ne *= d;q8off_[x.name] = tot;@@ -549,7 +591,7 @@ bool Window::init(const strata::core::ModelGeometry& g, strata::core::SessionStaif (!st_->read(raw.data(), x.bytes, x.offset)) { std::fprintf(stderr, "strata metal: read failed\n"); std::exit(6); }src = raw.data();}- ref::dequant(ref::BF16, src, tmp.data(), ne);+ ref::dequant(x.type, src, tmp.data(), ne);ref::quantize_q8_0(tmp.data(), ne, Q8_.as<uint8_t>() + q8off_[names[i]]);}});@@ -676,6 +718,35 @@ void Window::gr_read(std::vector<mtl::Dispatch>& ds, const std::string& pre, boocur_inj_ = 1 - cur_inj_;}+namespace {+struct HadArgs { uint32_t width, x_stride, y_stride; float scale; uint32_t has_signs, pad; }; // quant.metal+}+// y = H (s * x) per block of 128 values of `rows` rows of `width` (x_off / y_off: byte offsets; f16: half rows, no signs)+void Window::had_rows(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& x, size_t x_off,+ const mtl::Buffer& y, size_t y_off, int rows, int width, const mtl::Buffer* signs, bool f16) {+ HadArgs a{(uint32_t) width, (uint32_t) width, (uint32_t) width, 1.0f / std::sqrt((float) had_block_), signs ? 1u : 0u, 0u};+ const char* fn = f16 ? "had128_f16" : "had128_f32";+ mtl::Dispatch d;+ d.p = P(lib, fn, {}, pfx + fn);+ d.args = {mtl::Arg::B(x, x_off), mtl::Arg::B(y, y_off), mtl::Arg::B(signs ? *signs : x), mtl::Arg::V(a)};+ d.grid[0] = width / 128;+ d.grid[1] = rows;+ d.tg[0] = 128;+ push(ds, d);+}++void Window::had_rows_ind(std::vector<mtl::Dispatch>& ds, int lib, const std::string& pfx, const mtl::Buffer& h,+ const mtl::Buffer& meta) {+ HadArgs a{(uint32_t) NFF, (uint32_t) NFF, (uint32_t) NFF, 1.0f / std::sqrt((float) had_block_), hsh_.handle ? 1u : 0u, 0u};+ mtl::Dispatch d;+ d.p = P(lib, "had128_f16_rows", {}, pfx + "had128_f16_rows");+ d.args = {mtl::Arg::B(h), mtl::Arg::B(h), mtl::Arg::B(hsh_.handle ? hsh_ : h), mtl::Arg::V(a)};+ d.tg[0] = 128;+ d.indirect = &meta;+ d.ind_off = 8 * 4; // (rows, 1, 1): the padded rows of the tiles+ push(ds, d);+}+void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0) {const std::string b = "blk." + std::to_string(l) + ".";// ---- the mixer: layer 1's read after the PLE is not applied again (its pending write was applied before it)@@ -874,8 +945,10 @@ void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0ac.b(23, ct);const std::string ak = std::string(stream_ ? "#arena" : "") + (ct ? "#ct" : "");g.p = P(lib_, gun, ac, gun + ak);- g.args = {mtl::Arg::B(EW, ebase), mtl::Arg::B(XF_.b, XF_.plane_bytes() * xkind(tg_.type)), mtl::Arg::B(h_), mtl::Arg::B(plan_),- mtl::Arg::V(a), mtl::Arg::B(XF_.b, XF_.plane_bytes() * 4)};+ const bool hg = had_gu(l), hd = had_dn(l);+ if (hg) had_rows(ds, lib_, "d:", XF_.b, 0, XR_, 0, ntok, N, hsx_.handle ? &hsx_ : nullptr, false); // before the host's turn+ g.args = {mtl::Arg::B(EW, ebase), hg ? mtl::Arg::B(XR_) : mtl::Arg::B(XF_.b, XF_.plane_bytes() * xkind(tg_.type)), mtl::Arg::B(h_),+ mtl::Arg::B(plan_), mtl::Arg::V(a), mtl::Arg::B(XF_.b, XF_.plane_bytes() * 4)};if (stream_)for (int c = -1; c < 3; ++c)g.args.push_back(c < 0 ? mtl::Arg::B(st_->smap(), (size_t) l * 512 * 4) : mtl::Arg::B(st_->chunk(c + 1)));@@ -886,6 +959,7 @@ void Window::layer(std::vector<mtl::Dispatch>& ds, int l, int ntok, int64_t pos0push(ds, g);nb_ = true;gemv(ds, T(b + "ffn_down_shexp.weight"), XH_, ysh_, N, ntok);+ if (hd) had_rows(ds, lib_, "d:", h_, 0, h_, 0, ntok * TOPK, NFF, hsh_.handle ? &hsh_ : nullptr, false); // H h, in placemtl::Dispatch dd;dd.p = P(lib_, dnn, ac, dnn + ak);dd.args = {mtl::Arg::B(EW, ebase), mtl::Arg::B(h_), mtl::Arg::B(ye_), mtl::Arg::B(plan_), mtl::Arg::V(a), mtl::Arg::B(h_)};@@ -2023,6 +2097,7 @@ struct Window::PromptGemm {iqr, ikr, qcur, qcurh, qidx, scores, qids, part, XQ, xqh, logits, ids, wts, bcnt, boff, eoff, tiles, meta, stok, sw, pos,XS, HS, YE, yr, gsh, ush, hh, ys, sgv, emb, embh, pkey, pval, keyn, qn, gated, normW;mtl::Buffer WD, WU, Wqkv, Wz, Wal, Wbe, Wout, Wq, Wk, Wv, Wiq, Wik, Wr, Wsg, Wshg, Wshu, Wshd, Wpk, Wpv;+ mtl::Buffer mixr; // Gyro: the rotated gate/up input rows [MT][N]};int Window::prompt_chunk_rows() const { return pg_ ? pg_->MT : 0; }@@ -2063,6 +2138,7 @@ bool Window::prompt_gemm_init(std::string& err) {p.k4 = ::coop_k4(*cx_);const char* eg = std::getenv("STRATA_METAL_PROMPT_EXPERTS"); // v2 | v3 (default: v3 with the M5's layout)p.gu_v3 = p.dn_v3 = eg ? std::string(eg) == "v3" : p.k4;+ if (T("blk.0.ffn_gate_exps.weight").type >= 144) p.gu_v3 = p.dn_v3 = false; // the trellis experts have the v2 kernels onlyconst char* eq = std::getenv("STRATA_METAL_PROMPT_QSA"); // v1 | flash (default: flash with the M5's layout)p.flash = eq ? std::string(eq) == "flash" : p.k4;const size_t MT = (size_t) C;@@ -2102,6 +2178,7 @@ bool Window::prompt_gemm_init(std::string& err) {p.stok = AB(4 * mr); p.sw = F32(mr); p.pos = AB(4 * MT * TOPK); p.XS = F16(mr * N); p.HS = F16(mr * NFF);p.YE = F32(mr * N); p.yr = F32(MT * N); p.gsh = F32(MT * NFF); p.ush = F32(MT * NFF); p.hh = F16(MT * NFF);p.ys = F32(MT * N); p.sgv = F32(MT);+ if (had_on_) p.mixr = F32(MT * N);p.WD = F16((size_t) (LR + HC) * HC * N); p.WU = F16((size_t) HC * N * LR); p.Wqkv = F16((size_t) 10240 * N);p.Wz = F16((size_t) 6144 * N); p.Wal = F16((size_t) 48 * N); p.Wbe = F16((size_t) 48 * N); p.Wout = F16((size_t) N * 6144);p.Wq = F16((size_t) 12288 * N); p.Wk = F16((size_t) 512 * N); p.Wv = F16((size_t) 512 * N); p.Wiq = F16((size_t) 512 * N);@@ -2339,10 +2416,12 @@ bool Window::prompt_chunk(const int32_t* tokens, int Tn, int64_t pos0, std::striif (stream_) { ma.gate_off = 0; ma.up_off = st_->up_in_slot(l); ma.down_off = st_->down_in_slot(l); }const size_t ebase = moe_rebase(ma.gate_off, ma.up_off, ma.down_off);const uint32_t ne_ = N;+ const bool hg = had_gu(l), hd = had_dn(l);+ if (hg) had_rows(ds, p.lib, "pg:", p.mixed, 0, p.mixr, 0, T, N, hsx_.handle ? &hsx_ : nullptr, false);{mtl::Dispatch g;g.p = PP(p.lib, "mm_gather");- g.args = {mtl::Arg::B(p.mixed), mtl::Arg::B(p.stok), mtl::Arg::B(p.XS), mtl::Arg::V(ne_)};+ g.args = {mtl::Arg::B(hg ? p.mixr : p.mixed), mtl::Arg::B(p.stok), mtl::Arg::B(p.XS), mtl::Arg::V(ne_)};g.tg[0] = 256;g.indirect = &p.meta;g.ind_off = 8 * 4;@@ -2375,6 +2454,7 @@ bool Window::prompt_chunk(const int32_t* tokens, int Tn, int64_t pos0, std::strigu.ind_off = 2 * 4;if (stream_) cuts.push_back((int) ds.size()); // the host's turn: this layer's experts put in placepush(ds, gu);+ if (hd) had_rows_ind(ds, p.lib, "pg:", p.HS, p.meta);mtl::Dispatch dn;if (p.dn_v3) {const std::string fn = "mm_down3_" + std::to_string(ed.type) + "_16_64_32_2";diff --git a/metal/src/ref_quant.cpp b/metal/src/ref_quant.cppindex ede5e5b3d..51bae0f90 100644--- a/metal/src/ref_quant.cpp+++ b/metal/src/ref_quant.cpp@@ -12,6 +12,8 @@#define GGML_COMMON_DECL_CPP#define GGML_COMMON_IMPL_CPP#include "ggml-common.h"+#include "ggml-common-tq.h" // the trellis block structs and the codebook tq_lut_f16 (Gyro)+#include "../../src/kernels/cpu/tq_states.hpp" // the CPU trellis state extraction (the reference decoder)namespace ref {@@ -23,6 +25,7 @@ const char* type_name(int t) {case IQ2_XXS: return "IQ2_XXS"; case IQ2_XS: return "IQ2_XS"; case IQ2_S: return "IQ2_S";case IQ3_XXS: return "IQ3_XXS"; case IQ3_S: return "IQ3_S"; case IQ4_NL: return "IQ4_NL";case IQ4_XS: return "IQ4_XS"; case Q2_0: return "Q2_0";+ case TQ2_T: return "TQ2_T"; case TQK6: return "TQK6"; case TQK7: return "TQK7";}return "?";}@@ -32,6 +35,7 @@ int block_elems(int t) {case F32: case F16: case BF16: return 1;case Q4_0: case Q5_0: case Q5_1: case Q8_0: case IQ4_NL: return 32;case Q2_0: return 64;+ case TQ2_T: case TQK6: case TQK7: return 128;default: return 256;}}@@ -43,6 +47,7 @@ int block_bytes(int t) {case Q3_K: return 110; case Q4_K: return 144; case Q5_K: return 176; case Q6_K: return 210;case IQ2_XXS: return 66; case IQ2_XS: return 74; case IQ2_S: return 82; case IQ3_XXS: return 98;case IQ3_S: return 110; case IQ4_XS: return 136;+ case TQ2_T: return 34; case TQK6: return 26; case TQK7: return 30;}std::fprintf(stderr, "ref: unknown type %d\n", t);std::exit(2);@@ -102,8 +107,30 @@ static void get_scale_min_k4(int j, const uint8_t* q, uint8_t& d, uint8_t& m) {}}+// The trellis types (ggml 144 TQ2_T, 145 TQK6, 146 TQK7): 128 weights = 32 steps of 4; step t's state comes from+// the CPU engine's own extraction (src/kernels/cpu/tq_states.hpp), x = state * 0x9e3779b1, the two codebook points+// lut[x >> 21] and lut[(x >> 10) & 2047] (fp16 pairs) give weights 4t..4t+3, times the fp16 scale.+static void deq_tq(int kb, const uint8_t* b, float* y) {+ uint32_t st[32];+ if (kb == 8) strata::kernels::cpu::tq_detail::tq_states<8>(b + 2, st);+ else if (kb == 6) strata::kernels::cpu::tq_detail::tq_states<6>(b + 2, st);+ else strata::kernels::cpu::tq_detail::tq_states<7>(b + 2, st);+ const float d = f16_to_f32(rd16(b));+ for (int t = 0; t < 32; ++t) {+ const uint32_t x = st[t] * 0x9e3779b1u;+ const uint32_t pa = x >> 21, pb = (x >> 10) & 2047u;+ y[4 * t + 0] = f16_to_f32(tq_lut_f16[2 * pa]) * d;+ y[4 * t + 1] = f16_to_f32(tq_lut_f16[2 * pa + 1]) * d;+ y[4 * t + 2] = f16_to_f32(tq_lut_f16[2 * pb]) * d;+ y[4 * t + 3] = f16_to_f32(tq_lut_f16[2 * pb + 1]) * d;+ }+}+static void deq_block(int t, const uint8_t* b, float* y) {switch (t) {+ case TQ2_T: deq_tq(8, b, y); return;+ case TQK6: deq_tq(6, b, y); return;+ case TQK7: deq_tq(7, b, y); return;case Q2_0: {const float d = f16_to_f32(rd16(b));for (int j = 0; j < 64; ++j) y[j] = (float)(((b[2 + j / 4] >> ((j % 4) * 2)) & 3) - 1) * d;@@ -437,6 +464,7 @@ void random_blocks(int t, uint8_t* dst, int rows, int cols, uint64_t seed, floatcase IQ2_XXS: case IQ2_XS: case IQ2_S: put16(b, f32_to_f16(scale * u / 4)); break;case IQ3_XXS: put16(b, f32_to_f16(scale * u / 8)); break;case IQ3_S: put16(b, f32_to_f16(scale * u / 64)); break;+ case TQ2_T: case TQK6: case TQK7: put16(b, f32_to_f16(scale * u / 1.5f)); break; // codebook |point| ~ 1default: break;}}diff --git a/metal/src/ref_quant.h b/metal/src/ref_quant.hindex f20aeee99..47b51cf94 100644--- a/metal/src/ref_quant.h+++ b/metal/src/ref_quant.h@@ -13,6 +13,7 @@ namespace ref {enum : int {F32 = 0, F16 = 1, Q4_0 = 2, Q5_0 = 6, Q5_1 = 7, Q8_0 = 8, Q3_K = 11, Q4_K = 12, Q5_K = 13, Q6_K = 14,IQ2_XXS = 16, IQ2_XS = 17, IQ3_XXS = 18, IQ3_S = 21, IQ2_S = 22, IQ4_NL = 20, IQ4_XS = 23, BF16 = 30, Q2_0 = 42,+ TQ2_T = 144, TQK6 = 145, TQK7 = 146, // agentionai's trellis types (Gyro): 128 values a block, 34 / 26 / 30 bytes};const char* type_name(int t);diff --git a/metal/tests/expert_parity.cpp b/metal/tests/expert_parity.cppindex 3bd15ba35..7c101c0f1 100644--- a/metal/tests/expert_parity.cpp+++ b/metal/tests/expert_parity.cpp@@ -4,6 +4,13 @@// expert_parity [--gu Q2_0] [--dn Q2_0] [--ntok 1..8] [--gucfg L,R,S] [--dncfg L,R,S] parity (synthetic)// expert_parity --bench [--gu ..] [--dn ..] [--ntok N] [--layers 8] [--reps 96] [--sweep] speed//+// Gyro (agentionai's trellis experts, TQ2_T / TQK6 / TQK7):+// expert_parity --gu TQK6 --dn TQK7 [--had-pack <pack dir>] synthetic blocks; with a pack, the Hadamard rotation of+// x (CPU) and of h (the Metal had128 kernel) is part of it+// expert_parity --gguf <Gyro file> --layer L [--had-pack <pack dir>] [--ntok N] [--seed S] REAL experts of layer L (64 of+// the 512, random) instead of random blocks+// expert_parity --had-test [--had-pack <pack dir>] the Metal had128 kernels (f32, f16, f16 rows) against the CPU rotation, bitwise+//// Parity: random valid blocks for 64 experts, random activations x ~ U(-1,1), random distinct top-10 per token,// weights summing to 1. Checked: the plan (groups by expert id, entries in (token, k) order), h = silu(gate) * up// per entry and y = sum_k w * down . h, each against double; reported as max |err| / max |ref| and relative L2.@@ -11,6 +18,9 @@// a different random routing per rep; GB/s = bytes of the distinct experts the kernel must read / GPU time.#include "mtl_rt.h"#include "ref_quant.h"+#include "gguf_lite.h"+// Gyro: the engine's CPU Hadamard rotation (the reference of the Metal one) and hadamard.txt's parser+#include "../../src/kernels/cpu/hadamard.cpp"#include <algorithm>#include <cmath>@@ -36,7 +46,7 @@ struct MoeArgs {};int type_of(const std::string& s) {- for (int t : {ref::Q2_0, ref::IQ3_XXS, ref::IQ2_XXS, ref::IQ2_XS, ref::IQ2_S, ref::IQ3_S, ref::IQ4_NL, ref::IQ4_XS, ref::Q4_K, ref::Q5_K, ref::Q6_K, ref::Q8_0, ref::Q5_1})+ for (int t : {ref::TQ2_T, ref::TQK6, ref::TQK7, ref::Q2_0, ref::IQ3_XXS, ref::IQ2_XXS, ref::IQ2_XS, ref::IQ2_S, ref::IQ3_S, ref::IQ4_NL, ref::IQ4_XS, ref::Q4_K, ref::Q5_K, ref::Q6_K, ref::Q8_0, ref::Q5_1})if (s == ref::type_name(t)) return t;std::fprintf(stderr, "unknown type %s\n", s.c_str());std::exit(2);@@ -118,6 +128,11 @@ mtl::Dispatch prep_dispatch(const mtl::Pipeline& p, const mtl::Buffer& x, const}int g_gu_type = 0, g_dn_type = 0;+// Gyro: the rotation of h between the gate/up and the down dispatch (the Metal had128_f32 kernel, signs of width n_ff)+struct HadArgs { uint32_t width, x_stride, y_stride; float scale; uint32_t has_signs, pad; };+const mtl::Pipeline* g_had32 = nullptr;+const mtl::Buffer* g_hsh = nullptr;+bool g_had_h = false;bool g_dn_prep = false; // Q2_0 down: prepared h (an xprep dispatch) instead of the raw-x decoderconst mtl::Buffer* g_xpx = nullptr;const mtl::Buffer* g_xph = nullptr;@@ -144,6 +159,17 @@ void encode_layer(std::vector<mtl::Dispatch>& ds, const Kernels& K, const mtl::Bd.indirect = &plan;d.ind_off = plan_off + P_IGU * 4;ds.push_back(d);+ if (g_had_h) { // H h in place, one row per entry (the entries are the window's token x k)+ static HadArgs ha;+ ha = HadArgs{(uint32_t)N_FF, (uint32_t)N_FF, (uint32_t)N_FF, 1.0f / std::sqrt(128.0f), g_hsh ? 1u : 0u, 0u};+ mtl::Dispatch hd;+ hd.p = g_had32;+ hd.args = {mtl::Arg::B(h), mtl::Arg::B(h), mtl::Arg::B(g_hsh ? *g_hsh : h), mtl::Arg::V(ha)};+ hd.grid[0] = N_FF / 128;+ hd.grid[1] = a.n_tok * TOPK;+ hd.tg[0] = 128;+ ds.push_back(hd);+ }}if (with_dn) {const int dk = g_dn_prep ? xkind(g_dn_type) : 0;@@ -190,8 +216,11 @@ double median(std::vector<double> v) {int main(int argc, char** argv) {int gu = ref::Q2_0, dn = ref::Q2_0, ntok = 1, layers = 8, reps = 96;Cfg cgu{32, 2, 4}, cdn{8, 2, 4};- bool bench = false, sweep = false;+ bool bench = false, sweep = false, had_test = false;double share = 0.0;+ std::string gguf_path, had_pack;+ int real_layer = 0;+ uint64_t seed = 1;bool cfg_gu_set = false, cfg_dn_set = false;for (int i = 1; i < argc; ++i) {std::string a = argv[i];@@ -207,14 +236,123 @@ int main(int argc, char** argv) {else if (a == "--reps") reps = std::atoi(nx());else if (a == "--share") share = std::atof(nx());else if (a == "--dn-prep") g_dn_prep = true;+ else if (a == "--gguf") gguf_path = nx();+ else if (a == "--layer") real_layer = std::atoi(nx());+ else if (a == "--had-pack") had_pack = nx();+ else if (a == "--had-test") had_test = true;+ else if (a == "--seed") seed = std::strtoull(nx(), nullptr, 10);else { std::fprintf(stderr, "unknown argument %s\n", a.c_str()); return 2; }}(void)cfg_gu_set;(void)cfg_dn_set;if (dn == ref::IQ4_NL && cdn.L == 8 && !cfg_dn_set) cdn = {4, 2, 4};+ gguf::File gf;+ const gguf::Tensor *tg_ = nullptr, *tu_ = nullptr, *td_ = nullptr;+ if (!gguf_path.empty()) { // real experts: the types come from the file+ std::string err;+ if (!gf.open(gguf_path, err)) { std::fprintf(stderr, "%s\n", err.c_str()); return 2; }+ const std::string b = "blk." + std::to_string(real_layer) + ".";+ tg_ = gf.find(b + "ffn_gate_exps.weight");+ tu_ = gf.find(b + "ffn_up_exps.weight");+ td_ = gf.find(b + "ffn_down_exps.weight");+ if (!tg_ || !tu_ || !td_) { std::fprintf(stderr, "no routed experts in layer %d of %s\n", real_layer, gguf_path.c_str()); return 2; }+ gu = tg_->type;+ dn = td_->type;+ }mtl::Context cx;const int lib = cx.compile({"quant.metal", "units.metal", "moe.metal"});const Layout L = layout(gu, dn);+ // Gyro: the pack's hadamard.txt (the signs of the widths 2560 and 640) -> the engine's CPU spec (the reference)+ mtl::Buffer hsx, hsh;+ mtl::Pipeline had32 = cx.pipeline(lib, "had128_f32"), had16 = cx.pipeline(lib, "had128_f16"), had16r = cx.pipeline(lib, "had128_f16_rows");+ const float* sgx = nullptr;+ const float* sgh = nullptr;+ if (!had_pack.empty()) {+ strata::kernels::cpu::HadamardSpec spec;+ std::string err;+ if (!strata::kernels::cpu::hadamard_parse(had_pack + "/hadamard.txt", 48, spec, err)) {+ std::fprintf(stderr, "%s\n", err.c_str());+ return 2;+ }+ strata::kernels::cpu::hadamard_set(std::move(spec));+ sgx = strata::kernels::cpu::hadamard_signs(N_EMBD);+ sgh = strata::kernels::cpu::hadamard_signs(N_FF);+ if (sgx) { hsx = cx.alloc(4 * N_EMBD); std::memcpy(hsx.ptr, sgx, 4 * N_EMBD); }+ if (sgh) { hsh = cx.alloc(4 * N_FF); std::memcpy(hsh.ptr, sgh, 4 * N_FF); }+ g_had32 = &had32;+ g_hsh = sgh ? &hsh : nullptr;+ g_had_h = !had_test;+ std::printf("Hadamard: block %d, signs for %s%s\n", strata::kernels::cpu::hadamard_spec().block, sgx ? "2560 " : "", sgh ? "640" : "(none)");+ }+ if (had_test) {+ // the Metal rotation kernels against the CPU's (hadamard_rotate), bit for bit+ int bad = 0;+ ref::Rng r2(77 + seed);+ for (int width : {N_FF, N_EMBD}) {+ const int rows = 37;+ const float* sg = width == N_EMBD ? sgx : sgh;+ const mtl::Buffer& sb = width == N_EMBD ? hsx : hsh;+ const float scale = 1.0f / std::sqrt(128.0f);+ std::vector<float> in((size_t)rows * width);+ for (auto& v : in) v = r2.sym() * 3.0f;+ mtl::Buffer bx = cx.alloc(4 * in.size()), by = cx.alloc(4 * in.size());+ std::memcpy(bx.ptr, in.data(), 4 * in.size());+ HadArgs ha{(uint32_t)width, (uint32_t)width, (uint32_t)width, scale, sg ? 1u : 0u, 0u};+ auto run32 = [&](const mtl::Buffer& src, const mtl::Buffer& dst) {+ mtl::Dispatch d;+ d.p = &had32;+ d.args = {mtl::Arg::B(src), mtl::Arg::B(dst), mtl::Arg::B(sg ? sb : src), mtl::Arg::V(ha)};+ d.grid[0] = width / 128; d.grid[1] = rows; d.tg[0] = 128;+ cx.run({d});+ };+ run32(bx, by); // out of place+ std::vector<float> want = in;+ for (int r = 0; r < rows; ++r) strata::kernels::cpu::hadamard_rotate(want.data() + (size_t)r * width, width, 128, sg);+ int b32 = 0;+ for (size_t i = 0; i < in.size(); ++i) b32 += std::memcmp(&want[i], by.as<float>() + i, 4) != 0;+ run32(bx, bx); // in place+ int b32i = 0;+ for (size_t i = 0; i < in.size(); ++i) b32i += std::memcmp(&want[i], bx.as<float>() + i, 4) != 0;+ // f16 (no signs): out of place, and the rows kernel in place; the CPU's float result rounded to half+ std::vector<_Float16> hin(in.size());+ for (size_t i = 0; i < in.size(); ++i) hin[i] = (_Float16)in[i];+ std::vector<float> hw(in.size());+ for (size_t i = 0; i < in.size(); ++i) hw[i] = (float)hin[i];+ for (int r = 0; r < rows; ++r) strata::kernels::cpu::hadamard_rotate(hw.data() + (size_t)r * width, width, 128, nullptr);+ mtl::Buffer hx = cx.alloc(2 * in.size()), hy = cx.alloc(2 * in.size());+ std::memcpy(hx.ptr, hin.data(), 2 * in.size());+ HadArgs hb{(uint32_t)width, (uint32_t)width, (uint32_t)width, scale, 0u, 0u};+ {+ mtl::Dispatch d;+ d.p = &had16;+ d.args = {mtl::Arg::B(hx), mtl::Arg::B(hy), mtl::Arg::B(hx), mtl::Arg::V(hb)};+ d.grid[0] = width / 128; d.grid[1] = rows; d.tg[0] = 128;+ cx.run({d});+ }+ int b16 = 0;+ for (size_t i = 0; i < in.size(); ++i) {+ const _Float16 w = (_Float16)hw[i];+ b16 += std::memcmp(&w, hy.as<_Float16>() + i, 2) != 0;+ }+ {+ mtl::Dispatch d;+ d.p = &had16r;+ d.args = {mtl::Arg::B(hx), mtl::Arg::B(hx), mtl::Arg::B(hx), mtl::Arg::V(hb)};+ d.grid[0] = rows; d.tg[0] = 128;+ cx.run({d});+ }+ int b16r = 0;+ for (size_t i = 0; i < in.size(); ++i) {+ const _Float16 w = (_Float16)hw[i];+ b16r += std::memcmp(&w, hx.as<_Float16>() + i, 2) != 0;+ }+ std::printf("had-test width %d, %d rows%s: f32 %d / f32 in place %d / f16 %d / f16 rows %d of %zu values differ from the CPU\n",+ width, rows, sg ? " (with signs)" : " (no signs)", b32, b32i, b16, b16r, in.size());+ bad += b32 + b32i + b16 + b16r;+ }+ std::printf("had-test %s\n", bad == 0 ? "PASS (bitwise)" : "FAIL");+ return bad == 0 ? 0 : 1;+ }std::printf("device %s; gate/up %s, down %s, blob %zu bytes; %d token(s)\n", cx.device_name().c_str(),ref::type_name(gu), ref::type_name(dn), L.blob, ntok);@@ -255,8 +393,21 @@ int main(int argc, char** argv) {g_xph = &XPH;g_gu_type = gu;g_dn_type = dn;- ref::Rng rng(12345);+ ref::Rng rng(12345 + (seed - 1));for (int i = 0; i < 8 * N_EMBD; ++i) x.as<float>()[i] = rng.sym();+ if (g_had_h) { // H (s * x) per token on the GPU, checked against the CPU's; x then holds the rotated rows+ std::vector<float> want(x.as<float>(), x.as<float>() + 8 * N_EMBD);+ for (int t = 0; t < 8; ++t) strata::kernels::cpu::hadamard_rotate(want.data() + t * N_EMBD, N_EMBD, 128, sgx);+ HadArgs ha{(uint32_t)N_EMBD, (uint32_t)N_EMBD, (uint32_t)N_EMBD, 1.0f / std::sqrt(128.0f), sgx ? 1u : 0u, 0u};+ mtl::Dispatch d;+ d.p = &had32;+ d.args = {mtl::Arg::B(x), mtl::Arg::B(x), mtl::Arg::B(sgx ? hsx : x), mtl::Arg::V(ha)};+ d.grid[0] = N_EMBD / 128; d.grid[1] = 8; d.tg[0] = 128;+ cx.run({d});+ int bad = 0;+ for (int i = 0; i < 8 * N_EMBD; ++i) bad += std::memcmp(&want[i], x.as<float>() + i, 4) != 0;+ std::printf("x rotation on the GPU vs the CPU: %d of %d values differ\n", bad, 8 * N_EMBD);+ }{mtl::Pipeline pp = cx.pipeline(lib, "xprep");cx.run({prep_dispatch(pp, x, XPX, N_EMBD, 8)});@@ -266,10 +417,20 @@ int main(int argc, char** argv) {// ---------------- parity ----------------const int NE = 64;mtl::Buffer W = cx.alloc(L.blob * NE);+ std::vector<int> pick;for (int e = 0; e < NE; ++e) {uint8_t* b = W.as<uint8_t>() + e * L.blob;- ref::random_blocks(gu, b, 2 * N_FF, N_EMBD, 1000 + e);- ref::random_blocks(dn, b + L.down_off, N_EMBD, N_FF, 5000 + e);+ if (tg_) { // real experts of the file: expert `ex` of the layer's three tensors+ int ex;+ do ex = (int)(rng.next() % 512); while (std::find(pick.begin(), pick.end(), ex) != pick.end());+ pick.push_back(ex);+ std::memcpy(b, (const uint8_t*)gf.map + tg_->offset + (size_t)ex * L.gu_row * N_FF, L.gu_row * N_FF);+ std::memcpy(b + L.up_off, (const uint8_t*)gf.map + tu_->offset + (size_t)ex * L.gu_row * N_FF, L.gu_row * N_FF);+ std::memcpy(b + L.down_off, (const uint8_t*)gf.map + td_->offset + (size_t)ex * L.d_row * N_EMBD, L.d_row * N_EMBD);+ } else {+ ref::random_blocks(gu, b, 2 * N_FF, N_EMBD, 1000 + e + 100 * (seed - 1));+ ref::random_blocks(dn, b + L.down_off, N_EMBD, N_FF, 5000 + e + 100 * (seed - 1));+ }}std::vector<int> ids;std::vector<float> w;@@ -323,8 +484,14 @@ int main(int argc, char** argv) {u += (double)wu[r * N_EMBD + k] * xt[k];}hr[r] = g / (1.0 + std::exp(-g)) * u;- eh.add(h.as<float>()[ent * N_FF + r], hr[r]);}+ if (g_had_h) { // the reference of the down input: H h, in float as the engine's CPU path does+ std::vector<float> hf(N_FF);+ for (int r = 0; r < N_FF; ++r) hf[r] = (float)hr[r];+ strata::kernels::cpu::hadamard_rotate(hf.data(), N_FF, 128, sgh);+ for (int r = 0; r < N_FF; ++r) hr[r] = hf[r];+ }+ for (int r = 0; r < N_FF; ++r) eh.add(h.as<float>()[ent * N_FF + r], hr[r]);for (int r = 0; r < N_EMBD; ++r) {double s = 0;// the reference down uses the GPU's h (float) so that the y check isolates the down kernel ...diff --git a/metal/tools/gen_tq_lut.py b/metal/tools/gen_tq_lut.pynew file mode 100644index 000000000..c94d23c05--- /dev/null+++ b/metal/tools/gen_tq_lut.py@@ -0,0 +1,50 @@+#!/usr/bin/env python3+"""metal/tools/gen_tq_lut.py - writes the trellis codebook into metal/kernels/quant.metal.++third_party/ggml/ggml-common-tq.h holds tq_lut_f16[4096]: 2048 codebook points of two fp16 values each. The Metal+kernels read a point as one 32-bit word (low half = the first value), so this script rewrites the table between the+"BEGIN tq_lut" and "END tq_lut" markers of quant.metal with tq_lut_u[2048] = lut[2i] | lut[2i+1] << 16.++ python metal/tools/gen_tq_lut.py rewrite quant.metal+ python metal/tools/gen_tq_lut.py --check exit 1 when quant.metal differs from what would be written+"""+import pathlib+import re+import sys++ROOT = pathlib.Path(__file__).resolve().parents[2]+SRC = ROOT / "third_party" / "ggml" / "ggml-common-tq.h"+DST = ROOT / "metal" / "kernels" / "quant.metal"+BEGIN, END = "// BEGIN tq_lut", "// END tq_lut"+++def table():+ text = SRC.read_text()+ m = re.search(r"tq_lut_f16\[4096\]\s*=\s*\{(.*?)\};", text, re.S)+ vals = [int(v, 16) for v in re.findall(r"0x[0-9a-fA-F]+", m.group(1))]+ assert len(vals) == 4096, len(vals)+ return [vals[2 * i] | (vals[2 * i + 1] << 16) for i in range(2048)]+++def block():+ words = table()+ lines = [BEGIN + " (generated by metal/tools/gen_tq_lut.py from third_party/ggml/ggml-common-tq.h; do not edit)",+ "constant uint tq_lut_u[2048] = {"]+ for i in range(0, 2048, 8):+ lines.append(" " + ", ".join("0x%08x" % w for w in words[i:i + 8]) + ",")+ lines.append("};")+ lines.append(END)+ return "\n".join(lines)+++def main():+ src = DST.read_text()+ i, j = src.index(BEGIN), src.index(END) + len(END)+ new = src[:i] + block() + src[j:]+ if "--check" in sys.argv:+ sys.exit(0 if new == src else 1)+ DST.write_text(new)+++if __name__ == "__main__":+ main()