Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Gyro-L: read the pair-trellis types TQP5/TQP6/TQP7 (ggml 147-149) on the CPU, CUDA and the MMQ prompt path (#3806)
Kevin Read38b06316efff
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 57c6b4117..6fdd7d940 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -1149,7 +1149,7 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)# the Gyro trellis decode kernels (TQ2_T / TQK6 / TQK7) on synthetic blocks: decode-once vs per-column bitwise, output# checksums to compare two builds byte for byte, --bench for timings (GPU, no model)add_executable(tq_pascal_bench src/kernels/tq_pascal_bench.cpp)- target_include_directories(tq_pascal_bench PRIVATE ${_strata_gpu_include_directories})+ target_include_directories(tq_pascal_bench PRIVATE ${_strata_gpu_include_directories} ${CMAKE_CURRENT_SOURCE_DIR}/third_party/ggml)target_link_libraries(tq_pascal_bench PRIVATE strata_kernels ${_strata_gpu_runtime_target})add_test(NAME tq_pascal_bench COMMAND tq_pascal_bench)set_tests_properties(iq_multi_parity PROPERTIES ENVIRONMENT "STRATA_KQ_SPLIT=1") # the Q4_K / Q5_K / Q8_0 Split path against the old kernels@@ -1881,6 +1881,11 @@ if(STRATA_NATIVE_EXPERTS)# tree leaves them to the FP16 prompt pathif(STRATA_GGML_DIR AND EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqk6.cu")list(APPEND _strata_mmq_types tq2_t tqk6 tqk7)+ # Gyro-L's pair trellis (types 147-149): the same fork's ggml has their MMQ instances+ if(EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqp5.cu")+ list(APPEND _strata_mmq_types tqp5 tqp6 tqp7)+ set(_strata_mmq_tqp ON) # moe_mmq.cu names GGML_TYPE_TQP5-7 only where the ggml has them (STRATA_MMQ_TQP)+ endif()set(_strata_mmq_tq ON)elseif(STRATA_MMQ_TQ_OVERLAY AND (STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP))# an upstream llama.cpp: the trellis MMQ instances come from an overlay of the pinned ggml-cuda headers written at@@ -1897,6 +1902,7 @@ if(STRATA_NATIVE_EXPERTS)configure_file(third_party/ggml/ggml-common-tq.h "${_strata_tq_overlay_dir}/ggml-common-tq.h" COPYONLY)configure_file(src/kernels/cuda/tq_lut_i8.cuh "${_strata_tq_overlay_dir}/tq_lut_i8.cuh" COPYONLY)set(_strata_mmq_tq ON)+ set(_strata_mmq_tqp ON) # the overlay adds the pair-trellis enumerators and instancesset(_strata_mmq_tq_overlay ON)message(STATUS "trellis MMQ (Gyro prompts): overlay on the pinned ggml-cuda in ${_strata_tq_overlay_dir}")if(STRATA_BUILD_TESTS)@@ -1946,7 +1952,7 @@ if(STRATA_NATIVE_EXPERTS)target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)endforeach()if(_strata_mmq_tq_overlay)- foreach(_t tq2_t tqk6 tqk7)+ foreach(_t tq2_t tqk6 tqk7 tqp5 tqp6 tqp7)target_sources(strata_mmq PRIVATE ${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu)endforeach()# first on the include path: the patched mmq.cuh / ggml.h shadow the pinned ones for this target only@@ -2064,6 +2070,9 @@ if(STRATA_NATIVE_EXPERTS)if(_strata_mmq_tq)target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)endif()+ if(_strata_mmq_tqp)+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQP=1)+ endif()target_link_libraries(strata_prefill PUBLIC strata_mmq)target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)if(STRATA_BUILD_TESTS)@@ -2141,7 +2150,7 @@ if(STRATA_NATIVE_EXPERTS)list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")endforeach()if(_strata_mmq_tq_overlay)- foreach(_t tq2_t tqk6 tqk7)+ foreach(_t tq2_t tqk6 tqk7 tqp5 tqp6 tqp7)list(APPEND _strata_mmq_sources "${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu")endforeach()endif()@@ -2166,6 +2175,9 @@ if(STRATA_NATIVE_EXPERTS)if(_strata_mmq_tq)target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQ=1)endif()+ if(_strata_mmq_tqp)+ target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_TQP=1)+ endif()target_link_libraries(strata_mmq PUBLIC ggml-base hip::host roc::hipblas)target_link_libraries(strata_prefill PUBLIC strata_mmq)target_compile_definitions(strata_prefill PUBLIC STRATA_PREFILL_MMQ=1 STRATA_PREFILL_FUSED=1)@@ -2257,7 +2269,11 @@ if(STRATA_NATIVE_EXPERTS)# APR: the trellis expert types and the Hadamard activation rotation, when ggml comes from the agention fork# (the only tree with these types; marked by its ggml-cuda/tq.cuh)if(STRATA_GGML_DIR AND EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/tq.cuh")- foreach(_pair tqk6/tqk7 tq2_t/tq2_t)+ set(_strata_tq_pairs tqk6/tqk7 tq2_t/tq2_t)+ if(EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqp5.cu") # Gyro-L's pair trellis+ list(APPEND _strata_tq_pairs tqp5/tqp5 tqp6/tqp6 tqp7/tqp7)+ endif()+ foreach(_pair ${_strata_tq_pairs})string(REPLACE "/" "_" _name "${_pair}")add_test(NAME native_expert_parity_${_name} COMMAND native_expert_parity --synthetic ${_pair})add_test(NAME native_expert_parity_hadamard_${_name} COMMAND native_expert_parity --synthetic-hadamard ${_pair})@@ -2331,6 +2347,13 @@ if(STRATA_BUILD_TESTS)add_test(NAME tq_cpu_parity COMMAND tq_cpu_parity)add_test(NAME tq_cpu_parity_avx2 COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_AVX2=1 $<TARGET_FILE:tq_cpu_parity>)set_tests_properties(tq_cpu_parity tq_cpu_parity_avx2 PROPERTIES TIMEOUT 300 SKIP_RETURN_CODE 77)+ # Gyro-L's pair trellis (types 147-149) and the older trellis rows on synthetic blocks, no ggml and no model: the scalar+ # kernel against a bit-by-bit reference, every ISA bit for bit equal to scalar, the type predicates unchanged+ add_executable(tqp_cpu_test src/kernels/cpu/tqp_cpu_test.cpp)+ target_link_libraries(tqp_cpu_test PRIVATE strata_kernels_cpu)+ add_test(NAME tqp_cpu_test COMMAND tqp_cpu_test)+ add_test(NAME tqp_cpu_test_avx2 COMMAND ${CMAKE_COMMAND} -E env STRATA_FORCE_AVX2=1 $<TARGET_FILE:tqp_cpu_test>)+ set_tests_properties(tqp_cpu_test tqp_cpu_test_avx2 PROPERTIES TIMEOUT 120)endif()add_executable(expert_parity src/kernels/cpu/expert_parity.cpp)@@ -2466,7 +2489,7 @@ if(STRATA_HIP_GFX906)# f16kq::covers() answers false and the int8 kernels run, as on every other card)if(_strata_mmq_tq_overlay)set(_strata_tq_hip ${_strata_tq_overlay_dir}/quantize.cu)- foreach(_t tq2_t tqk6 tqk7)+ foreach(_t tq2_t tqk6 tqk7 tqp5 tqp6 tqp7)list(APPEND _strata_tq_hip ${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu)endforeach()list(APPEND _strata_cu ${_strata_tq_hip})diff --git a/cmake/ggml_tq_overlay_test.cmake b/cmake/ggml_tq_overlay_test.cmakeindex e56604c81..02f8eb16a 100644--- a/cmake/ggml_tq_overlay_test.cmake+++ b/cmake/ggml_tq_overlay_test.cmake@@ -1,6 +1,7 @@# cmake -DLLAMA=<llama.cpp dir> -DOUT=<scratch dir> -P cmake/ggml_tq_overlay_test.cmake# The trellis MMQ overlay on a pinned llama.cpp: it generates, every copied header is the pinned one except the patched# ones, and the patched ones only gain lines (TQ routing, three externs, config copies of IQ2_XS's).+cmake_policy(SET CMP0057 NEW) # script mode starts with the old policies: IN_LIST below needs this oneget_filename_component(CMAKE_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)include(${CMAKE_CURRENT_LIST_DIR}/strata_ggml_tq_overlay.cmake)file(REMOVE_RECURSE "${OUT}")@@ -24,7 +25,8 @@ foreach(_f IN LISTS _cuh)endforeach()file(READ "${OUT}/mmq.cuh" _m)foreach(_pat "case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a"- "extern DECL_MMQ_CASE(GGML_TYPE_TQK7);")+ "extern DECL_MMQ_CASE(GGML_TYPE_TQK7);" "extern DECL_MMQ_CASE(GGML_TYPE_TQP5);" "extern DECL_MMQ_CASE(GGML_TYPE_TQP7);"+ "case GGML_TYPE_TQP6: return MMQ_DP4A_TXS_Q8_0_16;")string(FIND "${_m}" "${_pat}" _p)if(_p LESS 0)message(FATAL_ERROR "mmq.cuh lacks: ${_pat}")@@ -35,6 +37,15 @@ string(FIND "${_g}" "GGML_TYPE_TQK7 = 146," _p)if(_p LESS 0)message(FATAL_ERROR "ggml.h lacks the trellis enumerators")endif()+string(FIND "${_g}" "GGML_TYPE_TQP7 = 149," _p) # Gyro-L's pair trellis+if(_p LESS 0)+ message(FATAL_ERROR "ggml.h lacks the pair-trellis enumerators")+endif()+foreach(_t tqp5 tqp6 tqp7)+ if(NOT EXISTS "${OUT}/template-instances/mmq-instance-${_t}.cu")+ message(FATAL_ERROR "the overlay has no MMQ instance for ${_t}")+ endif()+endforeach()foreach(_c IN ITEMS ampere rdna4)file(READ "${CU}/mmq-config-${_c}.cuh" _o)file(READ "${OUT}/mmq-config-${_c}.cuh" _n2)diff --git a/cmake/strata_ggml_tq_overlay.cmake b/cmake/strata_ggml_tq_overlay.cmakeindex df24af259..a29ae4be0 100644--- a/cmake/strata_ggml_tq_overlay.cmake+++ b/cmake/strata_ggml_tq_overlay.cmake@@ -79,6 +79,9 @@ function(strata_ggml_tq_overlay LLAMA OUT)" GGML_TYPE_TQ2_T = 144, // Agention trellis types (Strata overlay; the fork's ids)GGML_TYPE_TQK6 = 145,GGML_TYPE_TQK7 = 146,+ GGML_TYPE_TQP5 = 147, // Gyro-L pair trellis (the same fork's 147-149)+ GGML_TYPE_TQP6 = 148,+ GGML_TYPE_TQP7 = 149,")file(WRITE "${OUT}/include/ggml.h" "${_h}")@@ -99,15 +102,15 @@ function(strata_ggml_tq_overlay LLAMA OUT)set(_tq_fail "the MMQ helper signatures of this ggml-cuda are not the ones the overlay knows (ggml_cuda_mmq_get_nthreads)")endif()_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n"- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n")+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n case GGML_TYPE_TQP5:\n case GGML_TYPE_TQP6:\n case GGML_TYPE_TQP7:\n")_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;\n"- " case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK7: return MMQ_DP4A_TXS_Q8_0_16;\n")+ " case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK7: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQP5: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQP6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQP7: return MMQ_DP4A_TXS_Q8_0_16;\n")_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ3_XXS_Q8_1_MMQ,"- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback${_ps}>);\n")+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n case GGML_TYPE_TQP5:\n case GGML_TYPE_TQP6:\n case GGML_TYPE_TQP7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback${_ps}>);\n")_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,"- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback${_ps}>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback${_ps}>);\n")+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n case GGML_TYPE_TQP5:\n case GGML_TYPE_TQP6:\n case GGML_TYPE_TQP7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback${_ps}>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback${_ps}>);\n")_tq_edit(_m AFTER "extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n"- "extern DECL_MMQ_CASE(GGML_TYPE_TQ2_T);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK7);\n")+ "extern DECL_MMQ_CASE(GGML_TYPE_TQ2_T);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK7);\nextern DECL_MMQ_CASE(GGML_TYPE_TQP5);\nextern DECL_MMQ_CASE(GGML_TYPE_TQP6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQP7);\n")file(WRITE "${OUT}/mmq.cuh" "${_m}")# ---- mmq-load-tiles.cuh: the loader, at the end -------------------------------------------------------------------@@ -131,7 +134,7 @@ function(strata_ggml_tq_overlay LLAMA OUT)list(REMOVE_DUPLICATES _lines)foreach(_ln IN LISTS _lines)set(_add "")- foreach(_ty TQ2_T TQK6 TQK7)+ foreach(_ty TQ2_T TQK6 TQK7 TQP5 TQP6 TQP7)string(REPLACE "GGML_TYPE_IQ2_XS," "GGML_TYPE_${_ty}," _x "${_ln}")string(APPEND _add "${_x}")endforeach()@@ -142,7 +145,7 @@ function(strata_ggml_tq_overlay LLAMA OUT)endforeach()# ---- template instances -------------------------------------------------------------------------------------------- foreach(_ty tq2_t tqk6 tqk7)+ foreach(_ty tq2_t tqk6 tqk7 tqp5 tqp6 tqp7)string(TOUPPER "${_ty}" _TY)file(WRITE "${OUT}/template-instances/mmq-instance-${_ty}.cu""// Strata trellis overlay (cmake/strata_ggml_tq_overlay.cmake)\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_${_TY});\n")diff --git a/docs/GYRO.md b/docs/GYRO.mdindex 6c390e1b5..6ab2bd223 100644--- a/docs/GYRO.md+++ b/docs/GYRO.md@@ -39,8 +39,10 @@ ready-made engine is too old for the pack, setup replaces it or the app offers UAbout half of each file is the n-gram (PLE) table. It stays on the SSD and is read row by row for every token, so keepthe file on a fast local drive. The memory figure is what the weights take once loaded.-Gyro-L (agentionai's 50 GiB file) is not supported. It uses another trellis family (types 147 to 149) that the engine-does not have.+Gyro-L (agentionai's 50 GiB file) uses another trellis family, the pair trellis (types 147 to 149, TQP5 to TQP7). The+engine reads it with the same pack step and expert kernels as the other Gyro files, on the CPU and on CUDA cards. The+prompt path uses the trellis MMQ build where the build has it; Metal and SYCL have no kernels for it, so they use the CPU+rows and the FP16 prompt path. We have not run the 50 GiB file ourselves yet.**Pictures.** Gyro reads pictures with the original model's picture reader file, the same one everyQwen3.8-Flash-Next model uses. Setup downloads it with the model. The reader opens only the vocabulary and the header ofdiff --git a/include/strata/artifact/gguf_reader.hpp b/include/strata/artifact/gguf_reader.hppindex 5b67f54ba..5800fe94c 100644--- a/include/strata/artifact/gguf_reader.hpp+++ b/include/strata/artifact/gguf_reader.hpp@@ -124,6 +124,12 @@ inline const char* ggml_type_name(uint32_t t) {return "TQK6";case 146:return "TQK7";+ case 147:+ return "TQP5";+ case 148:+ return "TQP6";+ case 149:+ return "TQP7";default:return "?";}@@ -252,6 +258,19 @@ inline bool block_geometry(uint32_t t, int& elems, int& bytes) {elems = 128;bytes = 30;return true;+ // Gyro-L pair trellis: 128 values = fp16 scale + 64 steps of K bits, 2 weights per step (8*K bytes)+ case 147:+ elems = 128;+ bytes = 42;+ return true;+ case 148:+ elems = 128;+ bytes = 50;+ return true;+ case 149:+ elems = 128;+ bytes = 58;+ return true;default:return false;}diff --git a/include/strata/kernels/cpu/tq_cpu.hpp b/include/strata/kernels/cpu/tq_cpu.hppindex 7555fccfb..3eaaf5336 100644--- a/include/strata/kernels/cpu/tq_cpu.hpp+++ b/include/strata/kernels/cpu/tq_cpu.hpp@@ -44,7 +44,12 @@ bool tq_supported(int ggml_type) noexcept;int tq_kbits(int ggml_type) noexcept;/// Values per block of the trellis types, and bytes per block (2 + 4 * bits: 34, 26, 30, 22, 18).constexpr int kTqBlock = 128;-inline size_t tq_block_bytes(int ggml_type) noexcept { return (size_t) (2 + 4 * tq_kbits(ggml_type)); }+// TQ2_T / TQK6 / TQK7: 4 weights per step (2 + 4K bytes per 128-value block). TQP5/6/7 (147-149, Gyro-L) are the+// pair layout: 2 weights per step, 8K bytes per block (2 + 8K).+inline bool tq_pair_layout(int ggml_type) noexcept { return ggml_type >= 147 && ggml_type <= 149; }+inline size_t tq_block_bytes(int ggml_type) noexcept {+ return (size_t) (2 + (tq_pair_layout(ggml_type) ? 8 : 4) * tq_kbits(ggml_type));+}/// The best ISA this CPU runs (capped by STRATA_TQ_ISA).TqIsa tq_isa();/// Whether this CPU can run `isa` at all (scalar: always).@@ -88,6 +93,13 @@ void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytfloat* const* out, int r0, int r1);void tq_rows_avx2(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1);+/// The TQP pair layout (types 147-149) on AVX2. AVX-512 also takes this kernel: the pair layout has no+/// AVX-512 form yet (no AVX-512 machine has verified one), and the AVX2 path is exact.+void tq_rows_avx2_tqp(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1);+/// The scalar reference of the pair layout (any CPU).+void tq_rows_scalar_tqp(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1);void tq_rows_avx512(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1);} // namespace tq_detaildiff --git a/src/kernels/cpu/tq_avx2.cpp b/src/kernels/cpu/tq_avx2.cppindex 9a43393fe..dbc6b5aa2 100644--- a/src/kernels/cpu/tq_avx2.cpp+++ b/src/kernels/cpu/tq_avx2.cpp@@ -75,8 +75,79 @@ void rows_t(const int16_t* table, const uint8_t* w, size_t row_bytes, int n, con}}+// TQP (147-149): 64 pair-steps per 128-value block. A step's two weights are the FIRST pair of its state+// table row, so 8 states make one YMM of 16 int16 weights, and a 32-value group (16 steps) is two YMM of+// weights against two YMM of activations: VPMADDWD, one exact int32 sum per group, then the four group sums+// go into the same per-token __m128 the other kernels use, so the float part is the scalar kernel's.+inline __m256i pairs8(const int16_t* tab, uint32_t s0, uint32_t s1, uint32_t s2, uint32_t s3, uint32_t s4,+ uint32_t s5, uint32_t s6, uint32_t s7) {+ const __m128i a0 = _mm_unpacklo_epi32(_mm_loadl_epi64((const __m128i*) (tab + 4 * s0)),+ _mm_loadl_epi64((const __m128i*) (tab + 4 * s1)));+ const __m128i a1 = _mm_unpacklo_epi32(_mm_loadl_epi64((const __m128i*) (tab + 4 * s2)),+ _mm_loadl_epi64((const __m128i*) (tab + 4 * s3)));+ const __m128i a2 = _mm_unpacklo_epi32(_mm_loadl_epi64((const __m128i*) (tab + 4 * s4)),+ _mm_loadl_epi64((const __m128i*) (tab + 4 * s5)));+ const __m128i a3 = _mm_unpacklo_epi32(_mm_loadl_epi64((const __m128i*) (tab + 4 * s6)),+ _mm_loadl_epi64((const __m128i*) (tab + 4 * s7)));+ return _mm256_set_m128i(_mm_unpacklo_epi64(a2, a3), _mm_unpacklo_epi64(a0, a1));+}++template <int KB>+void rows_tqp_t(const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt, float* const* out, int r0,+ int r1) {+ const int nb = n / 128;+ constexpr size_t bsz = 2 + 8 * KB;+ const int16_t* tab = tq_state_table();+ for (int r = r0; r < r1; ++r) {+ const uint8_t* x = w + (size_t) r * row_bytes;+ __m128 acc[kTqMaxTokens];+ for (int t = 0; t < nt; ++t) acc[t] = _mm_setzero_ps();+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = x + (size_t) i * bsz;+ uint32_t st[64];+ tq_states_tqp<KB>(blk + 2, st);+ __m256i wv[8]; // wv[2g], wv[2g+1]: the weights of the 32-value group g+#pragma GCC unroll 4+ for (int g = 0; g < 4; ++g) {+ const uint32_t* s = st + 16 * g;+ wv[2 * g] = pairs8(tab, s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]);+ wv[2 * g + 1] = pairs8(tab, s[8], s[9], s[10], s[11], s[12], s[13], s[14], s[15]);+ }+ const __m128 dx = _mm_set1_ps(tq_half_to_float(tq_ld16(blk)));+ for (int t = 0; t < nt; ++t) {+ const __m256i* q = (const __m256i*) (a[t]->q + 128 * i);+ int32_t sum[4];+#pragma GCC unroll 4+ for (int g = 0; g < 4; ++g) {+ const __m256i p = _mm256_add_epi32(_mm256_madd_epi16(wv[2 * g], _mm256_load_si256(q + 2 * g)),+ _mm256_madd_epi16(wv[2 * g + 1], _mm256_load_si256(q + 2 * g + 1)));+ __m128i s = _mm_add_epi32(_mm256_castsi256_si128(p), _mm256_extracti128_si256(p, 1));+ s = _mm_hadd_epi32(s, s);+ s = _mm_hadd_epi32(s, s);+ sum[g] = _mm_cvtsi128_si32(s);+ }+ const __m128 sc = _mm_mul_ps(dx, _mm_loadu_ps(a[t]->d + 4 * i));+ const __m128i s4 = _mm_set_epi32(sum[3], sum[2], sum[1], sum[0]);+ acc[t] = _mm_fmadd_ps(_mm_cvtepi32_ps(s4), sc, acc[t]);+ }+ }+ for (int t = 0; t < nt; ++t) {+ alignas(16) float v[4];+ _mm_store_ps(v, acc[t]);+ out[t][r] = ((v[0] + v[1]) + (v[2] + v[3])) * (1.0f / 8192.0f);+ }+ }+}+} // namespace+void tq_rows_avx2_tqp(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ if (kb == 5) rows_tqp_t<5>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_tqp_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else rows_tqp_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+}+void tq_rows_avx2(int kb, const int16_t* table, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,float* const* out, int r0, int r1) {if (kb == 8) rows_t<8>(table, w, row_bytes, n, a, nt, out, r0, r1);diff --git a/src/kernels/cpu/tq_cpu.cpp b/src/kernels/cpu/tq_cpu.cppindex 22a874bcb..9c898cdfb 100644--- a/src/kernels/cpu/tq_cpu.cpp+++ b/src/kernels/cpu/tq_cpu.cpp@@ -86,6 +86,41 @@ void rows_scalar_t(const int16_t* tab, const uint8_t* w, size_t row_bytes, int n}}+// TQP (Gyro-L): 64 pair-trellis steps of 2 weights; a 32-value group is 16 steps. The table row holds the+// TQ2_T state's two pairs; TQP uses the first pair only (weight 2t+j = lut[2*hyb(2*s)+j]).+template <int KB>+void rows_scalar_tqp_t(const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ const int nb = n / 128;+ constexpr size_t bsz = 2 + 8 * KB;+ const int16_t* tab = tq_state_table();+ for (int r = r0; r < r1; ++r) {+ const uint8_t* x = w + (size_t) r * row_bytes;+ float acc[kTqMaxTokens][4] = {};+ for (int i = 0; i < nb; ++i) {+ const uint8_t* blk = x + (size_t) i * bsz;+ const float dx = tq_detail::tq_half_to_float(tq_detail::tq_ld16(blk));+ uint32_t st[64];+ tq_detail::tq_states_tqp<KB>(blk + 2, st);+ for (int t = 0; t < nt; ++t) {+ const int16_t* q = a[t]->q + 128 * i;+ for (int k = 0; k < 4; ++k) {+ int32_t s = 0;+ for (int j = 0; j < 16; ++j) { // step 16k+j: two weights+ const int16_t* wv = tab + 4 * st[16 * k + j];+ const int16_t* qv = q + 32 * k + 2 * j;+ s += wv[0] * qv[0] + wv[1] * qv[1];+ }+ const float sc = dx * a[t]->d[4 * i + k];+ acc[t][k] = std::fma((float) s, sc, acc[t][k]);+ }+ }+ }+ for (int t = 0; t < nt; ++t)+ out[t][r] = ((acc[t][0] + acc[t][1]) + (acc[t][2] + acc[t][3])) * (1.0f / 8192.0f);+ }+}+} // namespaceconst int16_t* tq_state_table() {@@ -103,11 +138,18 @@ const int16_t* tq_state_table_computed() {const int16_t* tq_state_table_for(int t) { return (t == kTqTypeK5X || t == kTqTypeK4X) ? tq_state_table_computed() : tq_state_table(); }bool tq_supported(int t) noexcept {- return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;+ return t == 144 || t == 145 || t == 146 || t == 147 || t == 148 || t == 149 ||+ t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;}int tq_kbits(int t) noexcept {- return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : (t == kTqTypeK5X || t == kTqTypeK5T) ? 5 : (t == kTqTypeK4X || t == kTqTypeK4T) ? 4 : 0;+ if (t == 148) return 6; // TQP6+ if (t == 144) return 8;+ if (t == 146 || t == 149) return 7;+ if (t == 145) return 6;+ if (t == 147 || t == kTqTypeK5X || t == kTqTypeK5T) return 5;+ if (t == kTqTypeK4X || t == kTqTypeK4T) return 4;+ return 0;}bool tq_isa_ok(TqIsa isa) {@@ -166,6 +208,13 @@ void tq_rows_scalar(int kb, const int16_t* tab, const uint8_t* w, size_t row_bytelse rows_scalar_t<7>(tab, w, row_bytes, n, a, nt, out, r0, r1);}+void tq_rows_scalar_tqp(int kb, const uint8_t* w, size_t row_bytes, int n, const TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+ if (kb == 5) rows_scalar_tqp_t<5>(w, row_bytes, n, a, nt, out, r0, r1);+ else if (kb == 6) rows_scalar_tqp_t<6>(w, row_bytes, n, a, nt, out, r0, r1);+ else rows_scalar_tqp_t<7>(w, row_bytes, n, a, nt, out, r0, r1);+}+} // namespace tq_detailnamespace {@@ -197,6 +246,21 @@ void rows_on(TqIsa isa, int kb, const int16_t* tab, const uint8_t* w, size_t row}}+// The pair layout (types 147-149): AVX2 and AVX-512 both take the AVX2 pair kernel; scalar, and any build without the+// x86 units (arm64: the Metal build), the scalar one.+void rows_pair_on(TqIsa isa, int kb, const uint8_t* w, size_t row_bytes, int n, const tq_detail::TqAct* const* a, int nt,+ float* const* out, int r0, int r1) {+#if defined(STRATA_TQ_X86)+ if (isa != TqIsa::scalar) {+ tq_detail::tq_rows_avx2_tqp(kb, w, row_bytes, n, a, nt, out, r0, r1);+ return;+ }+#else+ (void) isa;+#endif+ tq_detail::tq_rows_scalar_tqp(kb, w, row_bytes, n, a, nt, out, r0, r1);+}+} // namespacevoid tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int n, const void* const* act, int nt,@@ -204,6 +268,11 @@ void tq_rows(TqIsa isa, int ggml_type, const uint8_t* w, size_t row_bytes, int nif (!tq_isa_ok(isa)) std::abort(); // a caller bug: asked for an ISA this CPU does not haveconst int16_t* tab = tq_state_table_for(ggml_type);const ActSet& s = unpack_all(act, n, nt);+ if (ggml_type >= 147 && ggml_type <= 149) { // Gyro-L pair trellis: the AVX2 pair kernel (scalar is the reference)+ const int kb = tq_kbits(ggml_type);+ rows_pair_on(isa, kb, w, row_bytes, n, s.ptr, nt, out, r0, r1);+ return;+ }rows_on(isa, tq_kbits(ggml_type), tab, w, row_bytes, n, s.ptr, nt, out, r0, r1);}@@ -222,8 +291,13 @@ void tq_gu_rows(TqIsa isa, int ggml_type, const uint8_t* blob, size_t gu_row, siup[t] = ubuf.data() + (size_t) t * nr;}const int kb = tq_kbits(ggml_type);- rows_on(isa, kb, tab, blob, gu_row, n, s.ptr, nt, gp, r0, r1);- rows_on(isa, kb, tab, blob + up_off, gu_row, n, s.ptr, nt, up, r0, r1);+ const bool pair = ggml_type >= 147 && ggml_type <= 149; // Gyro-L pair trellis: its own kernels+ auto rows = [&](const uint8_t* w, float* const* o) {+ if (pair) rows_pair_on(isa, kb, w, gu_row, n, s.ptr, nt, o, r0, r1);+ else rows_on(isa, kb, tab, w, gu_row, n, s.ptr, nt, o, r0, r1);+ };+ rows(blob, gp);+ rows(blob + up_off, up);for (int t = 0; t < nt; ++t)for (int r = r0; r < r1; ++r) {const float g = gp[t][r], u = up[t][r];diff --git a/src/kernels/cpu/tq_states.hpp b/src/kernels/cpu/tq_states.hppindex bb4eaa7a1..8bdbfb667 100644--- a/src/kernels/cpu/tq_states.hpp+++ b/src/kernels/cpu/tq_states.hpp@@ -60,6 +60,26 @@ inline void tq_states(const uint8_t* qs, uint32_t* st) {}}+/// TQP (Gyro-L): 64 pair-trellis steps; state t = the 16 stream bits from bit (63 - t)*KB, LSB first,+/// circular over 8*KB bytes (tqp_state in the fork's ggml-common.h).+template <int KB>+inline uint32_t tq_state_tqp(const uint8_t* qs, int t) {+ constexpr int NB = 8 * KB;+ const uint32_t off = (uint32_t) (63 - t) * (uint32_t) KB;+ const uint32_t i0 = off >> 3;+ const uint32_t i1 = (i0 + 1) % (uint32_t) NB;+ const uint32_t i2 = (i0 + 2) % (uint32_t) NB;+ const uint32_t w = (uint32_t) qs[i0] | ((uint32_t) qs[i1] << 8) | ((uint32_t) qs[i2] << 16);+ return (w >> (off & 7)) & kTqStateMask;+}+template <int KB>+inline void tq_states_tqp(const uint8_t* qs, uint32_t* st) {+#if defined(__GNUC__)+#pragma GCC unroll 64+#endif+ for (int t = 0; t < 64; ++t) st[t] = tq_state_tqp<KB>(qs, t);+}+/// fp16 bits -> float, exact (the block scales and the Q8_0 activation scales).inline float tq_half_to_float(uint16_t h) {const uint32_t sign = (uint32_t) (h & 0x8000u) << 16;diff --git a/src/kernels/cpu/tqp_cpu_test.cpp b/src/kernels/cpu/tqp_cpu_test.cppnew file mode 100644index 000000000..a8da31b2e--- /dev/null+++ b/src/kernels/cpu/tqp_cpu_test.cpp@@ -0,0 +1,229 @@+// src/kernels/cpu/tqp_cpu_test.cpp - the trellis CPU rows (tq_cpu.hpp) on synthetic blocks, with no ggml and no model:+//+// * the pair-trellis types TQP5 / TQP6 / TQP7 (ggml 147-149, Gyro-L): the scalar kernel against a reference written+// here from the format definition (state t = the 16 stream bits that start at bit (63 - t) * K, LSB first,+// circular over 64 * K bits; the step's two weights are the first pair of the state's codebook entry), summed in+// double; then every ISA this CPU has (AVX2, AVX-512 takes the AVX2 pair kernel) bit for bit equal to the scalar+// kernel, for groups of 1..8 tokens, one row range or two, and for the fused gate/up rows;+// * the older trellis types (144-146): AVX2 / AVX-512 bit for bit equal to the scalar kernel (the pair types were+// added next to them: this shows they still agree), and their block sizes and K unchanged;+// * the type predicates (tq_supported, tq_kbits, tq_block_bytes) for every id 0..299: the answers the code gave+// before the pair types, plus 147-149.+//+// Random stream bytes are valid codes (any 15-bit state indexes the codebook). CPU only. Exit 0 = pass.+#include "strata/kernels/cpu/tq_cpu.hpp"++#include "tq_states.hpp"++#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstring>+#include <random>+#include <vector>++using namespace strata::kernels::cpu;++namespace {++int g_fail = 0;++void fail(const char* what, int a = 0, int b = 0) {+ std::printf(" FAIL: %s %d %d\n", what, a, b);+ ++g_fail;+}++// ---- the type predicates as they were before the pair types (the old formulas, copied)+bool old_supported(int t) {+ return t == 144 || t == 145 || t == 146 || t == kTqTypeK5X || t == kTqTypeK4X || t == kTqTypeK5T || t == kTqTypeK4T;+}+int old_kbits(int t) {+ return t == 144 ? 8 : t == 145 ? 6 : t == 146 ? 7 : (t == kTqTypeK5X || t == kTqTypeK5T) ? 5 : (t == kTqTypeK4X || t == kTqTypeK4T) ? 4 : 0;+}++void check_predicates() {+ for (int t = 0; t < 300; ++t) {+ const bool pair = t >= 147 && t <= 149;+ if (tq_supported(t) != (old_supported(t) || pair)) fail("tq_supported", t);+ const int want_k = pair ? (t == 147 ? 5 : t == 148 ? 6 : 7) : old_kbits(t);+ if (tq_kbits(t) != want_k) fail("tq_kbits", t, tq_kbits(t));+ const size_t want_b = pair ? (size_t) (2 + 8 * want_k) : (size_t) (2 + 4 * want_k);+ if (tq_block_bytes(t) != want_b) fail("tq_block_bytes", t, (int) tq_block_bytes(t));+ }+ if (tq_block_bytes(144) != 34 || tq_block_bytes(145) != 26 || tq_block_bytes(146) != 30) fail("sizes of 144-146");+ if (tq_block_bytes(147) != 42 || tq_block_bytes(148) != 50 || tq_block_bytes(149) != 58) fail("sizes of 147-149");+ std::printf("predicates (ids 0..299): %s\n", g_fail ? "FAIL" : "ok");+}++// ---- synthetic data+uint16_t random_half(std::mt19937& rng) {+ std::uniform_int_distribution<int> ex(2, 8), man(0, 1023), sgn(0, 3);+ return (uint16_t) ((sgn(rng) == 0 ? 0x8000 : 0) | (ex(rng) << 10) | man(rng)); // 2^-13 .. 2^-6+}++std::vector<uint8_t> random_rows(int type, int rows, int n, std::mt19937& rng, size_t& row_bytes) {+ const size_t bs = tq_block_bytes(type);+ row_bytes = bs * (size_t) (n / 128);+ std::vector<uint8_t> w(row_bytes * (size_t) rows);+ std::uniform_int_distribution<int> byte(0, 255);+ for (auto& b : w) b = (uint8_t) byte(rng);+ for (size_t o = 0; o < w.size(); o += bs) {+ const uint16_t h = random_half(rng);+ std::memcpy(&w[o], &h, 2);+ }+ return w;+}++// n values as Q8_0 blocks (fp16 d, 32 x int8): 34 bytes per 32 values+std::vector<uint8_t> random_q8_0(int n, std::mt19937& rng, std::vector<float>& d, std::vector<int>& q) {+ std::vector<uint8_t> a((size_t) (n / 32) * 34);+ std::uniform_int_distribution<int> val(-127, 127), ex(8, 13), man(0, 1023);+ d.assign((size_t) n / 32, 0.f);+ q.assign((size_t) n, 0);+ for (int b = 0; b < n / 32; ++b) {+ const uint16_t h = (uint16_t) ((ex(rng) << 10) | man(rng));+ std::memcpy(&a[(size_t) b * 34], &h, 2);+ d[(size_t) b] = strata::kernels::cpu::tq_detail::tq_half_to_float(h);+ for (int j = 0; j < 32; ++j) {+ const int v = val(rng);+ q[(size_t) b * 32 + j] = v;+ a[(size_t) b * 34 + 2 + j] = (uint8_t) (int8_t) v;+ }+ }+ return a;+}++// ---- the reference for the pair types, from the format definition (bit by bit, not the window code of the kernels)+uint32_t ref_state(const uint8_t* qs, int K, int t) {+ const int nbits = 64 * K;+ uint32_t s = 0;+ for (int b = 0; b < 16; ++b) {+ const int pos = ((63 - t) * K + b) % nbits;+ s |= (uint32_t) ((qs[pos >> 3] >> (pos & 7)) & 1) << b;+ }+ return s & 0x7FFFu;+}++// w . a in double; `bound` = the sum of the absolute terms (the scale of the float rounding allowed)+double ref_dot(int type, const uint8_t* row, int n, const std::vector<float>& d, const std::vector<int>& q, double& bound) {+ const int K = tq_kbits(type);+ const int16_t* tab = tq_state_table();+ const size_t bs = tq_block_bytes(type);+ double sum = 0, abs_sum = 0;+ for (int i = 0; i < n / 128; ++i) {+ const uint8_t* blk = row + (size_t) i * bs;+ uint16_t hd;+ std::memcpy(&hd, blk, 2);+ const double dw = strata::kernels::cpu::tq_detail::tq_half_to_float(hd);+ uint32_t st[64];+ for (int t = 0; t < 64; ++t) st[t] = ref_state(blk + 2, K, t);+ for (int k = 0; k < 4; ++k) {+ long long s = 0, sa = 0;+ for (int j = 0; j < 32; ++j) {+ const int v = 32 * k + j; // weight v of the block: step v / 2, element v & 1 of the first pair+ const int w16 = tab[4 * st[v >> 1] + (v & 1)];+ s += (long long) w16 * q[(size_t) 128 * i + v];+ sa += (long long) std::abs(w16) * std::abs(q[(size_t) 128 * i + v]);+ }+ const double sc = dw * d[(size_t) 4 * i + k];+ sum += sc * (double) s;+ abs_sum += std::fabs(sc) * (double) sa;+ }+ }+ bound = abs_sum / 8192.0;+ return sum / 8192.0;+}++bool same_bits(const float* a, const float* b, size_t n) { return std::memcmp(a, b, n * sizeof(float)) == 0; }++constexpr int NT = kTqMaxTokens;++struct Case {+ int type, rows, n;+ std::vector<uint8_t> w;+ size_t rb = 0;+ std::vector<std::vector<uint8_t>> act;+ std::vector<std::vector<float>> d;+ std::vector<std::vector<int>> q;+ const void* ap[NT];+ Case(int type_, int rows_, int n_, uint32_t seed) : type(type_), rows(rows_), n(n_) {+ std::mt19937 rng(seed);+ w = random_rows(type, rows, n, rng, rb);+ act.resize(NT); d.resize(NT); q.resize(NT);+ for (int t = 0; t < NT; ++t) {+ act[t] = random_q8_0(n, rng, d[t], q[t]);+ ap[t] = act[t].data();+ }+ }+ void run(TqIsa isa, int nt, const void* const* a, std::vector<float>& o) const {+ o.assign((size_t) nt * rows, 0.f);+ float* op[NT];+ for (int t = 0; t < nt; ++t) op[t] = o.data() + (size_t) t * rows;+ tq_rows(isa, type, w.data(), rb, n, a, nt, op, 0, rows / 3); // two row ranges, as the pool splits them+ tq_rows(isa, type, w.data(), rb, n, a, nt, op, rows / 3, rows);+ }+};++void check_type(int type, bool reference) {+ Case c(type, 37, 1280, 1000u + (uint32_t) type);+ std::vector<float> ref, got;+ c.run(TqIsa::scalar, NT, c.ap, ref);+ int bad = 0;+ if (reference) { // the pair types: the scalar kernel against the definition+ double worst = 0;+ for (int t = 0; t < NT; ++t)+ for (int r = 0; r < c.rows; ++r) {+ double bound = 0;+ const double want = ref_dot(type, c.w.data() + (size_t) r * c.rb, c.n, c.d[t], c.q[t], bound);+ const double err = std::fabs((double) ref[(size_t) t * c.rows + r] - want);+ worst = std::fmax(worst, bound > 0 ? err / bound : err);+ if (err > 2e-5 * bound + 1e-30) ++bad;+ }+ std::printf("type %d scalar vs the definition (double): %d of %d outside 2e-5 of the term sum, worst %.2e\n", type, bad,+ NT * c.rows, worst);+ if (bad) fail("scalar kernel differs from the definition", type);+ }+ for (TqIsa isa : {TqIsa::avx2, TqIsa::avx512}) {+ if (!tq_isa_ok(isa)) { std::printf("type %d %-6s not on this CPU: skipped\n", type, tq_isa_name(isa)); continue; }+ bool ok = true;+ for (int nt = 1; nt <= NT; ++nt)+ for (int t0 = 0; t0 + nt <= NT; t0 += nt) {+ c.run(isa, nt, c.ap + t0, got);+ ok = ok && same_bits(got.data(), ref.data() + (size_t) t0 * c.rows, (size_t) nt * c.rows);+ }+ std::printf("type %d %-6s groups of 1..%d tokens vs the scalar kernel: %s\n", type, tq_isa_name(isa), NT,+ ok ? "bitwise equal" : "DIFFERENT");+ if (!ok) fail("ISA differs from scalar", type);+ }+ // the fused gate/up rows: gate rows at the blob, up rows after them+ {+ std::vector<uint8_t> blob = c.w;+ std::mt19937 rng(77u + (uint32_t) type);+ size_t rb2 = 0;+ const auto up = random_rows(type, c.rows, c.n, rng, rb2);+ blob.insert(blob.end(), up.begin(), up.end());+ const size_t up_off = c.w.size();+ std::vector<float> ff0((size_t) NT * c.rows), ff1((size_t) NT * c.rows);+ float* p0[NT]; float* p1[NT];+ for (int t = 0; t < NT; ++t) { p0[t] = ff0.data() + (size_t) t * c.rows; p1[t] = ff1.data() + (size_t) t * c.rows; }+ tq_gu_rows(TqIsa::scalar, type, blob.data(), c.rb, up_off, c.n, c.ap, NT, p0, 0, c.rows);+ bool ok = true;+ for (TqIsa isa : {TqIsa::avx2, TqIsa::avx512}) {+ if (!tq_isa_ok(isa)) continue;+ tq_gu_rows(isa, type, blob.data(), c.rb, up_off, c.n, c.ap, NT, p1, 0, c.rows);+ ok = ok && same_bits(ff0.data(), ff1.data(), ff0.size());+ }+ std::printf("type %d gate/up rows, every ISA vs scalar: %s\n", type, ok ? "bitwise equal" : "DIFFERENT");+ if (!ok) fail("gate/up rows differ", type);+ }+}++} // namespace++int main() {+ check_predicates();+ for (int type : {147, 148, 149}) check_type(type, true);+ for (int type : {144, 145, 146}) check_type(type, false);+ std::printf("tqp_cpu_test: %d failures\n", g_fail);+ return g_fail ? 1 : 0;+}diff --git a/src/kernels/cuda/iq_kernels.cu b/src/kernels/cuda/iq_kernels.cuindex d2f2e34e8..f81efd692 100644--- a/src/kernels/cuda/iq_kernels.cu+++ b/src/kernels/cuda/iq_kernels.cu@@ -616,6 +616,10 @@ template<int TY> struct TqInfo;template<> struct TqInfo<144> { using block_t = block_tq2_t; }; // byte-aligned, no Ktemplate<> struct TqInfo<145> { using block_t = block_tqk6; static constexpr int K = 6; };template<> struct TqInfo<146> { using block_t = block_tqk7; static constexpr int K = 7; };+// Gyro-L: pair trellis (64 steps x K bits, 2 weights per step), the TQ2_T codebook's first pair+template<> struct TqInfo<147> { using block_t = block_tqp5; static constexpr int K = 5; };+template<> struct TqInfo<148> { using block_t = block_tqp6; static constexpr int K = 6; };+template<> struct TqInfo<149> { using block_t = block_tqp7; static constexpr int K = 7; };__device__ __forceinline__ float tq_h2f(uint32_t bits) { return __half2float(__ushort_as_half((unsigned short) bits)); }// the four unscaled weights of the step with state s: (a.x, a.y, b.x, b.y)__device__ __forceinline__ void tq_step(uint32_t s, float2& a, float2& b) {@@ -716,6 +720,71 @@ __device__ __forceinline__ float vec_dot_tq_q8_1(const void* __restrict__ vbq, creturn tq_apply(tq_load<TY>(vbq, kbx, iqs), bq8_1, iqs);}+// ---- TQP (Gyro-L): 64 trellis steps of 2 weights; a lane is 8 steps (16 values).+// State of step t: the 16 stream bits from bit (63 - t)*K, LSB-first, circular over 64*K bits+// (tqp_state in the fork's ggml-common.h). The step's 2 weights are tq2t_lut_f16[2*hyb(2*s)+j],+// i.e. the FIRST pair of TQ2_T state s = tq_lut_i8[x >> 21] (tq_lut_i8 holds the pair as two int8).+// A lane's 8 states are the 16-bit fields at bit i*K of the bit stream starting at its 8th-to-last+// step, so the whole lane needs K + 2 bytes (7, 8 or 9 for K = 5, 6, 7): one byte-built 64-bit+// window plus one byte for K = 7's first state (which ends at bit 65), then eight shifts. That is+// 7-9 loads per lane instead of 24. The bytes are loaded (not one u64) because a block's qs array+// is only 2-byte aligned (block strides are 42/50/58 bytes).+template<int TY>+__device__ __forceinline__ void tqp_states8(const uint8_t* __restrict__ qs, int lane, uint32_t s[8]) {+ constexpr int K = TqInfo<TY>::K, NB = 8 * K, NW = K + 2 < 8 ? K + 2 : 8;+ const int pidx = (7 - lane) * K; // the lane's first byte, always < NB+ uint64_t w = 0; // bits pidx*8 .. pidx*8+63+#pragma unroll+ for (int j = 0; j < NW; ++j) {+ const int idx0 = pidx + j;+ w |= (uint64_t) qs[idx0 < NB ? idx0 : idx0 - NB] << (8 * j);+ }+ const int i8 = pidx + 8;+ const uint32_t hi = K == 7 ? (uint32_t) qs[i8 < NB ? i8 : i8 - NB] : 0u; // the 65th bit+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const int sh = (7 - i) * K; // 0 .. 7K (<= 49)+ const uint32_t lo = (uint32_t) (w >> sh);+ const uint32_t top = sh > 48 ? (uint32_t) (hi << (64 - sh)) : 0u;+ s[i] = (lo | top) & 0xFFFFu;+ }+}+// The per-state form: 3 byte loads and a shift, no window. Used by dq_tqp (the f16 expert uploads), which+// benches faster with it than with the window (the window's register pressure costs the copy kernels).+template<int TY>+__device__ __forceinline__ void tqp_states8_cheap(const uint8_t* __restrict__ qs, int lane, uint32_t s[8]) {+ constexpr int K = TqInfo<TY>::K, NB = 8 * K;+#pragma unroll+ for (int i = 0; i < 8; ++i) {+ const uint32_t off = (uint32_t) (63 - (8 * lane + i)) * (uint32_t) K;+ const uint32_t i0 = off >> 3;+ const uint32_t i1 = i0 + 1 < (uint32_t) NB ? i0 + 1 : 0;+ const uint32_t i2 = i1 + 1 < (uint32_t) NB ? i1 + 1 : 0;+ const uint32_t w = (uint32_t) qs[i0] | ((uint32_t) qs[i1] << 8) | ((uint32_t) qs[i2] << 16);+ s[i] = (w >> (off & 7)) & 0xFFFFu;+ }+}+template<int TY>+__device__ __forceinline__ TqLane tq_load_tqp(const void* __restrict__ vbq, int kbx, int iqs) {+ const auto* b = (const typename TqInfo<TY>::block_t*) vbq + kbx;+ uint32_t s[8];+ tqp_states8<TY>(b->qs, iqs / 4, s);+ TqLane r;+#pragma unroll+ for (int i = 0; i < 4; ++i) { // two steps (2+2 weights) per int+ const uint32_t x0 = (s[2 * i] & TQ_STATE_MASK) * 0x9e3779b1u;+ const uint32_t x1 = (s[2 * i + 1] & TQ_STATE_MASK) * 0x9e3779b1u;+ r.q[i] = (int) ((uint32_t) tq_lut_i8[x0 >> 21] | ((uint32_t) tq_lut_i8[x1 >> 21] << 16));+ }+ r.d = tq_h2f(b->d);+ return r;+}+template<int TY>+__device__ __forceinline__ float vec_dot_tqp_q8_1(const void* __restrict__ vbq, const block_q8_1* __restrict__ bq8_1,+ const int& kbx, const int& iqs) {+ return tq_apply(tq_load_tqp<TY>(vbq, kbx, iqs), bq8_1, iqs);+}+// ---------------------------------------------------------------- the formats// qk = values per block, ipb = dot calls per block (qi / vdr), step = the iqs stride between calls.template<int TY> struct Fmt;@@ -760,21 +829,28 @@ template<> struct Fmt<145> { static constexpr int qk = 128, ipb = 8, step = 4;__device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<145>(v, y, kbx, iqs); } };template<> struct Fmt<146> { static constexpr int qk = 128, ipb = 8, step = 4;__device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tq_q8_1<146>(v, y, kbx, iqs); } };+// Gyro-L pair trellis: 8 lanes of 8 steps (16 values) per 128-value block+#define STRATA_TQP_FMT(T) template<> struct Fmt<T> { static constexpr int qk = 128, ipb = 8, step = 4; \+ __device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_tqp_q8_1<T>(v, y, kbx, iqs); } };+STRATA_TQP_FMT(147)+STRATA_TQP_FMT(148)+STRATA_TQP_FMT(149)+#undef STRATA_TQP_FMT// The formats of each role, one list each so a type cannot be in one switch and missing from another. Every// entry is a kernel template for each CUDA architecture of the build, hence two lists rather than one.#ifdef STRATA_Q6K_EXPERTS // opt-in build (-DSTRATA_Q6K_EXPERTS=ON): one more instance per kernel, loaded at start-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8) X(40) X(144) X(145)+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(14) X(6) X(2) X(3) X(8) X(40) X(144) X(145) X(147) X(148) X(149)#else-#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8) X(40) X(144) X(145)+#define STRATA_GU_FMTS(X) X(16) X(17) X(18) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(6) X(2) X(3) X(8) X(40) X(144) X(145) X(147) X(148) X(149)#endif// Q5_K / Q6_K downs: Qwen3.6-35B-A3B (FF 512, whole 256-blocks per row); Flash-Next's FF 640 never selects them#ifdef STRATA_Q6K_EXPERTS-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(13) X(14) X(40) X(144) X(146)+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(13) X(14) X(40) X(144) X(146) X(147) X(148) X(149)#else-#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(13) X(40) X(144) X(146)+#define STRATA_D_FMTS(X) X(20) X(23) X(42) X(7) X(6) X(2) X(3) X(8) X(13) X(40) X(144) X(146) X(147) X(148) X(149)#endif-#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8) X(40) X(144) X(145) X(146)+#define STRATA_MMVQ_FMTS(X) X(16) X(17) X(18) X(20) X(21) X(22) X(23) X(29) X(42) X(12) X(13) X(7) X(6) X(2) X(3) X(8) X(40) X(144) X(145) X(146) X(147) X(148) X(149)template<> struct Fmt<40> { static constexpr int qk = 64, ipb = 2, step = 4; // NVFP4: 2 calls of 32 values__device__ static float dot(const void* v, const block_q8_1* y, int kbx, int iqs) { return vec_dot_nvfp4_q8_1(v, y, kbx, iqs); } };@@ -1390,6 +1466,23 @@ template<int TY> struct SplitTq {template<> struct Split<144> : SplitTq<144> {};template<> struct Split<145> : SplitTq<145> {};template<> struct Split<146> : SplitTq<146> {};+// Gyro-L TQP (147-149): the pair layout needs tq_load_tqp, not tq_load, so its split is its own trait.+// Until this existed the launchers took the per-entry mmvq/native kernels for TQP, and row_dot_multi's+// callers re-decoded the lane once per activation.+template<int TY> struct SplitTqp {+ using W = TqLane;+ template<bool STAGE_GRID = false> // (no grid table to stage: the codebook is read where it is)+ __device__ static W load(const void* __restrict__ vbq, int kbx, int iqs, const uint32_t* __restrict__ = nullptr) {+ return tq_load_tqp<TY>(vbq, kbx, iqs);+ }+ __device__ static float apply(const W& r, const block_q8_1* __restrict__ bq8_1, int iqs) { return tq_apply(r, bq8_1, iqs); }+};+template<> inline constexpr bool kSplit<147> = true;+template<> inline constexpr bool kSplit<148> = true;+template<> inline constexpr bool kSplit<149> = true;+template<> struct Split<147> : SplitTqp<147> {};+template<> struct Split<148> : SplitTqp<148> {};+template<> struct Split<149> : SplitTqp<149> {};// One row against the n <= NC activations x + off[0..n) (n >= 1, warp-uniform; offsets in q8_1 blocks, 32-bit to// spare registers), the whole warp. Per activation this is row_dot: the same calls k, lane-strided the same way,@@ -3170,6 +3263,31 @@ __device__ void dq_tq(const void* vx, int64_t ibs, dst_t* yy, int tid) {}}+// TQP (Gyro-L): pair trellis; the step's 2 weights are the first pair of TQ2_T state s.+__device__ __forceinline__ float2 tqp_pair(uint32_t s) {+ const uint32_t* lut = (const uint32_t*) tq_lut_f16;+ const uint32_t x = (s & TQ_STATE_MASK) * 0x9e3779b1u;+ const uint32_t pa = lut[x >> 21];+ return make_float2(tq_h2f(pa & 0xFFFFu), tq_h2f(pa >> 16));+}+// one 256-value call = two 128-blocks; thread tid decodes block tid/16, lane (tid>>1)&7, steps+// 8*l + 4*half .. +3 (8 values, 2 per step)+template<int TY, typename dst_t>+__device__ void dq_tqp(const void* vx, int64_t ibs, dst_t* yy, int tid) {+ const int blk = tid >> 4, l = (tid >> 1) & 7, half = tid & 1;+ const auto* b = (const typename TqInfo<TY>::block_t*) vx + 2 * ibs + blk;+ const float d = tq_h2f(b->d);+ uint32_t s[8];+ tqp_states8_cheap<TY>(b->qs, l, s);+ dst_t* y = yy + blk * 128 + 16 * l + 8 * half;+#pragma unroll+ for (int i = 0; i < 4; ++i) {+ const float2 a = tqp_pair(s[4 * half + i]);+ y[2 * i + 0] = cvt<dst_t>(d * a.x);+ y[2 * i + 1] = cvt<dst_t>(d * a.y);+ }+}+// Every type below must also be in is_iq() (BF16: embed_type_supported): the host entry points refuse the others,// so the default is unreachable.template<typename dst_t>@@ -3198,6 +3316,9 @@ __device__ __forceinline__ void dq_dispatch(int ty, const void* vx, int64_t ibs,case 144: dq_tq<144>(vx, ibs, y, tid); break;case 145: dq_tq<145>(vx, ibs, y, tid); break;case 146: dq_tq<146>(vx, ibs, y, tid); break;+ case 147: dq_tqp<147>(vx, ibs, y, tid); break;+ case 148: dq_tqp<148>(vx, ibs, y, tid); break;+ case 149: dq_tqp<149>(vx, ibs, y, tid); break;default: break;}}@@ -3239,9 +3360,9 @@ __global__ void dequant_gu_kernel(int ty, const void* __restrict__ gate, const vbool is_iq(int t) {return t == 16 || t == 17 || t == 18 || t == 20 || t == 21 || t == 22 || t == 23 || t == 29 || t == 42 || t == 11 ||#ifdef STRATA_Q6K_EXPERTS- t == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8 || t == 40 || t == 144 || t == 145 || t == 146;+ t == 12 || t == 13 || t == 14 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8 || t == 40 || t == 144 || t == 145 || t == 146 || t == 147 || t == 148 || t == 149;#else- t == 12 || t == 13 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8 || t == 40 || t == 144 || t == 145 || t == 146;+ t == 12 || t == 13 || t == 7 || t == 6 || t == 2 || t == 3 || t == 8 || t == 40 || t == 144 || t == 145 || t == 146 || t == 147 || t == 148 || t == 149;#endif}// values per block of the types the grouped expert kernels take (0 = none)@@ -4202,6 +4323,9 @@ size_t iq_row_bytes(int t, int64_t n) noexcept {case 144: return (size_t) (n / 128) * sizeof(block_tq2_t); // APR trellis typescase 145: return (size_t) (n / 128) * sizeof(block_tqk6);case 146: return (size_t) (n / 128) * sizeof(block_tqk7);+ case 147: return (size_t) (n / 128) * sizeof(block_tqp5);+ case 148: return (size_t) (n / 128) * sizeof(block_tqp6);+ case 149: return (size_t) (n / 128) * sizeof(block_tqp7);default: return 0;}}diff --git a/src/kernels/native_expert_parity.cpp b/src/kernels/native_expert_parity.cppindex ffe8a6ca6..a49a8515e 100644--- a/src/kernels/native_expert_parity.cpp+++ b/src/kernels/native_expert_parity.cpp@@ -253,6 +253,19 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int sestd::printf(" | %s %s", cpu::tq_isa_name(isa), same ? "bitwise equal" : "DIFFERENT");if (!same) ++failures;}+ { // time: the scalar kernel against the best SIMD one this CPU has, same rows and tokens+ const int it = 30;+ const auto t0 = std::chrono::steady_clock::now();+ for (int k = 0; k < it; ++k) cpu::tq_rows(cpu::TqIsa::scalar, type, w, rb, n, acts, NT, rp, 0, rows);+ const auto t1 = std::chrono::steady_clock::now();+ const cpu::TqIsa best = cpu::tq_isa();+ for (int k = 0; k < it; ++k) cpu::tq_rows(best, type, w, rb, n, acts, NT, gp, 0, rows);+ const auto t2 = std::chrono::steady_clock::now();+ const double us0 = std::chrono::duration<double, std::micro>(t1 - t0).count() / it;+ const double us1 = std::chrono::duration<double, std::micro>(t2 - t1).count() / it;+ std::printf(" | %s %.0f us, scalar %.0f us (%.2fx)", cpu::tq_isa_name(best), us1, us0,+ us1 > 0 ? us0 / us1 : 0.0);+ }std::printf("\n");}// UD-Q4_K_XL's formats: the multi-token AVX2 kernels (kq_avx2.cpp) against ggml-cpu's vec_dot per token,@@ -482,6 +495,11 @@ int check_blob(const cpu::NativeFmt& f, const std::vector<uint8_t>& blob, int seint type_of(std::string name) {for (auto& c : name) c = (char) std::tolower((unsigned char) c);+ // The trellis types sit above GGML_TYPE_COUNT on an upstream ggml (the overlay adds them to ggml-cuda, not to the+ // type table), so ggml_type_name cannot resolve them: name them here, as the engine's own type ids do.+ static const struct { const char* n; int t; } trellis[] = {{"tq2_t", 144}, {"tqk6", 145}, {"tqk7", 146},+ {"tqp5", 147}, {"tqp6", 148}, {"tqp7", 149}};+ for (const auto& e : trellis) if (name == e.n) return e.t;for (int t = 0; t < GGML_TYPE_COUNT; ++t) {const char* n = ggml_type_name((ggml_type) t);if (n == nullptr) continue;@@ -837,6 +855,15 @@ int main(int argc, char** argv) {const int dn = slash == std::string::npos ? -1 : type_of(arg.substr(slash + 1));cpu::NativeFmt f;std::string err;+ // The synthetic check quantizes the blob with ggml and compares against ggml's to_float. An upstream+ // ggml (the MMQ overlay) has no traits for the trellis types (they sit above GGML_TYPE_COUNT), so the+ // check needs a ggml that carries them (a build with -DSTRATA_GGML_DIR=<the agention llama.cpp>): skip it+ // here with a message instead of reading past the type table.+ if (gu >= (int) GGML_TYPE_COUNT || dn >= (int) GGML_TYPE_COUNT) { // (ggml_type_name of an id past the table is not a null pointer: it reads past it)+ std::printf("%s: skipped (this ggml has no trellis type traits: the check needs STRATA_GGML_DIR=<a "+ "llama.cpp with the trellis types>)\n", arg.c_str());+ continue;+ }if (gu < 0 || dn < 0 || !cpu::native_fmt(gu, dn, H, FF, f, err) ||!strata::kernels::native_expert_supported(gu, dn, H, FF)) {std::printf("%s: %s\n", arg.c_str(), err.empty() ? "not a pair the GPU expert kernels take" : err.c_str());diff --git a/src/kernels/tq_pascal_bench.cpp b/src/kernels/tq_pascal_bench.cppindex bbed71aca..1c39d8272 100644--- a/src/kernels/tq_pascal_bench.cpp+++ b/src/kernels/tq_pascal_bench.cpp@@ -1,4 +1,4 @@-// src/kernels/tq_pascal_bench.cpp - the Gyro trellis decode kernels (TQ2_T 144, TQK6 145, TQK7 146) on synthetic blocks.+// src/kernels/tq_pascal_bench.cpp - the Gyro trellis decode kernels (TQ2_T 144, TQK6 145, TQK7 146 and Gyro-L's TQP5-7 147-149) on synthetic blocks.//// build/tq_pascal_bench checks: the decode-once kernels against the per-column ones (bitwise), and a// checksum of every output, so two builds can be compared byte for byte@@ -11,6 +11,8 @@#include "strata/kernels/iq_kernels.hpp"#include "strata/kernels/native_mmvq.hpp"+#include "ggml-common-tq.h" // tq_lut_f16: the codebook, as host data+#include <cuda_runtime.h>#include <algorithm>@@ -45,7 +47,10 @@ T* dalloc(size_t n) {return p;}-const char* name_of(int t) { return t == 144 ? "TQ2_T" : t == 145 ? "TQK6" : t == 146 ? "TQK7" : t == 42 ? "Q2_0" : "?"; }+const char* name_of(int t) {+ return t == 144 ? "TQ2_T" : t == 145 ? "TQK6" : t == 146 ? "TQK7" : t == 147 ? "TQP5" : t == 148 ? "TQP6" : t == 149 ? "TQP7"+ : t == 42 ? "Q2_0" : "?";+}uint64_t fnv(const void* p, size_t n, uint64_t h = 1469598103934665603ull) {const uint8_t* b = (const uint8_t*) p;@@ -206,6 +211,124 @@ void check_grouped(int gu, int dt, int64_t H, int64_t FF, cudaStream_t s, std::mif (!ok) { std::printf(" %zu of %zu floats differ%s\n", diff, a.size(), finite ? "" : ", non-finite outputs"); ++g_fail; }}+// ---- Gyro-L pair trellis (TQP5/6/7, ggml 147-149) against a reference written from the format definition (no ggml):+// state t of a block = the 16 stream bits that start at bit (63 - t) * K, LSB first, circular over 64 * K bits (15 bits+// are used); the step's two weights are the first pair of the state's codebook entry, times the block's fp16 scale.+uint32_t tqp_ref_state(const uint8_t* qs, int K, int t) {+ const int nbits = 64 * K;+ uint32_t s = 0;+ for (int b = 0; b < 16; ++b) {+ const int pos = ((63 - t) * K + b) % nbits;+ s |= (uint32_t) ((qs[pos >> 3] >> (pos & 7)) & 1) << b;+ }+ return s & 0x7FFFu;+}++float half_to_f(uint16_t h) { // normal and subnormal fp16 (no inf / nan in these tests)+ const int e = (h >> 10) & 31, m = h & 1023;+ const float v = e == 0 ? std::ldexp((float) m, -24) : std::ldexp((float) (1024 + m), e - 25);+ return (h & 0x8000) ? -v : v;+}++uint16_t f_to_half_rn(float f) { // round to nearest even, the GPU's __float2half_rn+ uint32_t x;+ std::memcpy(&x, &f, 4);+ const uint32_t sign = (x >> 16) & 0x8000u;+ const int e = (int) ((x >> 23) & 0xFF) - 127 + 15;+ uint32_t m = x & 0x7FFFFFu;+ if (e >= 31) return (uint16_t) (sign | 0x7C00u);+ if (e <= 0) {+ if (e < -10) return (uint16_t) sign;+ m |= 0x800000u;+ const int shift = 14 - e;+ uint32_t half = m >> shift;+ const uint32_t rem = m & ((1u << shift) - 1), mid = 1u << (shift - 1);+ if (rem > mid || (rem == mid && (half & 1u))) ++half;+ return (uint16_t) (sign | half);+ }+ uint32_t half = ((uint32_t) e << 10) | (m >> 13);+ const uint32_t rem = m & 0x1FFFu;+ if (rem > 0x1000u || (rem == 0x1000u && (half & 1u))) ++half;+ return (uint16_t) (sign | half);+}++// the weights of `rows` rows of `n` values, as floats (d * codebook, unrounded)+std::vector<float> tqp_ref_weights(int t, const std::vector<uint8_t>& w, int64_t rows, int64_t n) {+ const int K = t - 142; // 147 -> 5, 148 -> 6, 149 -> 7+ const size_t bs = k::iq_row_bytes(t, 128);+ std::vector<float> out((size_t) (rows * n));+ for (size_t b = 0; b < (size_t) (rows * n / 128); ++b) {+ const uint8_t* blk = &w[b * bs];+ uint16_t hd;+ std::memcpy(&hd, blk, 2);+ const float d = half_to_f(hd);+ for (int step = 0; step < 64; ++step) {+ const uint32_t s = tqp_ref_state(blk + 2, K, step);+ const uint32_t x = (s & 0x7FFFu) * 0x9e3779b1u;+ const uint32_t pt = x >> 21;+ out[b * 128 + 2 * step + 0] = d * half_to_f(tq_lut_f16[2 * pt + 0]);+ out[b * 128 + 2 * step + 1] = d * half_to_f(tq_lut_f16[2 * pt + 1]);+ }+ }+ return out;+}++void check_tqp_reference(int t, int n_in, int n_out, cudaStream_t s, std::mt19937& rng) {+ const auto w = random_rows(t, n_out, n_in, rng);+ const auto ref = tqp_ref_weights(t, w, n_out, n_in);+ uint8_t* dw = dalloc<uint8_t>(w.size());+ ck(cudaMemcpy(dw, w.data(), w.size(), cudaMemcpyHostToDevice), "w");+ // (1) the prompt path's fp16 dequantizer: bit for bit the fp16 of the reference weights+ const int64_t nv = (int64_t) n_out * n_in;+ uint16_t* dh = dalloc<uint16_t>((size_t) nv);+ k::iq_dequant_f16(t, dw, nv, dh, s);+ ck(cudaStreamSynchronize(s), "sync");+ std::vector<uint16_t> h((size_t) nv);+ ck(cudaMemcpy(h.data(), dh, h.size() * 2, cudaMemcpyDeviceToHost), "h");+ size_t bad = 0;+ for (size_t i = 0; i < h.size(); ++i) bad += h[i] != f_to_half_rn(ref[i]);+ std::printf("tqp ref %-6s dequant_f16 %5d x %5d: %zu of %zu fp16 values differ from the definition %s\n", name_of(t), n_out, n_in,+ bad, h.size(), bad ? "FAIL" : "bitwise equal");+ g_fail += bad != 0;+ // (2) the decode kernels (int8 codebook, q8_1 activations): relative L1 error against the float weights . the+ // dequantized q8_1 activations, in double, below the 3e-2 every expert type gets in native_expert_parity+ const int nc = 3;+ const auto x = random_x((size_t) nc * n_in, rng);+ float* dx = dalloc<float>(x.size());+ ck(cudaMemcpy(dx, x.data(), x.size() * 4, cudaMemcpyHostToDevice), "x");+ uint8_t* dxq = dalloc<uint8_t>((size_t) nc * (n_in / 32) * 36);+ k::quantize_q8_1_rows(dx, nc, n_in, dxq, s);+ float* dy = dalloc<float>((size_t) nc * n_out);+ k::iq_mmvq(t, dw, dxq, dy, n_in, n_out, nc, s);+ ck(cudaStreamSynchronize(s), "sync");+ std::vector<float> y((size_t) nc * n_out);+ ck(cudaMemcpy(y.data(), dy, y.size() * 4, cudaMemcpyDeviceToHost), "y");+ std::vector<uint8_t> xq((size_t) nc * (n_in / 32) * 36);+ ck(cudaMemcpy(xq.data(), dxq, xq.size(), cudaMemcpyDeviceToHost), "xq");+ double l1 = 0, l1ref = 0;+ for (int c = 0; c < nc; ++c) {+ std::vector<double> xd((size_t) n_in);+ for (int b = 0; b < n_in / 32; ++b) {+ const uint8_t* blk = &xq[((size_t) c * (n_in / 32) + b) * 36];+ uint16_t hd;+ std::memcpy(&hd, blk, 2);+ const double d = half_to_f(hd);+ for (int j = 0; j < 32; ++j) xd[(size_t) b * 32 + j] = d * (double) (int8_t) blk[4 + j];+ }+ for (int r = 0; r < n_out; ++r) {+ double acc = 0;+ for (int i = 0; i < n_in; ++i) acc += (double) ref[(size_t) r * n_in + i] * xd[(size_t) i];+ l1 += std::fabs((double) y[(size_t) c * n_out + r] - acc);+ l1ref += std::fabs(acc);+ }+ }+ const double rel = l1 / l1ref;+ std::printf("tqp ref %-6s mmvq %5d x %5d x %d cols: relative L1 error %.2e (bound 3e-2) %s\n", name_of(t), n_out, n_in, nc, rel,+ rel < 3e-2 ? "ok" : "FAIL");+ g_fail += !(rel < 3e-2);+ cudaFree(dw); cudaFree(dh); cudaFree(dx); cudaFree(dxq); cudaFree(dy);+}+float time_ms(cudaStream_t s, int it, const auto& fn) {cudaEvent_t e0, e1;cudaEventCreate(&e0);@@ -286,6 +409,7 @@ int main(int argc, char** argv) {cudaGetDeviceProperties(&prop, 0);std::printf("device %s cc %d.%d\n", prop.name, prop.major, prop.minor);std::mt19937 rng(18);+ // Gyro-S / Gyro-M types first, on the generator they always had: their "SUM" lines are the same bits as before the pair typesfor (int t : {144, 145, 146}) {check_mmvq(t, 2560, 67, s, rng);check_mmvq(t, 1024, 5, s, rng);@@ -299,6 +423,19 @@ int main(int argc, char** argv) {check_grouped(gu, 144, 2560, 512, s, rng);check_grouped(gu, 146, 2048, 256, s, rng);}+ // Gyro-L's pair trellis (own generator): decode-once == per-column, then a reference from the format definition+ // (fp16 dequantizer bit for bit, decode kernels to 3e-2)+ std::mt19937 rng_p(19);+ for (int t : {147, 148, 149}) {+ check_mmvq(t, 2560, 67, s, rng_p);+ check_mmvq(t, 1024, 5, s, rng_p);+ check_tqp_reference(t, 2560, 67, s, rng_p);+ check_tqp_reference(t, 1024, 5, s, rng_p);+ check_grouped(t, t, 2560, 640, s, rng_p); // the pair trellis in both roles, as its file has it (tqp6 / tqp6)+ check_grouped(t, t, 5120, 640, s, rng_p);+ }+ check_grouped(148, 146, 2560, 640, s, rng_p);+ check_grouped(145, 148, 2560, 640, s, rng_p);if (do_bench) bench(s, rng);std::printf("tq_pascal_bench: %d failures\n", g_fail);cudaStreamDestroy(s);diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cuindex e7ff5d45b..96098dfad 100644--- a/src/prefill/moe_mmq.cu+++ b/src/prefill/moe_mmq.cu@@ -21,11 +21,15 @@ namespace strata::prefill::mmq {namespace {// The Gyro trellis types (ids 144-146) are outside the type table of an upstream ggml (indexed by the id): their block// geometry is known here (128 weights; 34 / 26 / 30 bytes), everything else asks ggml.-inline bool is_tq(int t) { return t >= 144 && t <= 146; }+inline bool is_tq(int t) { return t >= 144 && t <= 149; } // 144-146 TQ/TQK, 147-149 the pair trellis (Gyro-L)inline int64_t type_blck(int t) { return is_tq(t) ? 128 : (int64_t) ggml_blck_size((ggml_type) t); }-inline size_t type_bytes(int t) { return t == 144 ? 34 : t == 145 ? 26 : t == 146 ? 30 : ggml_type_size((ggml_type) t); }+inline size_t type_bytes(int t) {+ return t == 144 ? 34 : t == 145 ? 26 : t == 146 ? 30 : t == 147 ? 42 : t == 148 ? 50 : t == 149 ? 58+ : ggml_type_size((ggml_type) t);+}inline const char* type_label(int t) {- return t == 144 ? "tq2_t" : t == 145 ? "tqk6" : t == 146 ? "tqk7" : ggml_type_name((ggml_type) t);+ return t == 144 ? "tq2_t" : t == 145 ? "tqk6" : t == 146 ? "tqk7"+ : t == 147 ? "tqp5" : t == 148 ? "tqp6" : t == 149 ? "tqp7" : ggml_type_name((ggml_type) t);}} // namespace#if defined(STRATA_KQ_DSO)@@ -275,6 +279,9 @@ bool supported(int t) {case GGML_TYPE_Q8_0: // the draft layer's dense matrices (E-9)#ifdef STRATA_MMQ_TQcase GGML_TYPE_TQ2_T: case GGML_TYPE_TQK6: case GGML_TYPE_TQK7: // APR experts (agention fork's MMQ)+#endif+#ifdef STRATA_MMQ_TQP // a ggml with the pair-trellis types (the overlay, or an agention tree that has their instances)+ case GGML_TYPE_TQP5: case GGML_TYPE_TQP6: case GGML_TYPE_TQP7: // Gyro-L pair trellis (types 147-149)#endifreturn true;#ifdef STRATA_MMQ_KQUANTS@@ -462,6 +469,11 @@ void Context::run(const Product& p, void* stream) {case GGML_TYPE_TQK6: mul_mat_q_case<GGML_TYPE_TQK6>(ctx, a, s); break;case GGML_TYPE_TQK7: mul_mat_q_case<GGML_TYPE_TQK7>(ctx, a, s); break;#endif+#ifdef STRATA_MMQ_TQP+ case GGML_TYPE_TQP5: mul_mat_q_case<GGML_TYPE_TQP5>(ctx, a, s); break;+ case GGML_TYPE_TQP6: mul_mat_q_case<GGML_TYPE_TQP6>(ctx, a, s); break;+ case GGML_TYPE_TQP7: mul_mat_q_case<GGML_TYPE_TQP7>(ctx, a, s); break;+#endif#if defined(STRATA_MMQ_KQUANTS) && defined(STRATA_KQ_DSO)case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_1: {const int e = kq::run((int) t, &a, s);diff --git a/third_party/ggml-tq/tq.cuh b/third_party/ggml-tq/tq.cuhindex 9e9e8a262..a20fe3b22 100644--- a/third_party/ggml-tq/tq.cuh+++ b/third_party/ggml-tq/tq.cuh@@ -15,6 +15,7 @@#define TQ_LUT_POINTS 2048#define TQ_LANES 8 // lanes per 128-weight block#define TQ_STEPS_LANE 4 // trellis steps per lane+#define TQ_PAIRS_LANE 8 // codebook pairs per lane (the TQP layout: 2 weights per step)#define QR_TQ 4#define QI_TQ (QK_TQK/QR_TQ)@@ -44,14 +45,34 @@ struct ggml_cuda_type_traits<GGML_TYPE_TQK7> {static constexpr int bs = sizeof(block_tqk7);};+// Gyro-L pair trellis (types 147-149): the same lane packing (16 values = 4 ints of 4 int8), 64 steps of+// 2 weights per 128-value block.+template<> struct ggml_cuda_type_traits<GGML_TYPE_TQP5> {+ static constexpr int qk = QK_TQP; static constexpr int qr = QR_TQ; static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tqp5); };+template<> struct ggml_cuda_type_traits<GGML_TYPE_TQP6> {+ static constexpr int qk = QK_TQP; static constexpr int qr = QR_TQ; static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tqp6); };+template<> struct ggml_cuda_type_traits<GGML_TYPE_TQP7> {+ static constexpr int qk = QK_TQP; static constexpr int qr = QR_TQ; static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tqp7); };++static constexpr __host__ __device__ bool ggml_cuda_type_is_tqp(ggml_type type) {+ return type == GGML_TYPE_TQP5 || type == GGML_TYPE_TQP6 || type == GGML_TYPE_TQP7;+}+static constexpr __host__ __device__ bool ggml_cuda_type_is_tq(ggml_type type) {- return type == GGML_TYPE_TQ2_T || type == GGML_TYPE_TQK6 || type == GGML_TYPE_TQK7;+ return type == GGML_TYPE_TQ2_T || type == GGML_TYPE_TQK6 || type == GGML_TYPE_TQK7 || ggml_cuda_type_is_tqp(type);}template <ggml_type type> struct tq_block_info;template <> struct tq_block_info<GGML_TYPE_TQ2_T> { typedef block_tq2_t block_t; static constexpr int K = 8; };template <> struct tq_block_info<GGML_TYPE_TQK6> { typedef block_tqk6 block_t; static constexpr int K = 6; };template <> struct tq_block_info<GGML_TYPE_TQK7> { typedef block_tqk7 block_t; static constexpr int K = 7; };+template <> struct tq_block_info<GGML_TYPE_TQP5> { typedef block_tqp5 block_t; static constexpr int K = 5; };+template <> struct tq_block_info<GGML_TYPE_TQP6> { typedef block_tqp6 block_t; static constexpr int K = 6; };+template <> struct tq_block_info<GGML_TYPE_TQP7> { typedef block_tqp7 block_t; static constexpr int K = 7; };+static_assert(QK_TQP == QK_TQK, "trellis block sizes differ");// States of steps 4l..4l+3 of one block. qs is the block's qs array, 2-byte aligned// (every trellis block has an even size and qs sits right after the fp16 scale).@@ -88,6 +109,34 @@ static __device__ __forceinline__ void tq_states4(const uint8_t * qs, const int}}+// TQP: the 15-bit states of steps 8l..8l+7 (s[j] = step 8l + j). Step t's window starts at stream bit+// (63 - t)*K, so the lane's steps lie in the bits from S = (56 - 8l)*K: step 8l + j at S + (7 - j)*K.+// S = 8*(7 - l)*K, so r = S & 15 is 0 or 8 and r + 7*K + 15 <= 72: five u16 words (wrapped modulo the+// 4*K words of the stream) cover it, the fifth only when r + 7*K + 15 > 64 (never for K = 5).+template <ggml_type type>+static __device__ __forceinline__ void tqp_states8(const uint8_t * qs, const int l, uint32_t s[TQ_PAIRS_LANE]) {+ constexpr int K = tq_block_info<type>::K;+ constexpr int nwords = 4*K;+ const uint16_t * qs16 = (const uint16_t *) qs;+ const int S = (56 - 8*l)*K;+ const int wb = S >> 4;+ const int r = S & 15;+ const int wi1 = wb + 1 >= nwords ? wb + 1 - nwords : wb + 1;+ const int wi2 = wb + 2 >= nwords ? wb + 2 - nwords : wb + 2;+ const int wi3 = wb + 3 >= nwords ? wb + 3 - nwords : wb + 3;+ uint64_t v = (uint64_t) qs16[wb] | ((uint64_t) qs16[wi1] << 16) |+ ((uint64_t) qs16[wi2] << 32) | ((uint64_t) qs16[wi3] << 48);+ v >>= r;+ if constexpr (8 + 7*K + TQ_STATE_BITS > 64) {+ const int wi4 = wb + 4 >= nwords ? wb + 4 - nwords : wb + 4;+ v |= r ? (uint64_t) qs16[wi4] << (64 - r) : 0;+ }+#pragma unroll+ for (int j = 0; j < TQ_PAIRS_LANE; ++j) {+ s[j] = (uint32_t) (v >> ((7 - j)*K)) & TQ_STATE_MASK;+ }+}+// MMQ (batched) path: lane l of block x as 16 int8 values (4 packed ints, weight order) with one float scale per// lane, i.e. weights ~= scale * q. The int8 codebook (one global scale) is already int8: no decode to float.template <ggml_type type>@@ -95,13 +144,24 @@ static __device__ __forceinline__ float tq_lane_to_q8(const void * x, const inttypedef typename tq_block_info<type>::block_t block_t;const block_t * b = (const block_t *) x;- uint32_t s[TQ_STEPS_LANE];- tq_states4<type>(b->qs, l, s);-+ if constexpr (ggml_cuda_type_is_tqp(type)) {+ // 8 pairs of 2 weights, in weight order: two pairs per int, as the TQ path packs two steps+ uint32_t s[TQ_PAIRS_LANE];+ tqp_states8<type>(b->qs, l, s);+#pragma unroll+ for (int i = 0; i < TQ_STEPS_LANE; ++i) {+ const uint32_t p0 = tq_lut_i8[((s[2 * i] & TQ_STATE_MASK) * 0x9e3779b1u) >> 21];+ const uint32_t p1 = tq_lut_i8[((s[2 * i + 1] & TQ_STATE_MASK) * 0x9e3779b1u) >> 21];+ q[i] = (int) (p0 | (p1 << 16));+ }+ } else {+ uint32_t s[TQ_STEPS_LANE];+ tq_states4<type>(b->qs, l, s);#pragma unroll- for (int i = 0; i < TQ_STEPS_LANE; ++i) {- const uint32_t xh = (s[i] & TQ_STATE_MASK) * 0x9e3779b1u;- q[i] = (int) ((uint32_t) tq_lut_i8[xh >> 21] | ((uint32_t) tq_lut_i8[(xh >> 10) & 2047u] << 16));+ for (int i = 0; i < TQ_STEPS_LANE; ++i) {+ const uint32_t xh = (s[i] & TQ_STATE_MASK) * 0x9e3779b1u;+ q[i] = (int) ((uint32_t) tq_lut_i8[xh >> 21] | ((uint32_t) tq_lut_i8[(xh >> 10) & 2047u] << 16));+ }}// Strata's block structs hold the fp16 scale as uint16_t (the fork's as ggml_half): reinterpret, do not convertreturn __half2float(__ushort_as_half(b->d)) * TQ_LUT_I8_SCALE;diff --git a/third_party/ggml/ggml-common-tq.h b/third_party/ggml/ggml-common-tq.hindex 1ce3773bf..6e00a97d9 100644--- a/third_party/ggml/ggml-common-tq.h+++ b/third_party/ggml/ggml-common-tq.h@@ -32,6 +32,26 @@ typedef struct {} block_tqk7;static_assert(sizeof(block_tqk7) == 30, "wrong tqk7 block size/padding");+// Gyro-L pair trellis (agentionai llama.cpp release/gyro-l): 64 steps of K bits, 2 weights per step+// (the TQ2_T codebook's first pair). Defined in the fork's ggml-common.h; repeated here so Strata's+// device and CPU kernels compile without GGML_COMMON_IMPL.+#define QK_TQP 128+typedef struct {+ uint16_t d;+ uint8_t qs[8*5]; // 64 steps x 5 bits+} block_tqp5;+static_assert(sizeof(block_tqp5) == 42, "wrong tqp5 block size/padding");+typedef struct {+ uint16_t d;+ uint8_t qs[8*6]; // 64 steps x 6 bits+} block_tqp6;+static_assert(sizeof(block_tqp6) == 50, "wrong tqp6 block size/padding");+typedef struct {+ uint16_t d;+ uint8_t qs[8*7]; // 64 steps x 7 bits+} block_tqp7;+static_assert(sizeof(block_tqp7) == 58, "wrong tqp7 block size/padding");+// device memory in a CUDA / HIP translation unit, ordinary host data everywhere else#if defined(__CUDACC__) || defined(__HIPCC__)#define STRATA_TQ_LUT_SPACE __device__diff --git a/tools/gguf_reader.py b/tools/gguf_reader.pyindex 4da9d2d24..1c65a2889 100644--- a/tools/gguf_reader.py+++ b/tools/gguf_reader.py@@ -27,6 +27,8 @@ GGML_TYPES: dict[int, str] = {39: "MXFP4", 40: "NVFP4", 41: "Q4_0_8_8", 42: "Q2_0",# Agention's llama.cpp fork (agentionai/llama.cpp): the APR trellis expert types (128-value blocks)143: "PTQ1_0", 144: "TQ2_T", 145: "TQK6", 146: "TQK7",+ # pair-trellis family added for Gyro-L (agentionai llama.cpp release/gyro-l)+ 147: "TQP5", 148: "TQP6", 149: "TQP7",}# GGUF metadata value type ids@@ -49,6 +51,8 @@ BLOCK_GEOMETRY: dict[str, tuple[int, int]] = {"IQ1_M": (256, 56), "Q2_0": (64, 18), "MXFP4": (32, 17), "NVFP4": (64, 36),# fp16 scale + trellis stream (32 steps x 8 / 6 / 7 bits); PTQ1_0: 26 B of trits + fp16 scale"TQ2_T": (128, 34), "TQK6": (128, 26), "TQK7": (128, 30), "PTQ1_0": (128, 28),+ # TQP5/6/7: fp16 scale + 8*K bytes of a 64-step x K-bit circular stream (K = 5/6/7)+ "TQP5": (128, 42), "TQP6": (128, 50), "TQP7": (128, 58),}diff --git a/tools/test_gguf_tqp.py b/tools/test_gguf_tqp.pynew file mode 100644index 000000000..338eda96b--- /dev/null+++ b/tools/test_gguf_tqp.py@@ -0,0 +1,34 @@+"""The pair-trellis ggml types (147-149: TQP5 / TQP6 / TQP7, Gyro-L): the Python reader and the C++ reader agree on+their names and block sizes, and the sizes follow the layout (fp16 scale + 8 * K bytes per 128 values)."""++import re+import sys+import unittest+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT / "tools"))++import gguf_reader # noqa: E402+++class PairTrellisTypes(unittest.TestCase):+ def test_python_reader_names_and_sizes(self):+ for t, name, k in ((147, "TQP5", 5), (148, "TQP6", 6), (149, "TQP7", 7)):+ self.assertEqual(gguf_reader.GGML_TYPES[t], name)+ self.assertEqual(gguf_reader.BLOCK_GEOMETRY[name], (128, 2 + 8 * k))++ def test_older_trellis_types_unchanged(self):+ self.assertEqual(gguf_reader.BLOCK_GEOMETRY["TQ2_T"], (128, 34))+ self.assertEqual(gguf_reader.BLOCK_GEOMETRY["TQK6"], (128, 26))+ self.assertEqual(gguf_reader.BLOCK_GEOMETRY["TQK7"], (128, 30))++ def test_cpp_reader_has_the_same_numbers(self):+ text = (ROOT / "include" / "strata" / "artifact" / "gguf_reader.hpp").read_text(encoding="utf-8")+ for t, name, k in ((147, "TQP5", 5), (148, "TQP6", 6), (149, "TQP7", 7)):+ self.assertRegex(text, r"case %d:\s*return \"%s\";" % (t, name))+ self.assertRegex(text, r"case %d:\s*elems = 128;\s*bytes = %d;" % (t, 2 + 8 * k))+++if __name__ == "__main__":+ unittest.main()