Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Step B: Gyro's trellis MMQ prompt kernels on an upstream ggml, so the packaged engine reads Gyro prompts as fast as rc2
Niko1221b15b209a0368
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 44b6ec4e9..fb0de7975 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -76,6 +76,7 @@ option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF# CUDA_MODULE_LOADING=EAGER, and these took ~3.8 MB of VRAM - 3 IQ3_XXS expert slots on a P100 - from every model); on# HIP a code object loads on first use, so they cost other models nothing there (R9700: the same slots). Used at run# time with STRATA_MMQ_KQ=1 (opt-in: it changes the prompt's rounding against the FP16 path).+option(STRATA_MMQ_TQ_OVERLAY "Gyro trellis MMQ prompt kernels on an upstream llama.cpp: an overlay of its ggml-cuda headers written at configure time (cmake/strata_ggml_tq_overlay.cmake)" ON)option(STRATA_MMQ_KQUANTS "Build the MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K: the experts of Unsloth's K-quant files, and on HIP the dense GGUF projections (STRATA_DENSE_MMQ)" ON)# Q6_K gate/up in the grouped expert kernels (Unsloth's UD-Q6_K_XL, and layer 2 of UD-Q5_K_XL; docs/QUANTS.md). On by# default since MODEL-UPDATE Phase 1: one more instance per grouped kernel in iq_kernels.cu, +19 KB of sm_60 code,@@ -1449,6 +1450,28 @@ if(STRATA_NATIVE_EXPERTS)if(STRATA_GGML_DIR AND EXISTS "${STRATA_GGML_DIR}/ggml/src/ggml-cuda/template-instances/mmq-instance-tqk6.cu")list(APPEND _strata_mmq_types tq2_t tqk6 tqk7)set(_strata_mmq_tq ON)+ elseif(STRATA_MMQ_TQ_OVERLAY AND (STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP))+ # an upstream llama.cpp: the trellis MMQ instances come from an overlay of the pinned ggml-cuda headers written at+ # configure time (cmake/strata_ggml_tq_overlay.cmake); off with a message if the llama.cpp tree is not the pinned one+ if(STRATA_GGML_DIR)+ set(_strata_tq_llama "${STRATA_GGML_DIR}")+ else()+ set(_strata_tq_llama "${strata_llamacpp_SOURCE_DIR}")+ endif()+ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/strata_ggml_tq_overlay.cmake)+ set(_strata_tq_overlay_dir "${CMAKE_BINARY_DIR}/ggml-tq-overlay")+ strata_ggml_tq_overlay("${_strata_tq_llama}" "${_strata_tq_overlay_dir}")+ if(STRATA_TQ_OVERLAY_OK)+ configure_file(third_party/ggml/ggml-common-tq.h "${_strata_tq_overlay_dir}/ggml-common-tq.h" COPYONLY)+ configure_file(src/kernels/cuda/tq_lut_i8.cuh "${_strata_tq_overlay_dir}/tq_lut_i8.cuh" COPYONLY)+ set(_strata_mmq_tq ON)+ set(_strata_mmq_tq_overlay ON)+ message(STATUS "trellis MMQ (Gyro prompts): overlay on the pinned ggml-cuda in ${_strata_tq_overlay_dir}")+ if(STRATA_BUILD_TESTS)+ add_test(NAME ggml_tq_overlay COMMAND ${CMAKE_COMMAND} -DLLAMA=${_strata_tq_llama}+ -DOUT=${CMAKE_BINARY_DIR}/ggml-tq-overlay-test -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/ggml_tq_overlay_test.cmake)+ endif()+ endif()endif()# #296: OrcaRouter Q4_K_S's Q5_0 experts (Q4_K and Q5_1 come with STRATA_MMQ_KQUANTS on CUDA, S5's branch)if(STRATA_ORCA_Q4KS_MMQ)@@ -1464,11 +1487,15 @@ if(STRATA_NATIVE_EXPERTS)set(_strata_gg ${strata_llamacpp_SOURCE_DIR}/ggml)endif()set(_strata_mmq ${_strata_gg}/src/ggml-cuda)+ set(_strata_quantize_cu ${_strata_mmq}/quantize.cu) # the overlay's copy when the trellis overlay is on: it includes mmq.cuh+ if(_strata_mmq_tq_overlay)+ set(_strata_quantize_cu ${_strata_tq_overlay_dir}/quantize.cu)+ endif()# #136: the Q2_0 pack's prompt experts on Strata's own int8 kernels (moe_fused.cu, opt-in STRATA_PF_FUSED=1; CUDA# only: the HIP branch below keeps MMQ, and an sm_75 device or an sm_75-only build reports them unavailable)add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cusrc/prefill/mmq_nvfp4_w4a8.cu- src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)+ src/prefill/ggml_cuda_host.cu ${_strata_quantize_cu})# q8_0: the drafter's matrices (E-9); STRATA_ORCA_Q4KS_MMQ adds q5_0 (#296, in _strata_mmq_types), and# STRATA_MMQ_KQUANTS q4_k q5_k q5_1 q6_k, the experts of Unsloth's UD-Q4/Q5/Q6_K_XL (docs/QUANTS.md; the dispatch# follows eddoursul/Strata 7706c1b). MODEL-UPDATE Phase 1: on CUDA they are a library of their own, strata_kq@@ -1486,6 +1513,13 @@ if(STRATA_NATIVE_EXPERTS)foreach(_t IN LISTS _strata_mmq_cuda_types)target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)endforeach()+ if(_strata_mmq_tq_overlay)+ foreach(_t tq2_t tqk6 tqk7)+ target_sources(strata_mmq PRIVATE ${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu)+ endforeach()+ # first on the include path: the patched mmq.cuh / ggml.h shadow the pinned ones for this target only+ target_include_directories(strata_mmq BEFORE PRIVATE ${_strata_tq_overlay_dir} ${_strata_tq_overlay_dir}/include)+ endif()# NVFP4: W4A8 (default) in mmq_nvfp4_w4a8.cu with the plain architectures; the opt-in W4A4 needs RTX 50's FP4 MMA,# which exists only in the arch-specific target, so its one unit alone is built with 12x -> 12xa (the default# binary is unchanged and runs on sm_121; a card without that image falls back to W4A8 at run time)@@ -1639,6 +1673,10 @@ if(STRATA_NATIVE_EXPERTS)set(_strata_gg "${strata_llamacpp_SOURCE_DIR}/ggml")endif()set(_strata_mmq "${_strata_gg}/src/ggml-cuda")+ set(_strata_quantize_cu "${_strata_mmq}/quantize.cu") # the overlay's copy when the trellis overlay is on: it includes mmq.cuh+ if(_strata_mmq_tq_overlay)+ set(_strata_quantize_cu "${_strata_tq_overlay_dir}/quantize.cu")+ endif()if(NOT EXISTS "${_strata_mmq}/common.cuh" OR NOT EXISTS "${_strata_mmq}/quantize.cu")message(FATAL_ERROR "The selected GGML source is missing the ggml-cuda MMQ sources: ${_strata_mmq}")endif()@@ -1648,7 +1686,7 @@ if(STRATA_NATIVE_EXPERTS)src/prefill/moe_fused.cusrc/prefill/moe_fused_iq.cusrc/prefill/ggml_cuda_host.cu- "${_strata_mmq}/quantize.cu")+ "${_strata_quantize_cu}")if(STRATA_MMQ_KQUANTS)# the same K-quant instances as the CUDA branch: on HIP they serve the dense GGUF projections# of Gemm::native (STRATA_DENSE_MMQ=1), the mixed-quant packs' attention/PLE tensors@@ -1657,6 +1695,11 @@ if(STRATA_NATIVE_EXPERTS)foreach(_t IN LISTS _strata_mmq_types)list(APPEND _strata_mmq_sources "${_strata_mmq}/template-instances/mmq-instance-${_t}.cu")endforeach()+ if(_strata_mmq_tq_overlay)+ foreach(_t tq2_t tqk6 tqk7)+ list(APPEND _strata_mmq_sources "${_strata_tq_overlay_dir}/template-instances/mmq-instance-${_t}.cu")+ endforeach()+ endif()set_source_files_properties(${_strata_mmq_sources} PROPERTIES LANGUAGE HIP)# moe_fused.cu is Strata's own CUDA-shaped source: it takes the HIP compat headers, ggml's MMQ sources do notset_source_files_properties(src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu@@ -1665,6 +1708,9 @@ if(STRATA_NATIVE_EXPERTS)target_include_directories(strata_mmqPRIVATE "${_strata_gg}/include" "${_strata_gg}/src" "${_strata_mmq}"PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)+ if(_strata_mmq_tq_overlay)+ target_include_directories(strata_mmq BEFORE PRIVATE ${_strata_tq_overlay_dir} ${_strata_tq_overlay_dir}/include)+ endif()target_compile_definitions(strata_mmq PRIVATE GGML_USE_HIP=1)if(STRATA_MMQ_KQUANTS)target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_KQUANTS=1)diff --git a/cmake/ggml_tq_overlay_test.cmake b/cmake/ggml_tq_overlay_test.cmakenew file mode 100644index 000000000..c4e90339c--- /dev/null+++ b/cmake/ggml_tq_overlay_test.cmake@@ -0,0 +1,49 @@+# cmake -DLLAMA=<llama.cpp dir> -DOUT=<scratch dir> -P cmake/ggml_tq_overlay_test.cmake+# The trellis MMQ overlay on a pinned llama.cpp: it generates, every copied header is the pinned one except the patched+# ones, and the patched ones only gain lines (TQ routing, three externs, config copies of IQ2_XS's).+get_filename_component(CMAKE_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)+include(${CMAKE_CURRENT_LIST_DIR}/strata_ggml_tq_overlay.cmake)+file(REMOVE_RECURSE "${OUT}")+strata_ggml_tq_overlay("${LLAMA}" "${OUT}")+if(NOT STRATA_TQ_OVERLAY_OK)+ message(FATAL_ERROR "overlay did not generate on ${LLAMA}")+endif()+set(CU "${LLAMA}/ggml/src/ggml-cuda")+file(GLOB _cuh "${CU}/*.cuh" "${CU}/*.h")+set(_patched mmq.cuh mmq-load-tiles.cuh)+foreach(_f IN LISTS _cuh)+ get_filename_component(_n "${_f}" NAME)+ if(_n MATCHES "^mmq-config-" OR _n IN_LIST _patched)+ continue()+ endif()+ file(READ "${_f}" _a)+ file(READ "${OUT}/${_n}" _b)+ if(NOT _a STREQUAL _b)+ message(FATAL_ERROR "${_n} differs from the pinned file but is not meant to")+ endif()+endforeach()+file(READ "${OUT}/mmq.cuh" _m)+foreach(_pat "case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a"+ "extern DECL_MMQ_CASE(GGML_TYPE_TQK7);")+ string(FIND "${_m}" "${_pat}" _p)+ if(_p LESS 0)+ message(FATAL_ERROR "mmq.cuh lacks: ${_pat}")+ endif()+endforeach()+file(READ "${OUT}/include/ggml.h" _g)+string(FIND "${_g}" "GGML_TYPE_TQK7 = 146," _p)+if(_p LESS 0)+ message(FATAL_ERROR "ggml.h lacks the trellis enumerators")+endif()+foreach(_c IN ITEMS ampere rdna4)+ file(READ "${CU}/mmq-config-${_c}.cuh" _o)+ file(READ "${OUT}/mmq-config-${_c}.cuh" _n2)+ string(REGEX MATCHALL "CASE[(]GGML_TYPE_IQ2_XS," _x "${_o}")+ string(REGEX MATCHALL "CASE[(]GGML_TYPE_TQK6," _y "${_n2}")+ list(LENGTH _x _lx)+ list(LENGTH _y _ly)+ if(NOT _lx EQUAL _ly OR _lx EQUAL 0)+ message(FATAL_ERROR "mmq-config-${_c}.cuh: ${_lx} IQ2_XS cases, ${_ly} TQK6 cases")+ endif()+endforeach()+message(STATUS "ggml_tq_overlay_test: ok")diff --git a/cmake/strata_ggml_tq_overlay.cmake b/cmake/strata_ggml_tq_overlay.cmakenew file mode 100644index 000000000..7324b8a89--- /dev/null+++ b/cmake/strata_ggml_tq_overlay.cmake@@ -0,0 +1,137 @@+# cmake/strata_ggml_tq_overlay.cmake - MMQ prompt kernels for the Gyro trellis types (TQ2_T, TQK6, TQK7) on an UPSTREAM llama.cpp.+#+# The agentionai llama.cpp fork has these types in its ggml (ggml.h enum, ggml-cuda/tq.cuh, a tile loader, MMQ configs and+# routing, three template instances). Upstream does not. This script writes, at configure time, a small OVERLAY of the+# pinned ggml-cuda headers into the build tree: copies of ggml.h, mmq.cuh, mmq-load-tiles.cuh and the mmq-config-*.cuh with+# the trellis types added the way IQ2_XS is handled (same Q8_0_16 tile layout, same dot kernels, same configs - the fork's+# own numbers for them are copies of IQ2_XS's), plus tq.cuh / tq-mmq-tiles.cuh from third_party/ggml-tq and three template+# instances. Only the strata_mmq target sees the overlay (it comes first on its include path); every other type's+# instances are compiled from the untouched pinned files, and the additions are additive (identity gate: Q2_0, IQ3_XXS).+#+# Every edit is anchored on pinned text; if an anchor is missing (another llama.cpp), the script says so and the trellis+# MMQ stays off (STRATA_TQ_OVERLAY_OK=OFF): the engine then reads trellis prompts through the FP16 path, as before.+#+# strata_ggml_tq_overlay(<llama.cpp dir> <out dir>) -> STRATA_TQ_OVERLAY_OK (parent scope)++function(strata_ggml_tq_overlay LLAMA OUT)+ set(STRATA_TQ_OVERLAY_OK OFF PARENT_SCOPE)+ set(_cu "${LLAMA}/ggml/src/ggml-cuda")+ set(_inc "${LLAMA}/ggml/include")+ set(_tq "${CMAKE_SOURCE_DIR}/third_party/ggml-tq")+ foreach(_f "${_inc}/ggml.h" "${_cu}/mmq.cuh" "${_cu}/mmq-load-tiles.cuh" "${_cu}/common.cuh")+ if(NOT EXISTS "${_f}")+ message(STATUS "trellis MMQ overlay: ${_f} is missing: off")+ return()+ endif()+ endforeach()++ # ---- helpers: read a file with LF line ends; insert text before / after the single occurrence of an anchor ----------+ macro(_tq_read VAR PATH)+ file(READ "${PATH}" ${VAR})+ string(REPLACE "\r\n" "\n" ${VAR} "${${VAR}}")+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${PATH}")+ endmacro()+ macro(_tq_edit VAR WHERE ANCHOR TEXT) # WHERE = BEFORE | AFTER; the anchor must occur exactly once+ string(FIND "${${VAR}}" "${ANCHOR}" _p)+ if(_p LESS 0)+ set(_tq_fail "anchor not found: ${ANCHOR}")+ else()+ string(LENGTH "${ANCHOR}" _alen)+ math(EXPR _after "${_p} + ${_alen}")+ string(SUBSTRING "${${VAR}}" ${_after} -1 _tail)+ string(FIND "${_tail}" "${ANCHOR}" _p2)+ if(NOT _p2 LESS 0)+ set(_tq_fail "anchor is not unique: ${ANCHOR}")+ else()+ string(SUBSTRING "${${VAR}}" 0 ${_p} _head)+ if("${WHERE}" STREQUAL "AFTER")+ set(${VAR} "${_head}${ANCHOR}${TEXT}${_tail}")+ else()+ set(${VAR} "${_head}${TEXT}${ANCHOR}${_tail}")+ endif()+ endif()+ endif()+ endmacro()+ set(_tq_fail "")++ file(MAKE_DIRECTORY "${OUT}/include" "${OUT}/template-instances")++ # ---- the whole header set of ggml-cuda and ggml/include is copied: quote includes resolve next to the including file,+ # so a header left in the pinned directory would pull the pinned mmq.cuh / ggml.h back in beside the patched ones -------+ file(GLOB _all_cu "${_cu}/*.cuh" "${_cu}/*.h")+ foreach(_f IN LISTS _all_cu)+ get_filename_component(_n "${_f}" NAME)+ configure_file("${_f}" "${OUT}/${_n}" COPYONLY)+ endforeach()+ configure_file("${_cu}/quantize.cu" "${OUT}/quantize.cu" COPYONLY) # compiled from here: it includes mmq.cuh+ if(IS_DIRECTORY "${_cu}/vendors")+ file(COPY "${_cu}/vendors" DESTINATION "${OUT}")+ endif()+ file(GLOB _all_inc "${_inc}/*.h")+ foreach(_f IN LISTS _all_inc)+ get_filename_component(_n "${_f}" NAME)+ configure_file("${_f}" "${OUT}/include/${_n}" COPYONLY)+ endforeach()++ # ---- ggml.h: the three enumerators (values above GGML_TYPE_COUNT: ggml's own tables never see them) ---------------------+ _tq_read(_h "${_inc}/ggml.h")+ _tq_edit(_h BEFORE " GGML_TYPE_COUNT = 43,"+ " GGML_TYPE_TQ2_T = 144, // Agention trellis types (Strata overlay; the fork's ids)+ GGML_TYPE_TQK6 = 145,+ GGML_TYPE_TQK7 = 146,+")+ file(WRITE "${OUT}/include/ggml.h" "${_h}")++ # ---- mmq.cuh: routing of the trellis types as IQ2_XS ----------------------------------------------------------------+ _tq_read(_m "${_cu}/mmq.cuh")+ _tq_edit(_m AFTER " case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n"+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n")+ _tq_edit(_m AFTER " case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;\n"+ " case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK7: return MMQ_DP4A_TXS_Q8_0_16;\n")+ _tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ3_XXS_Q8_1_MMQ,"+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);\n")+ _tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,"+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback>);\n")+ _tq_edit(_m AFTER "extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n"+ "extern DECL_MMQ_CASE(GGML_TYPE_TQ2_T);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK7);\n")+ file(WRITE "${OUT}/mmq.cuh" "${_m}")++ # ---- mmq-load-tiles.cuh: the loader, at the end -------------------------------------------------------------------+ _tq_read(_l "${_cu}/mmq-load-tiles.cuh")+ file(WRITE "${OUT}/mmq-load-tiles.cuh" "${_l}\n#include \"tq-mmq-tiles.cuh\"\n")+ configure_file("${_tq}/tq.cuh" "${OUT}/tq.cuh" COPYONLY)+ configure_file("${_tq}/tq-mmq-tiles.cuh" "${OUT}/tq-mmq-tiles.cuh" COPYONLY)++ # ---- mmq-config-*.cuh: every IQ2_XS CASE line gets one copy per trellis type ---------------------------------------+ file(GLOB _cfgs "${_cu}/mmq-config-*.cuh")+ foreach(_c ${_cfgs})+ get_filename_component(_cn "${_c}" NAME)+ _tq_read(_t "${_c}")+ string(REPLACE ";" "<SEMI>" _t "${_t}") # the CASE lines hold semicolons; a CMake list would split them+ string(REGEX MATCHALL "[^\n]*CASE\\(GGML_TYPE_IQ2_XS,[^\n]*\n" _lines "${_t}")+ list(REMOVE_DUPLICATES _lines)+ foreach(_ln IN LISTS _lines)+ set(_add "")+ foreach(_ty TQ2_T TQK6 TQK7)+ string(REPLACE "GGML_TYPE_IQ2_XS," "GGML_TYPE_${_ty}," _x "${_ln}")+ string(APPEND _add "${_x}")+ endforeach()+ _tq_edit(_t AFTER "${_ln}" "${_add}")+ endforeach()+ string(REPLACE "<SEMI>" ";" _t "${_t}")+ file(WRITE "${OUT}/${_cn}" "${_t}")+ endforeach()++ # ---- template instances -------------------------------------------------------------------------------------------+ foreach(_ty tq2_t tqk6 tqk7)+ string(TOUPPER "${_ty}" _TY)+ file(WRITE "${OUT}/template-instances/mmq-instance-${_ty}.cu"+ "// Strata trellis overlay (cmake/strata_ggml_tq_overlay.cmake)\n\n#include \"../mmq.cuh\"\n\nDECL_MMQ_CASE(GGML_TYPE_${_TY});\n")+ endforeach()++ if(_tq_fail)+ message(STATUS "trellis MMQ overlay off: ${_tq_fail}")+ return()+ endif()+ set(STRATA_TQ_OVERLAY_OK ON PARENT_SCOPE)+endfunction()diff --git a/docs/GYRO.md b/docs/GYRO.mdindex ca8d890b0..f3eda9737 100644--- a/docs/GYRO.md+++ b/docs/GYRO.md@@ -40,16 +40,14 @@ ready-made engine that is older is replaced by one compiled from this source.gfx1201 with identical output to rc2). **Not supported:** Intel Arc (SYCL) and Apple Metal: the engines there have notrellis kernels; the Metal engine refuses the file at load with the reason. Images are off for these files.-**Prompt speed and the ggml build.** The trellis prompt kernels (MMQ) come from agentionai's llama.cpp fork-(MIT, `STRATA_GGML_DIR`). A build against upstream ggml works and gives the same answers, but reads prompts through the-FP16 path, which is slower. For the fast prompt path compile with the fork's ggml:--```sh-git clone https://github.com/agentionai/llama.cpp ../agention-llama.cpp-cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DSTRATA_ENABLE_CUDA=ON \- -DCMAKE_CUDA_ARCHITECTURES=86 -DSTRATA_GGML_DIR=$PWD/../agention-llama.cpp-# AMD: -DSTRATA_ENABLE_HIP=ON -DSTRATA_PREFILL_MMQ=ON -DCMAKE_HIP_ARCHITECTURES=gfx1201 -DSTRATA_GGML_DIR=...-```+**Prompt speed and the ggml build.** The trellis prompt kernels (MMQ) are agentionai's, from their llama.cpp fork (MIT).+Strata builds against upstream ggml, which does not have them, so it adds them itself at configure time:+`cmake/strata_ggml_tq_overlay.cmake` writes a copy of the pinned ggml-cuda headers into the build folder with the three+trellis types routed like IQ2_XS (the tile loader is the fork's, `third_party/ggml-tq/`); only Strata's own MMQ library sees+the copy, every other type is compiled from the untouched files (identity gate below). The ready-made engines read Gyro+prompts through it as fast as rc2 does, and bit for bit the same. If the pinned ggml changes and an anchor of the script+no longer matches, the overlay switches itself off with a message and prompts take the slower FP16 path. A build against the+fork (`-DSTRATA_GGML_DIR=<agentionai/llama.cpp>`, as rc2 does) uses the fork's own instances instead.## Credit and licenses@@ -147,21 +145,24 @@ Gyro release (2026-10-07). A = agentionai/Strata rc2 9ff02c9, B = this port.| --- | --- | ---: | --- | ---: | ---: || agentionai fork | Gyro-S | 85.4 / 89.7 (1.051) | 5/5 | 5418 / 5313 (0.98) | 14358 / 13859 (0.97) || agentionai fork | Gyro-M | 72.7 / 82.7 (1.136) | 5/5 | 5183 / 5520 (1.07) | 13421 / 14164 (1.06) |-| upstream | Gyro-S | 85.3 / 89.6 (1.050) | 5/5 | 5425 / 6939 (1.28) | 14364 / 21677 (1.51) |+| upstream + overlay (the ready-made engine) | Gyro-S | 85.3 / 89.6 (1.050) | 5/5 | 5413 / 5298 (0.98) | 14372 / 13854 (0.96) |+| upstream + overlay | Gyro-M | 72.6 / 82.6 (1.138) | 5/5 | 5682 / 5524 (0.97) | 15005 / 14160 (0.94) |+| upstream, no overlay (FP16 prompt path) | Gyro-S | 85.3 / 89.6 (1.050) | 5/5 | 5425 / 6939 (1.28) | 14364 / 21677 (1.51) |Decode is faster than rc2's because the port sits on the newer Strata (0.1.45); the same engine change makes IQ3_XXS 7% faster(rc2 86.9, port 93.1 tok/s, 2 pairs). Prompts of the ISTA files gained 11-16% from it too (IQ3_XXS 4K 6184 to 5480 ms, 20K-16587 to 13864 ms); Gyro-M's prompts did not and are 5-7% slower than rc2's, Gyro-S's are 2-3% faster. The cause for Gyro-M is-not isolated (`STRATA_GFX12_DEFAULTS=0` does not give it back). With **upstream ggml** (the ready-made engine) the trellis+16587 to 13864 ms). Gyro-M's first run against rc2 showed its prompts 5-7% slower; in the second rc2's own Gyro-M prompts+took 15.0 s instead of 13.4 s (the 92 GB file's page-cache state), the port's stayed at 14.2 s, and Gyro-M's prompts are 3-6%+faster than rc2's: that difference was rc2's baseline, not a regression. Without the overlay (plain upstream ggml) the trellisprompt kernels are missing and prompts go through the FP16 path: 28% (4K) to 51% (20K) slower than rc2, and a different-rounding (the answer after a 4K prompt can differ). Decode is unaffected. That is why a source build against the fork's ggml is-the fast configuration.+rounding. Decode is unaffected either way.Identity (the fixed gate flags, 10 prompts, exact_pp 4096 and 20000 tokens, same card for both arms):- Q2_0 and IQ3_XXS, f-split against the port (upstream ggml): 10/10 identical, residual dumps and outputs IDENTICAL.-- Gyro-S and Gyro-M, rc2 against the port (fork ggml): 10/10 identical; exact_pp 4096 / 20000 residual dumps and outputs- IDENTICAL when run alone (a run beside other engines on the box once showed a different dump: not reproducible alone).+- Gyro-S and Gyro-M, rc2 against the port with the fork's ggml and with upstream + overlay: 10/10 identical; exact_pp 4096 /+ 20000 residual dumps and outputs IDENTICAL (run alone; a run beside other engines on the box once showed a different dump).+- Q2_0 and IQ3_XXS, f-split against the port with the overlay on, on AMD gfx1201 and on NVIDIA sm_86: IDENTICAL.- The pack step (`tools/iq_pack.py --compat-bf16`) writes byte-identical files to rc2's tools for Gyro-S.## Measured (the authors', rc2)diff --git a/setup.py b/setup.pyindex 4c1c92b62..811921f99 100644--- a/setup.py+++ b/setup.py@@ -6601,10 +6601,6 @@ def main() -> int:ok(f"size: {model}")if MODELS[model].get("license"): # the ready-made packs: each size has its own licensesay(f" Its license: {MODELS[model]['license']}")- if family == "gyro": # a tip, nothing changes (recommend, never force)- say(" Tip: Gyro's prompt kernels come from agentionai's llama.cpp fork. The engine setup installs reads prompts "- "through a slower path (measured 20-30% slower on an AMD R9700; answers speed is the same); compile with "- "-DSTRATA_GGML_DIR=<agentionai/llama.cpp> for the fast one (docs/GYRO.md).")tag = fam["tag"] + model # names of the pack, config and start scriptsmall = min(x["vram_gb"] for x in chosen) # each card keeps its layers' KV of the whole contextrec_ctx = 32768 if small < 14 else 65536 if small < 20 else 131072diff --git a/src/prefill/moe_mmq.cu b/src/prefill/moe_mmq.cuindex 21a9c2e72..5cb56da1e 100644--- a/src/prefill/moe_mmq.cu+++ b/src/prefill/moe_mmq.cu@@ -18,6 +18,16 @@#include <cstring>namespace strata::prefill::mmq {+namespace {+// The Gyro trellis types (ids 144-146) are outside the type table of an upstream ggml (indexed by the id): their block+// geometry is known here (128 weights; 34 / 26 / 30 bytes), everything else asks ggml.+inline bool is_tq(int t) { return t >= 144 && t <= 146; }+inline int64_t type_blck(int t) { return is_tq(t) ? 128 : (int64_t) ggml_blck_size((ggml_type) t); }+inline size_t type_bytes(int t) { return t == 144 ? 34 : t == 145 ? 26 : t == 146 ? 30 : ggml_type_size((ggml_type) t); }+inline const char* type_label(int t) {+ return t == 144 ? "tq2_t" : t == 145 ? "tqk6" : t == 146 ? "tqk7" : ggml_type_name((ggml_type) t);+}+} // namespace#if defined(STRATA_KQ_DSO)namespace kq {int abi_args_size() { return (int) sizeof(mmq_args); }@@ -301,7 +311,7 @@ bool fits(int t, int64_t w_rows) {said[t] = true;std::fprintf(stderr, "strata: prompt kernels: llama.cpp's MMQ has no tile for %s (%lld rows) on GPU %d ""(cc %d, %zu bytes of shared memory per block): that product takes the non-MMQ path "- "(#420)\n", ggml_type_name((ggml_type) t), (long long) w_rows, id, cc, smpbo);+ "(#420)\n", type_label((int) t), (long long) w_rows, id, cc, smpbo);}return false;}@@ -310,7 +320,7 @@ bool fits(int t, int64_t w_rows) {}size_t matrix_bytes(int t, int64_t rows, int64_t cols) {- return (size_t) rows * (size_t) (cols / ggml_blck_size((ggml_type) t)) * ggml_type_size((ggml_type) t);+ return (size_t) rows * (size_t) (cols / type_blck((int) t)) * type_bytes((int) t);}size_t q8_bytes(int64_t rows, int64_t cols) {@@ -394,10 +404,10 @@ Context::~Context() { delete (ggml_backend_cuda_context*) ctx_; }void Context::run(const Product& p, void* stream) {if (p.n <= 0 || p.max_rows <= 0) return;const ggml_type t = (ggml_type) p.type;- const int64_t qk = ggml_blck_size(t), bpr = p.w_cols / qk;+ const int64_t qk = type_blck((int) t), bpr = p.w_cols / qk;const mmq_args a = {(const char*) p.w, t, (const int*) p.xq, p.ids, p.bounds, p.dst, p.y_scale,p.w_cols, p.w_rows, p.total_rows, bpr, p.total_rows, p.ld_dst,- p.n, p.n, p.tab ? 0 : (int64_t) (p.expert_bytes / ggml_type_size(t)), 0, 0,+ p.n, p.n, p.tab ? 0 : (int64_t) (p.expert_bytes / type_bytes((int) t)), 0, 0,1, 1, 0, 0, 0,p.max_rows, p.max_rows};auto& ctx = *(ggml_backend_cuda_context*) ctx_;@@ -411,7 +421,7 @@ void Context::run(const Product& p, void* stream) {mmq_x_tab tab{};for (int z = 0; z < p.n; ++z) tab.p[z] = (const char*) p.tab->p[z];if (!run_tab(&ctx, &a, &tab, s)) {- std::fprintf(stderr, "prefill mmq: no pointer-table MMQ for %s with %lld rows\n", ggml_type_name(t),+ std::fprintf(stderr, "prefill mmq: no pointer-table MMQ for %s with %lld rows\n", type_label((int) t),(long long) p.w_rows);std::exit(1);}@@ -485,7 +495,7 @@ size_t overread_bytes(int t, int64_t cols) {// the K tile's over-read for each (fallback=false) config on any visible GPU, the largest of them: the over-read// is not monotonic in the tile length (20 blocks: 12 more with a 16-block tile, 4 with a 24-block one), so each// config's own; MMQ_ITER_K's is the floor (it is the tile when no config is known)- const int64_t qk = ggml_blck_size((ggml_type) t), bpn = cols / qk;+ const int64_t qk = type_blck((int) t), bpn = cols / qk;auto over = [&](int64_t k_vram) -> int64_t {const int64_t bpi = k_vram / qk;return bpi > 0 ? (bpi - bpn % bpi) % bpi : 0;@@ -497,7 +507,7 @@ size_t overread_bytes(int t, int64_t cols) {const ggml_cuda_mmq_config c = ggml_cuda_mmq_get_config((ggml_type) t, J, false, info.devices[id].cc);if (c.type != GGML_TYPE_COUNT) blocks = std::max<int64_t>(blocks, over(c.K_vram));}- return (size_t) blocks * ggml_type_size((ggml_type) t);+ return (size_t) blocks * type_bytes((int) t);}bool whole_tiles(int t, int64_t w_rows, int n_experts) {diff --git a/third_party/ggml-tq/tq-mmq-tiles.cuh b/third_party/ggml-tq/tq-mmq-tiles.cuhnew file mode 100644index 000000000..14ed32962--- /dev/null+++ b/third_party/ggml-tq/tq-mmq-tiles.cuh@@ -0,0 +1,67 @@+// third_party/ggml-tq/tq-mmq-tiles.cuh - the MMQ tile loader of the trellis types (TQ2_T, TQK6, TQK7), appended to ggml-cuda's+// mmq-load-tiles.cuh by cmake/strata_ggml_tq_overlay.cmake when ggml comes from upstream llama.cpp (the agentionai fork has+// it natively). The function is agentionai/llama.cpp's (ggml/src/ggml-cuda/mmq-load-tiles.cuh, MIT, the ggml authors,+// written by Laurent Zuijdwijk), unchanged; the other lines of this patch only route the trellis types to it the way+// IQ2_XS is routed (same Q8_0_16 tile layout, same dot kernels, same configs).+#pragma once+#include "tq.cuh"++// Trellis types (TQ2_T, TQK6, TQK7): a 128-weight block is TQ_LANES lanes of 16 weights. Each lane+// decodes its 4 trellis steps and requantizes them to int8 with one scale per 16 weights, filling+// the Q8_0_16 tile layout (as IQ2_XS). One MMQ_ITER_K slice of a row is 2 blocks = 16 lanes.+template <ggml_type type, int J, bool fallback> static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_tq(+ const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) {+ // ggml_cuda_mmq_get_util_funcs names this loader for every type; only trellis types reach it.+ constexpr ggml_type tq_type = ggml_cuda_type_is_tq(type) ? type : GGML_TYPE_TQ2_T;+ typedef typename tq_block_info<tq_type>::block_t block_t;+ constexpr int warp_size = ggml_cuda_get_physical_warp_size();+ constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size;+ constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback);+ constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback);++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2);+#else+ constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(type, I);+ int * x_qs = (int *) x_tile;+ float * x_df = (float *) (x_qs + txs.qs);+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)++ static_assert(MMQ_ITER_K % QK_TQK == 0, "bad MMQ_ITER_K");+ constexpr int threads_per_row = (MMQ_ITER_K / QK_TQK) * TQ_LANES; // 16 lanes x 16 weights = 256+ static_assert(warp_size % threads_per_row == 0, "bad warp size");+ constexpr int nrows = warp_size / threads_per_row;+ const int kqsx = threadIdx.x % threads_per_row;+ const int kbx = kqsx / TQ_LANES;+ const int lane = kqsx % TQ_LANES;++#pragma unroll+ for (int i0 = 0; i0 < I; i0 += nwarps * nrows) {+ int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row;++ if (fallback) {+ i = min(i, i_max);+ }++ const block_t * bxi = (const block_t *) x + kbx0 + i*stride + kbx;++ int q[TQ_STEPS_LANE];+ const float dq = tq_lane_to_q8<tq_type>(bxi, lane, q);++#pragma unroll+ for (int k = 0; k < TQ_STEPS_LANE; ++k) {+#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_qs[i*sram_stride + 4*kqsx + k] = q[k];+#else+ x_qs[i*(2*MMQ_TILE_NE_K + 1) + 4*kqsx + k] = q[k];+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }++#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ x_df[i*sram_stride + kqsx] = dq;+#else+ x_df[i*(2*MMQ_TILE_NE_K*2/QI8_0) + i/(QI8_0/4) + kqsx] = dq;+#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE)+ }+}diff --git a/third_party/ggml-tq/tq.cuh b/third_party/ggml-tq/tq.cuhnew file mode 100644index 000000000..9e9e8a262--- /dev/null+++ b/third_party/ggml-tq/tq.cuh@@ -0,0 +1,108 @@+// third_party/ggml-tq/tq.cuh - the trellis expert types (TQ2_T, TQK6, TQK7) for ggml-cuda's MMQ tile loader.+// Derived from agentionai/llama.cpp ggml/src/ggml-cuda/tq.cuh (MIT, the ggml authors; Laurent Zuijdwijk): only what the+// MMQ path needs (no mat-vec kernels), reading Strata's copies of the codebook tables (third_party/ggml/ggml-common-tq.h,+// src/kernels/cuda/tq_lut_i8.cuh) instead of the fork's ggml-common.h.+//+// A 128-weight block is 32 trellis steps of 4 weights. Step t has a 16-bit state s (the codebook index uses its low+// TQ_STATE_BITS bits) and its weights are d * lut[x >> 21].{lo,hi}, d * lut[(x >> 10) & 2047].{lo,hi},+// x = (s & TQ_STATE_MASK) * 0x9e3779b1. A block is split into 8 lanes of 4 consecutive steps (16 weights).+#pragma once++#include "common.cuh"+#include "ggml-common-tq.h"+#include "tq_lut_i8.cuh"++#define TQ_LUT_POINTS 2048+#define TQ_LANES 8 // lanes per 128-weight block+#define TQ_STEPS_LANE 4 // trellis steps per lane++#define QR_TQ 4+#define QI_TQ (QK_TQK/QR_TQ)+static_assert(QK_TQ2_T == QK_TQK, "trellis block sizes differ");++template<>+struct ggml_cuda_type_traits<GGML_TYPE_TQ2_T> {+ static constexpr int qk = QK_TQ2_T;+ static constexpr int qr = QR_TQ;+ static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tq2_t);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_TQK6> {+ static constexpr int qk = QK_TQK;+ static constexpr int qr = QR_TQ;+ static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tqk6);+};++template<>+struct ggml_cuda_type_traits<GGML_TYPE_TQK7> {+ static constexpr int qk = QK_TQK;+ static constexpr int qr = QR_TQ;+ static constexpr int qi = QI_TQ;+ static constexpr int bs = sizeof(block_tqk7);+};++static constexpr __host__ __device__ bool ggml_cuda_type_is_tq(ggml_type type) {+ return type == GGML_TYPE_TQ2_T || type == GGML_TYPE_TQK6 || type == GGML_TYPE_TQK7;+}++template <ggml_type type> struct tq_block_info;+template <> struct tq_block_info<GGML_TYPE_TQ2_T> { typedef block_tq2_t block_t; static constexpr int K = 8; };+template <> struct tq_block_info<GGML_TYPE_TQK6> { typedef block_tqk6 block_t; static constexpr int K = 6; };+template <> struct tq_block_info<GGML_TYPE_TQK7> { typedef block_tqk7 block_t; static constexpr int K = 7; };++// States of steps 4l..4l+3 of one block. qs is the block's qs array, 2-byte aligned+// (every trellis block has an even size and qs sits right after the fp16 scale).+template <ggml_type type>+static __device__ __forceinline__ void tq_states4(const uint8_t * qs, const int l, uint32_t s[TQ_STEPS_LANE]) {+ const uint16_t * qs16 = (const uint16_t *) qs;+ if constexpr (type == GGML_TYPE_TQ2_T) {+ // s_t = qs[(t + 31) % 32] << 8 | qs[t]+ const uint32_t w01 = qs16[2*l + 0];+ const uint32_t w23 = qs16[2*l + 1];+ const uint32_t prev = qs[(4*l + 31) & 31];+ const uint32_t b0 = w01 & 0xFF, b1 = w01 >> 8, b2 = w23 & 0xFF, b3 = w23 >> 8;+ s[0] = (prev << 8) | b0;+ s[1] = (b0 << 8) | b1;+ s[2] = (b1 << 8) | b2;+ s[3] = (b2 << 8) | b3;+ } else {+ // Step t's state is the 16 stream bits from bit (31 - t)*K (circular over 32*K bits, i.e. 2*K u16 words).+ // Steps 4l..4l+3 all lie in the four words from wb = off3 >> 4, off3 = (28 - 4l)*K.+ constexpr int K = tq_block_info<type>::K;+ constexpr int nwords = 2*K;+ const int off3 = (28 - 4*l)*K;+ const int wb = off3 >> 4;+ const int r = off3 & 15;+ const int wi1 = wb + 1 >= nwords ? wb + 1 - nwords : wb + 1;+ const int wi2 = wb + 2 >= nwords ? wb + 2 - nwords : wb + 2;+ const int wi3 = wb + 3 >= nwords ? wb + 3 - nwords : wb + 3;+ const uint64_t v = (uint64_t) qs16[wb] | ((uint64_t) qs16[wi1] << 16) |+ ((uint64_t) qs16[wi2] << 32) | ((uint64_t) qs16[wi3] << 48);+#pragma unroll+ for (int i = 0; i < TQ_STEPS_LANE; ++i) {+ s[i] = (uint32_t) (v >> (r + (3 - i)*K)) & 0xFFFFu;+ }+ }+}++// MMQ (batched) path: lane l of block x as 16 int8 values (4 packed ints, weight order) with one float scale per+// lane, i.e. weights ~= scale * q. The int8 codebook (one global scale) is already int8: no decode to float.+template <ggml_type type>+static __device__ __forceinline__ float tq_lane_to_q8(const void * x, const int l, int q[TQ_STEPS_LANE]) {+ typedef typename tq_block_info<type>::block_t block_t;+ const block_t * b = (const block_t *) x;++ uint32_t s[TQ_STEPS_LANE];+ tq_states4<type>(b->qs, l, s);++#pragma unroll+ for (int i = 0; i < TQ_STEPS_LANE; ++i) {+ const uint32_t xh = (s[i] & TQ_STATE_MASK) * 0x9e3779b1u;+ q[i] = (int) ((uint32_t) tq_lut_i8[xh >> 21] | ((uint32_t) tq_lut_i8[(xh >> 10) & 2047u] << 16));+ }+ // Strata's block structs hold the fp16 scale as uint16_t (the fork's as ggml_half): reinterpret, do not convert+ return __half2float(__ushort_as_half(b->d)) * TQ_LUT_I8_SCALE;+}