Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Trellis MMQ overlay follows the ggml-cuda it patches: the generated callbacks and the loader pass the precision argument when mmq.cuh has it (llama.cpp b11xxx), identical output on the pinned commit, off with a message for an unknown signature (#3217)

Niko1221cbc0e329c6fa
diff --git a/cmake/ggml_tq_overlay_test.cmake b/cmake/ggml_tq_overlay_test.cmake
index c4e90339c..e56604c81 100644
--- a/cmake/ggml_tq_overlay_test.cmake
+++ b/cmake/ggml_tq_overlay_test.cmake
@@ -46,4 +46,26 @@ foreach(_c IN ITEMS ampere rdna4)
message(FATAL_ERROR "mmq-config-${_c}.cuh: ${_lx} IQ2_XS cases, ${_ly} TQK6 cases")
endif()
endforeach()
+# #3217: the generated callbacks and the loader follow the helper signatures of the ggml-cuda they patch: with the precision argument
+# (llama.cpp b11xxx and newer: prec_src1) they carry it, on the pinned commit (no such argument) they do not
+file(READ "${CU}/mmq.cuh" _src)
+file(READ "${OUT}/tq-mmq-tiles.cuh" _tiles)
+string(FIND "${_src}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1)" _with_prec)
+if(_with_prec LESS 0)
+ string(FIND "${_m}" "ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);\n case GGML_TYPE_IQ3_XXS" _bad)
+ string(FIND "${_tiles}" "GGML_PREC_Q8" _bad2)
+ if(NOT _bad2 LESS 0)
+ message(FATAL_ERROR "the loader passes a precision argument the pinned ggml-cuda does not take")
+ endif()
+ string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);" _ok)
+else()
+ string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);" _ok)
+ string(FIND "${_tiles}" "ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8)" _loader_ok)
+ if(_loader_ok LESS 0)
+ message(FATAL_ERROR "the loader does not pass GGML_PREC_Q8 to ggml_cuda_mmq_get_sram_stride")
+ endif()
+endif()
+if(_ok LESS 0)
+ message(FATAL_ERROR "the generated DP4A callback does not match this ggml-cuda's signatures")
+endif()
message(STATUS "ggml_tq_overlay_test: ok")
diff --git a/cmake/strata_ggml_tq_overlay.cmake b/cmake/strata_ggml_tq_overlay.cmake
index 7324b8a89..df24af259 100644
--- a/cmake/strata_ggml_tq_overlay.cmake
+++ b/cmake/strata_ggml_tq_overlay.cmake
@@ -84,14 +84,28 @@ function(strata_ggml_tq_overlay LLAMA OUT)
# ---- mmq.cuh: routing of the trellis types as IQ2_XS ----------------------------------------------------------------
_tq_read(_m "${_cu}/mmq.cuh")
+ # #3217: llama.cpp from b11xxx on gives the MMQ helpers a ggml_prec argument (template <..., ggml_prec prec_src1> for the write-back and
+ # MMA dot callbacks, a fourth parameter for ggml_cuda_mmq_get_nthreads / get_I / get_sram_stride). The pinned commit has neither. The
+ # generated routing and the loader follow whichever signature mmq.cuh has; one that is neither turns the overlay off at configure time
+ # (the engine then reads trellis prompts through the FP16 path) instead of failing later inside nvcc.
+ set(_ps "") # ", prec_src1" in the generated callbacks
+ set(_pa "") # ", GGML_PREC_Q8" in the loader's calls (the value upstream's own IQ2_XS loader passes)
+ string(FIND "${_m}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1)" _sig_new)
+ string(FIND "${_m}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback)" _sig_old)
+ if(NOT _sig_new LESS 0)
+ set(_ps ", prec_src1")
+ set(_pa ", GGML_PREC_Q8")
+ elseif(_sig_old LESS 0)
+ set(_tq_fail "the MMQ helper signatures of this ggml-cuda are not the ones the overlay knows (ggml_cuda_mmq_get_nthreads)")
+ endif()
_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n"
" case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n")
_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;\n"
" case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK7: return MMQ_DP4A_TXS_Q8_0_16;\n")
_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ3_XXS_Q8_1_MMQ,"
- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);\n")
+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback${_ps}>);\n")
_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,"
- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback>);\n")
+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback${_ps}>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback${_ps}>);\n")
_tq_edit(_m AFTER "extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n"
"extern DECL_MMQ_CASE(GGML_TYPE_TQ2_T);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK7);\n")
file(WRITE "${OUT}/mmq.cuh" "${_m}")
@@ -100,7 +114,12 @@ function(strata_ggml_tq_overlay LLAMA OUT)
_tq_read(_l "${_cu}/mmq-load-tiles.cuh")
file(WRITE "${OUT}/mmq-load-tiles.cuh" "${_l}\n#include \"tq-mmq-tiles.cuh\"\n")
configure_file("${_tq}/tq.cuh" "${OUT}/tq.cuh" COPYONLY)
- configure_file("${_tq}/tq-mmq-tiles.cuh" "${OUT}/tq-mmq-tiles.cuh" COPYONLY)
+ file(READ "${_tq}/tq-mmq-tiles.cuh" _tiles)
+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_tq}/tq-mmq-tiles.cuh")
+ foreach(_fn get_nthreads get_I get_sram_stride)
+ string(REPLACE "ggml_cuda_mmq_${_fn}(type, J, fallback)" "ggml_cuda_mmq_${_fn}(type, J, fallback${_pa})" _tiles "${_tiles}")
+ endforeach()
+ file(WRITE "${OUT}/tq-mmq-tiles.cuh" "${_tiles}")
# ---- mmq-config-*.cuh: every IQ2_XS CASE line gets one copy per trellis type ---------------------------------------
file(GLOB _cfgs "${_cu}/mmq-config-*.cuh")