Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Trellis MMQ overlay follows the ggml-cuda it patches: the generated callbacks and the loader pass the precision argument when mmq.cuh has it (llama.cpp b11xxx), identical output on the pinned commit, off with a message for an unknown signature (#3217)
Niko1221cbc0e329c6fa
diff --git a/cmake/ggml_tq_overlay_test.cmake b/cmake/ggml_tq_overlay_test.cmakeindex c4e90339c..e56604c81 100644--- a/cmake/ggml_tq_overlay_test.cmake+++ b/cmake/ggml_tq_overlay_test.cmake@@ -46,4 +46,26 @@ foreach(_c IN ITEMS ampere rdna4)message(FATAL_ERROR "mmq-config-${_c}.cuh: ${_lx} IQ2_XS cases, ${_ly} TQK6 cases")endif()endforeach()+# #3217: the generated callbacks and the loader follow the helper signatures of the ggml-cuda they patch: with the precision argument+# (llama.cpp b11xxx and newer: prec_src1) they carry it, on the pinned commit (no such argument) they do not+file(READ "${CU}/mmq.cuh" _src)+file(READ "${OUT}/tq-mmq-tiles.cuh" _tiles)+string(FIND "${_src}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1)" _with_prec)+if(_with_prec LESS 0)+ string(FIND "${_m}" "ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);\n case GGML_TYPE_IQ3_XXS" _bad)+ string(FIND "${_tiles}" "GGML_PREC_Q8" _bad2)+ if(NOT _bad2 LESS 0)+ message(FATAL_ERROR "the loader passes a precision argument the pinned ggml-cuda does not take")+ endif()+ string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);" _ok)+else()+ string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);" _ok)+ string(FIND "${_tiles}" "ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8)" _loader_ok)+ if(_loader_ok LESS 0)+ message(FATAL_ERROR "the loader does not pass GGML_PREC_Q8 to ggml_cuda_mmq_get_sram_stride")+ endif()+endif()+if(_ok LESS 0)+ message(FATAL_ERROR "the generated DP4A callback does not match this ggml-cuda's signatures")+endif()message(STATUS "ggml_tq_overlay_test: ok")diff --git a/cmake/strata_ggml_tq_overlay.cmake b/cmake/strata_ggml_tq_overlay.cmakeindex 7324b8a89..df24af259 100644--- a/cmake/strata_ggml_tq_overlay.cmake+++ b/cmake/strata_ggml_tq_overlay.cmake@@ -84,14 +84,28 @@ function(strata_ggml_tq_overlay LLAMA OUT)# ---- mmq.cuh: routing of the trellis types as IQ2_XS ----------------------------------------------------------------_tq_read(_m "${_cu}/mmq.cuh")+ # #3217: llama.cpp from b11xxx on gives the MMQ helpers a ggml_prec argument (template <..., ggml_prec prec_src1> for the write-back and+ # MMA dot callbacks, a fourth parameter for ggml_cuda_mmq_get_nthreads / get_I / get_sram_stride). The pinned commit has neither. The+ # generated routing and the loader follow whichever signature mmq.cuh has; one that is neither turns the overlay off at configure time+ # (the engine then reads trellis prompts through the FP16 path) instead of failing later inside nvcc.+ set(_ps "") # ", prec_src1" in the generated callbacks+ set(_pa "") # ", GGML_PREC_Q8" in the loader's calls (the value upstream's own IQ2_XS loader passes)+ string(FIND "${_m}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1)" _sig_new)+ string(FIND "${_m}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback)" _sig_old)+ if(NOT _sig_new LESS 0)+ set(_ps ", prec_src1")+ set(_pa ", GGML_PREC_Q8")+ elseif(_sig_old LESS 0)+ set(_tq_fail "the MMQ helper signatures of this ggml-cuda are not the ones the overlay knows (ggml_cuda_mmq_get_nthreads)")+ endif()_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XXS:\n case GGML_TYPE_IQ2_XS:\n"" case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n")_tq_edit(_m AFTER " case GGML_TYPE_IQ2_XS: return MMQ_DP4A_TXS_Q8_0_16;\n"" case GGML_TYPE_TQ2_T: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK6: return MMQ_DP4A_TXS_Q8_0_16;\n case GGML_TYPE_TQK7: return MMQ_DP4A_TXS_Q8_0_16;\n")_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ3_XXS_Q8_1_MMQ,"- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);\n")+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n VDR_IQ2_XS_Q8_1_MMQ,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n ggml_cuda_mmq_write_back_dp4a<type, J, fallback${_ps}>);\n")_tq_edit(_m BEFORE " case GGML_TYPE_IQ3_XXS:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_iq3_xxs<type, J, fallback>,"- " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback>);\n")+ " case GGML_TYPE_TQ2_T:\n case GGML_TYPE_TQK6:\n case GGML_TYPE_TQK7:\n return ggml_cuda_mmq_util_funcs(\n -1,\n ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_mma<type, J, fallback${_ps}>,\n ggml_cuda_mmq_write_back_mma<type, J, fallback${_ps}>);\n")_tq_edit(_m AFTER "extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);\n""extern DECL_MMQ_CASE(GGML_TYPE_TQ2_T);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK6);\nextern DECL_MMQ_CASE(GGML_TYPE_TQK7);\n")file(WRITE "${OUT}/mmq.cuh" "${_m}")@@ -100,7 +114,12 @@ function(strata_ggml_tq_overlay LLAMA OUT)_tq_read(_l "${_cu}/mmq-load-tiles.cuh")file(WRITE "${OUT}/mmq-load-tiles.cuh" "${_l}\n#include \"tq-mmq-tiles.cuh\"\n")configure_file("${_tq}/tq.cuh" "${OUT}/tq.cuh" COPYONLY)- configure_file("${_tq}/tq-mmq-tiles.cuh" "${OUT}/tq-mmq-tiles.cuh" COPYONLY)+ file(READ "${_tq}/tq-mmq-tiles.cuh" _tiles)+ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_tq}/tq-mmq-tiles.cuh")+ foreach(_fn get_nthreads get_I get_sram_stride)+ string(REPLACE "ggml_cuda_mmq_${_fn}(type, J, fallback)" "ggml_cuda_mmq_${_fn}(type, J, fallback${_pa})" _tiles "${_tiles}")+ endforeach()+ file(WRITE "${OUT}/tq-mmq-tiles.cuh" "${_tiles}")# ---- mmq-config-*.cuh: every IQ2_XS CASE line gets one copy per trellis type ---------------------------------------file(GLOB _cfgs "${_cu}/mmq-config-*.cuh")