Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
4.3 KB
Raw Blame
# cmake -DLLAMA=<llama.cpp dir> -DOUT=<scratch dir> -P cmake/ggml_tq_overlay_test.cmake
# The trellis MMQ overlay on a pinned llama.cpp: it generates, every copied header is the pinned one except the patched
# ones, and the patched ones only gain lines (TQ routing, three externs, config copies of IQ2_XS's).
# a -P script has no cmake_minimum_required, so CMake 3.x applies the OLD CMP0057 and IN_LIST below is not an operator (#3412)
cmake_policy(SET CMP0057 NEW)
get_filename_component(CMAKE_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
include(${CMAKE_CURRENT_LIST_DIR}/strata_ggml_tq_overlay.cmake)
file(REMOVE_RECURSE "${OUT}")
strata_ggml_tq_overlay("${LLAMA}" "${OUT}")
if(NOT STRATA_TQ_OVERLAY_OK)
  message(FATAL_ERROR "overlay did not generate on ${LLAMA}")
endif()
set(CU "${LLAMA}/ggml/src/ggml-cuda")
file(GLOB _cuh "${CU}/*.cuh" "${CU}/*.h")
set(_patched mmq.cuh mmq-load-tiles.cuh)
foreach(_f IN LISTS _cuh)
  get_filename_component(_n "${_f}" NAME)
  if(_n MATCHES "^mmq-config-" OR _n IN_LIST _patched)
    continue()
  endif()
  file(READ "${_f}" _a)
  file(READ "${OUT}/${_n}" _b)
  if(NOT _a STREQUAL _b)
    message(FATAL_ERROR "${_n} differs from the pinned file but is not meant to")
  endif()
endforeach()
file(READ "${OUT}/mmq.cuh" _m)
foreach(_pat "case GGML_TYPE_TQ2_T:   return MMQ_DP4A_TXS_Q8_0_16;" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n                    ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a"
             "extern DECL_MMQ_CASE(GGML_TYPE_TQK7);" "extern DECL_MMQ_CASE(GGML_TYPE_TQP5);" "extern DECL_MMQ_CASE(GGML_TYPE_TQP7);"
             "case GGML_TYPE_TQP6:    return MMQ_DP4A_TXS_Q8_0_16;")
  string(FIND "${_m}" "${_pat}" _p)
  if(_p LESS 0)
    message(FATAL_ERROR "mmq.cuh lacks: ${_pat}")
  endif()
endforeach()
file(READ "${OUT}/include/ggml.h" _g)
string(FIND "${_g}" "GGML_TYPE_TQK7    = 146," _p)
if(_p LESS 0)
  message(FATAL_ERROR "ggml.h lacks the trellis enumerators")
endif()
string(FIND "${_g}" "GGML_TYPE_TQP7    = 149," _p)   # Gyro-L's pair trellis
if(_p LESS 0)
  message(FATAL_ERROR "ggml.h lacks the pair-trellis enumerators")
endif()
foreach(_t tqp5 tqp6 tqp7)
  if(NOT EXISTS "${OUT}/template-instances/mmq-instance-${_t}.cu")
    message(FATAL_ERROR "the overlay has no MMQ instance for ${_t}")
  endif()
endforeach()
foreach(_c IN ITEMS ampere rdna4)
  file(READ "${CU}/mmq-config-${_c}.cuh" _o)
  file(READ "${OUT}/mmq-config-${_c}.cuh" _n2)
  string(REGEX MATCHALL "CASE[(]GGML_TYPE_IQ2_XS," _x "${_o}")
  string(REGEX MATCHALL "CASE[(]GGML_TYPE_TQK6," _y "${_n2}")
  list(LENGTH _x _lx)
  list(LENGTH _y _ly)
  if(NOT _lx EQUAL _ly OR _lx EQUAL 0)
    message(FATAL_ERROR "mmq-config-${_c}.cuh: ${_lx} IQ2_XS cases, ${_ly} TQK6 cases")
  endif()
endforeach()
# #3217: the generated callbacks and the loader follow the helper signatures of the ggml-cuda they patch: with the precision argument
# (llama.cpp b11xxx and newer: prec_src1) they carry it, on the pinned commit (no such argument) they do not
file(READ "${CU}/mmq.cuh" _src)
file(READ "${OUT}/tq-mmq-tiles.cuh" _tiles)
string(FIND "${_src}" "ggml_cuda_mmq_get_nthreads(ggml_type type, int J, bool fallback, ggml_prec prec_src1)" _with_prec)
if(_with_prec LESS 0)
  string(FIND "${_m}" "ggml_cuda_mmq_write_back_mma<type, J, fallback, prec_src1>);\n        case GGML_TYPE_IQ3_XXS" _bad)
  string(FIND "${_tiles}" "GGML_PREC_Q8" _bad2)
  if(NOT _bad2 LESS 0)
    message(FATAL_ERROR "the loader passes a precision argument the pinned ggml-cuda does not take")
  endif()
  string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n                    ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n                    ggml_cuda_mmq_write_back_dp4a<type, J, fallback>);" _ok)
else()
  string(FIND "${_m}" "ggml_cuda_mmq_load_tiles_tq<type, J, fallback>,\n                    ggml_cuda_mmq_vec_dot_q8_0_16_q8_1_dp4a<type, J, fallback>,\n                    ggml_cuda_mmq_write_back_dp4a<type, J, fallback, prec_src1>);" _ok)
  string(FIND "${_tiles}" "ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8)" _loader_ok)
  if(_loader_ok LESS 0)
    message(FATAL_ERROR "the loader does not pass GGML_PREC_Q8 to ggml_cuda_mmq_get_sram_stride")
  endif()
endif()
if(_ok LESS 0)
  message(FATAL_ERROR "the generated DP4A callback does not match this ggml-cuda's signatures")
endif()
message(STATUS "ggml_tq_overlay_test: ok")