LLM inference in C/C++
C++ ggml
metal: add F16 input to the FWHT (#29094)
bri-prism9a9f939b8060
diff --git a/ggml/src/ggml-metal/ggml-metal-common.cpp b/ggml/src/ggml-metal/ggml-metal-common.cppindex 388ac4185c..9c0b9474c2 100644--- a/ggml/src/ggml-metal/ggml-metal-common.cpp+++ b/ggml/src/ggml-metal/ggml-metal-common.cpp@@ -7,6 +7,24 @@#include <vector>+// must stay in sync with the kernel_fwht_<type>_<N> templates in misc.metal+static bool ggml_metal_fwht_supported_size(int64_t n) {+ return n == 64 || n == 128 || n == 256 || n == 512;+}++// the FWHT kernels handle a Hadamard-hinted MUL_MAT only under these conditions. supports_op+// and the dispatch must ask the same question: an F16 src1 that is admitted but then falls+// through reaches the generic path, which has no F32 src0 by F16 src1 kernel.+bool ggml_metal_op_mul_mat_use_fwht(const struct ggml_tensor * op) {+ return ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD &&+ op->type == GGML_TYPE_F32 &&+ (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&+ ggml_is_contiguous(op->src[1]) &&+ ggml_is_contiguous(op) &&+ ggml_are_same_shape(op->src[1], op) &&+ ggml_metal_fwht_supported_size(op->src[1]->ne[0]);+}+bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {const int64_t ne00 = op->src[0]->ne[0];const int64_t ne11 = op->src[1]->ne[1];diff --git a/ggml/src/ggml-metal/ggml-metal-common.h b/ggml/src/ggml-metal/ggml-metal-common.hindex 66abdb52ef..e6a28d032d 100644--- a/ggml/src/ggml-metal/ggml-metal-common.h+++ b/ggml/src/ggml-metal/ggml-metal-common.h@@ -48,6 +48,7 @@ bool ggml_mem_ranges_check(ggml_mem_ranges_t mrs, const struct ggml_tensor * tenvoid ggml_graph_optimize(struct ggml_cgraph * gf);// mat-mat vs mat-vec dispatch; used by both supports_op and ggml_metal_op_mul_mat*+bool ggml_metal_op_mul_mat_use_fwht (const struct ggml_tensor * op);bool ggml_metal_op_mul_mat_use_mm (const struct ggml_tensor * op, bool has_simdgroup_mm);bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm);diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex c08ec10b6b..9657e7edb7 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -1472,11 +1472,11 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge(ggreturn res;}-ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n) {+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n, ggml_type tsrc) {char base[256];char name[256];- snprintf(base, 256, "kernel_fwht_f32_%d", n);+ snprintf(base, 256, "kernel_fwht_%s_%d", ggml_type_name(tsrc), n);snprintf(name, 256, "%s", base);ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex 2497e45c34..1bdaecc737 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -145,7 +145,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_idstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op);-struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n, enum ggml_type tsrc);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_radix (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op);diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.mindex 0f42d5700c..9650de2685 100644--- a/ggml/src/ggml-metal/ggml-metal-device.m+++ b/ggml/src/ggml-metal/ggml-metal-device.m@@ -1836,6 +1836,12 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_tecase GGML_OP_SOLVE_TRI:return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32;case GGML_OP_MUL_MAT:+ // the FWHT kernels read an F16 source directly; every other F16 src1 path+ // still goes through ggml_metal_supports_mul_mat_op+ if (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F16 &&+ ggml_metal_op_mul_mat_use_fwht(op)) {+ return has_simdgroup_reduction;+ }return ggml_metal_supports_mul_mat_op(has_simdgroup_reduction, op, true,ggml_metal_op_mul_mat_use_mm(op, has_simdgroup_mm));diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex c86a742367..0323dc3866 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2319,12 +2319,6 @@ int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) {return 1;}-// supported FWHT sizes, must stay in sync with the-// kernel_fwht_f32_<N> templates in ggml-metal.metal-static bool ggml_metal_fwht_supported_size(int64_t n) {- return n == 64 || n == 128 || n == 256 || n == 512;-}-int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) {ggml_tensor * op = ctx->node(idx);@@ -2340,7 +2334,7 @@ int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) {/*.nrows = */ (int32_t) nrows,};- auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n);+ auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n, src1->type);ggml_metal_encoder_set_pipeline(enc, pipeline);ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0);@@ -2426,17 +2420,8 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {ggml_metal_library_t lib = ctx->lib;ggml_metal_encoder_t enc = ctx->enc;- const int32_t hint = ggml_get_op_params_i32(op, 1);-- if (hint == GGML_HINT_SRC0_IS_HADAMARD) {- if (op->src[1]->type == GGML_TYPE_F32 &&- op->type == GGML_TYPE_F32 &&- ggml_is_contiguous(op->src[1]) &&- ggml_is_contiguous(op) &&- ggml_are_same_shape(op->src[1], op) &&- ggml_metal_fwht_supported_size(op->src[1]->ne[0])) {- return ggml_metal_op_fwht(ctx, idx);- }+ if (ggml_metal_op_mul_mat_use_fwht(op)) {+ return ggml_metal_op_fwht(ctx, idx);}const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);diff --git a/ggml/src/ggml-metal/kernels/misc.metal b/ggml/src/ggml-metal/kernels/misc.metalindex 15a18e04ab..877ccf2e15 100644--- a/ggml/src/ggml-metal/kernels/misc.metal+++ b/ggml/src/ggml-metal/kernels/misc.metal@@ -374,10 +374,10 @@ template [[host_name("kernel_snake_f16")]] kernel void kernel_snake<half>(consttemplate [[host_name("kernel_snake_bf16")]] kernel void kernel_snake<bfloat>(constant ggml_metal_kargs_snake &, device const bfloat *, device const float *, device const float *, device bfloat *, uint, uint, uint);#endif-template<int N>-kernel void kernel_fwht_f32(+template<int N, typename src_t>+kernel void kernel_fwht(constant ggml_metal_kargs_fwht & args,- device const float * src,+ device const src_t * src,device float * dst,uint3 tgpig[[threadgroup_position_in_grid]],ushort sgitg[[simdgroup_index_in_threadgroup]],@@ -402,13 +402,13 @@ kernel void kernel_fwht_f32(float reg[NE];for (int i = 0; i < NE; i++) {- reg[i] = src[i*NW + lane]*scale;+ reg[i] = float(src[i*NW + lane])*scale;}for (int i = 1; i < NW; i *= 2) {for (int j = 0; j < NE; j++) {const float val = reg[j];const float val2 = simd_shuffle_xor(val, i);- reg[j] = (lane & i) == 0 ? val2 + val : val2 - val;+ reg[j] = val2 - val + 2*((lane & i) == 0)*val;}}@@ -429,12 +429,18 @@ kernel void kernel_fwht_f32(}}-typedef decltype(kernel_fwht_f32<64>) kernel_fwht_t;+typedef decltype(kernel_fwht<64, float>) kernel_fwht_f32_t;+typedef decltype(kernel_fwht<64, half>) kernel_fwht_f16_t;-template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_t kernel_fwht_f32<64>;-template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f32<128>;-template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>;-template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>;+template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_f32_t kernel_fwht<64, float>;+template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_f32_t kernel_fwht<128, float>;+template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_f32_t kernel_fwht<256, float>;+template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_f32_t kernel_fwht<512, float>;++template [[host_name("kernel_fwht_f16_64")]] kernel kernel_fwht_f16_t kernel_fwht<64, half>;+template [[host_name("kernel_fwht_f16_128")]] kernel kernel_fwht_f16_t kernel_fwht<128, half>;+template [[host_name("kernel_fwht_f16_256")]] kernel kernel_fwht_f16_t kernel_fwht<256, half>;+template [[host_name("kernel_fwht_f16_512")]] kernel kernel_fwht_f16_t kernel_fwht<512, half>;kernel void kernel_dsv4_hc_comb_f32(constant ggml_metal_kargs_dsv4_hc_comb & args,diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex 34f5e4587e..80ca811271 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -9847,6 +9847,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 512, 256)); // many rowstest_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 32, 1, 32)); // too small (N<64)test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1024, 1, 1024)); // too big (N>512)+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 64, 1, 64));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 1, 128));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 1, 256));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 512, 1, 512));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 32, 128));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 128, 4, 128, {2, 3}));+ test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F16, 256, 512, 256)); // many rows#if 0// > 4GB A matrix. Too slow to be enabled by default.