LLM inference in C/C++
C++ ggml
metal: fix NaN in mul_mm_id when activations exceed f16 range (#26223)
Michael de Gans0a8b29a60760
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cppindex bf3d07e781..b510cb9571 100644--- a/ggml/src/ggml-metal/ggml-metal-device.cpp+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp@@ -1063,6 +1063,40 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv(ggml_metareturn res;}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib) {+ char base[256];+ char name[256];++ snprintf(base, 256, "kernel_mul_mm_id_amax_part_f32");+ snprintf(name, 256, "%s", base);++ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);+ if (!res.pipeline) {+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);+ }++ res.smem = 32*sizeof(float);++ return res;+}++ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib) {+ char base[256];+ char name[256];++ snprintf(base, 256, "kernel_mul_mm_id_amax_f32");+ snprintf(name, 256, "%s", base);++ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);+ if (!res.pipeline) {+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);+ }++ res.smem = 32*sizeof(float);++ return res;+}+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(ggml_metal_library_t lib, int ne02, int ne20) {char base[256];char name[256];diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.hindex ced33aadfb..f6243ffbd1 100644--- a/ggml/src/ggml-metal/ggml-metal-device.h+++ b/ggml/src/ggml-metal/ggml-metal-device.h@@ -138,6 +138,8 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_exstruct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0 (ggml_metal_library_t lib, int ne02, int ne20);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax(ggml_metal_library_t lib);+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_amax_part(ggml_metal_library_t lib);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id (ggml_metal_library_t lib, const struct ggml_tensor * op);struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op);diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.hindex 7ad21341e4..7a2c65aaa2 100644--- a/ggml/src/ggml-metal/ggml-metal-impl.h+++ b/ggml/src/ggml-metal/ggml-metal-impl.h@@ -14,6 +14,8 @@#define N_MM_SIMD_GROUP_X 2#define N_MM_SIMD_GROUP_Y 2+#define N_MM_NPART_AMAX 256+// kernel parameters for mat-vec threadgroups//// N_R0: number of src0 rows to process per simdgroup@@ -555,6 +557,14 @@ typedef struct {uint64_t nb21;} ggml_metal_kargs_mul_mm_id_map0;+typedef struct {+ int32_t ne00;+ int32_t ne01;+ int32_t ne02;+ uint64_t nb01;+ uint64_t nb02;+} ggml_metal_kargs_mul_mm_id_amax;+typedef struct {int32_t ne00;int32_t ne02;diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cppindex da0040a0cf..cc1bebfaaa 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp@@ -2631,6 +2631,15 @@ size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {return ggml_type_size(GGML_TYPE_I32)*ne02*ne21;}+size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) {+ assert(op->op == GGML_OP_MUL_MAT_ID);++ GGML_UNUSED(op);++ // 2 scaling factors (8 bytes) + N_MM_NPART_AMAX per-threadgroup scales for stage-1+ return 8 + N_MM_NPART_AMAX*sizeof(float);+}+int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {ggml_tensor * op = ctx->node(idx);@@ -2682,6 +2691,36 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {ggml_metal_buffer_id bid_ids = bid_tpe;bid_ids.offs += ggml_metal_op_mul_mat_id_extra_tpe(op);+ ggml_metal_buffer_id bid_amax = bid_ids;+ bid_amax.offs += ggml_metal_op_mul_mat_id_extra_ids(op);++ // src1 rescale factors, computed before the matmul+ // ref: https://github.com/ggml-org/llama.cpp/pull/26223+ {+ ggml_metal_kargs_mul_mm_id_amax args = {+ /*.ne00 =*/ ne10,+ /*.ne01 =*/ ne11,+ /*.ne02 =*/ ne12,+ /*.nb01 =*/ nb11,+ /*.nb02 =*/ nb12,+ };++ auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax_part(lib);++ const size_t smem = pipeline.smem;++ GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);++ ggml_metal_encoder_set_pipeline(enc, pipeline);+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);+ ggml_metal_encoder_set_buffer (enc, bid_src1, 1);+ ggml_metal_encoder_set_buffer (enc, bid_amax, 2);++ ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);++ ggml_metal_encoder_dispatch_threadgroups(enc, N_MM_NPART_AMAX, 1, 1, 256, 1, 1);+ }+{ggml_metal_kargs_mul_mm_id_map0 args = {ne02,@@ -2713,7 +2752,18 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, ne02, 1, 1);}- // this barrier is always needed because the next kernel has to wait for the id maps to be computed+ ggml_metal_op_concurrency_reset(ctx);++ {+ auto pipeline = ggml_metal_library_get_pipeline_mul_mm_id_amax(lib);++ ggml_metal_encoder_set_pipeline(enc, pipeline);+ ggml_metal_encoder_set_buffer (enc, bid_amax, 0);++ ggml_metal_encoder_dispatch_threadgroups(enc, 1, 1, 1, 32, 1, 1);+ }++ // the next kernel has to wait for the amax dataggml_metal_op_concurrency_reset(ctx);{@@ -2745,6 +2795,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {ggml_metal_encoder_set_buffer (enc, bid_tpe, 3);ggml_metal_encoder_set_buffer (enc, bid_ids, 4);ggml_metal_encoder_set_buffer (enc, bid_dst, 5);+ ggml_metal_encoder_set_buffer (enc, bid_amax, 6);const size_t smem = pipeline.smem;diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.hindex 4dd8ce7af6..ae72e8820a 100644--- a/ggml/src/ggml-metal/ggml-metal-ops.h+++ b/ggml/src/ggml-metal/ggml-metal-ops.h@@ -36,6 +36,7 @@ size_t ggml_metal_op_mul_mat_id_extra_tpe(const struct ggml_tensor * op);// id map [n_tokens, n_expert]size_t ggml_metal_op_mul_mat_id_extra_ids(const struct ggml_tensor * op);+size_t ggml_metal_op_mul_mat_id_extra_amax(const struct ggml_tensor * op);// return true if we should use the FA vector kernel for this opbool ggml_metal_op_flash_attn_ext_use_vec(const struct ggml_tensor * op);diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex 4cbec8645a..4f9440f9e6 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -226,6 +226,7 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_{res += ggml_metal_op_mul_mat_id_extra_tpe(tensor);res += ggml_metal_op_mul_mat_id_extra_ids(tensor);+ res += ggml_metal_op_mul_mat_id_extra_amax(tensor);} break;case GGML_OP_FLASH_ATTN_EXT:{diff --git a/ggml/src/ggml-metal/kernels/mul_mm.metal b/ggml/src/ggml-metal/kernels/mul_mm.metalindex 0a45bb1bbe..71d9911491 100644--- a/ggml/src/ggml-metal/kernels/mul_mm.metal+++ b/ggml/src/ggml-metal/kernels/mul_mm.metal@@ -413,6 +413,85 @@ kernel void kernel_mul_mm_id_map0(tpe_u32[ide] = n_all;}+kernel void kernel_mul_mm_id_amax_part_f32(+ constant ggml_metal_kargs_mul_mm_id_amax & args,+ device const char * src1,+ device char * dst,+ threadgroup char * shmem [[threadgroup(0)]],+ uint tgpig[[threadgroup_position_in_grid]],+ ushort tiitg[[thread_index_in_threadgroup]],+ ushort tiisg[[thread_index_in_simdgroup]],+ ushort sgitg[[simdgroup_index_in_threadgroup]],+ ushort ntg[[threads_per_threadgroup]]) {+ const int nrow = args.ne01*args.ne02;++ float lmax = 0.0f;++ for (int ir = tgpig; ir < nrow; ir += N_MM_NPART_AMAX) {+ const int i01 = ir % args.ne01;+ const int i02 = ir / args.ne01;++ device const float * row = (device const float *) (src1 + i02*args.nb02 + i01*args.nb01);++ for (int i00 = tiitg; i00 < args.ne00; i00 += ntg) {+ lmax = max(lmax, fabs(row[i00]));+ }+ }++ float amax = simd_max(lmax);++ threadgroup float * shared_amax = (threadgroup float *) shmem;++ if (ntg > N_SIMDWIDTH) {+ if (sgitg == 0) {+ shared_amax[tiisg] = 0.0f;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ if (tiisg == 0) {+ shared_amax[sgitg] = amax;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ amax = shared_amax[tiisg];+ amax = simd_max(amax);+ }++ if (tiitg == 0) {+ ((device float *) (dst + 8))[tgpig] = amax;+ }+}++kernel void kernel_mul_mm_id_amax_f32(+ device char * dst,+ ushort tiitg[[thread_index_in_threadgroup]]) {+ device const float * part = (device const float *) (dst + 8);++ float amax = 0.0f;++ for (int i = tiitg; i < N_MM_NPART_AMAX; i += N_SIMDWIDTH) {+ amax = max(amax, part[i]);+ }++ amax = simd_max(amax);++ if (tiitg == 0) {+ // leave a comfortable margin below the f16 max of 65504+ float scale = 1.0f;++ // isfinite: src1 already inf/nan is not ours to fix - keep the+ // scale at 1.0 instead of turning it into a different failure+ if (isfinite(amax) && amax > 32768.0f) {+ scale = exp2(ceil(log2(amax)) - 15.0f);+ }++ device float * d = (device float *) dst;++ d[0] = 1.0f/scale; // exact: scale is a power of two+ d[1] = scale;+ }+}+typedef decltype(kernel_mul_mm_id_map0<1>) kernel_mul_mm_id_map0_t;template [[host_name("kernel_mul_mm_id_map0_ne20_1" )]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<1>;@@ -433,6 +512,7 @@ kernel void kernel_mul_mm_id(device const char * htpe,device const char * hids,device char * dst,+ device const char * amax,threadgroup char * shmem [[threadgroup(0)]],uint3 tgpig[[threadgroup_position_in_grid]],ushort tiitg[[thread_index_in_threadgroup]],@@ -503,6 +583,10 @@ kernel void kernel_mul_mm_id(const short lb1 = (short) tiitg/NL1; // 0 .. NR1-1, this thread's row of the B tile+ // power-of-two rescaling+ const float s1_inv = ((device const float *) amax)[0];+ const float s1_scale = ((device const float *) amax)[1];+#ifndef GGML_METAL_HAS_TENSORS0_8x8 ma[4];S1_8x8 mb[2];@@ -586,7 +670,7 @@ kernel void kernel_mul_mm_id(const short ib = 4*sx + sy;- *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;+ *(sb + 64*ib + 8*ly + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0;}} else {const short sx = (tiitg%NL1);@@ -599,7 +683,7 @@ kernel void kernel_mul_mm_id(const short ib = 4*sx + sy;- *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)(*((device T1_2x4 *) y));+ *(threadgroup S1_2x4 *)(sb + 64*ib + 8*ly) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv);}#else// load data and store to threadgroup memory@@ -647,7 +731,7 @@ kernel void kernel_mul_mm_id(//const short lx = (tiitg/NL1)%8;//const short ly = i;- *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) *((device T1 *) y + i) : 0;+ *(sb + NK*(8*sy + ly) + 8*sx + lx) = loop_k + iy + i < args.ne00 ? (S1) (*((device T1 *) y + i) * (T1) s1_inv) : 0;}} else {const short sx = (tiitg%NL1);@@ -658,7 +742,7 @@ kernel void kernel_mul_mm_id(//const short lx = (tiitg/NL1)%8;//const short ly = i;- *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)(*((device T1_2x4 *) y));+ *(threadgroup S1_2x4 *)(sb + NK*(8*sy + ly) + 8*sx) = (S1_2x4)((*((device T1_2x4 *) y)) * (T1) s1_inv);}#endif@@ -749,12 +833,12 @@ kernel void kernel_mul_mm_id(int i = tiisg;for (; i < nr0/4; i += 32) {- *(D4 + i) = *(C4 + i);+ *(D4 + i) = *(C4 + i) * s1_scale;}i = (4*(nr0/4)) + tiisg;for (; i < nr0; i += 32) {- *(D + i) = *(C + i);+ *(D + i) = *(C + i) * s1_scale;}}}diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex bd75e2756d..c7e1d70104 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -5026,9 +5026,13 @@ static void init_mul_mat_id_ids(ggml_context * ctx, int n_mats) {}}-static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) {+static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats, float amax = 1.0f) {for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {- if (t->type != GGML_TYPE_I32) {+ if (t->type == GGML_TYPE_I32) {+ continue;+ } else if (amax != 1.0f && t->type == GGML_TYPE_F32) {+ init_tensor_uniform(t, -amax, amax);+ } else {init_tensor_uniform(t);}}@@ -5045,9 +5049,10 @@ struct test_mul_mat_id : public test_case {const int64_t m;const int64_t n;const int64_t k;+ const float amax; // magnitude of src1std::string vars() override {- return VARS_TO_STR8(type_a, type_b, n_mats, n_used, b, m, n, k);+ return VARS_TO_STR9(type_a, type_b, n_mats, n_used, b, m, n, k, amax);}double max_nmse_err() override {@@ -5069,9 +5074,10 @@ struct test_mul_mat_id : public test_case {test_mul_mat_id(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32,int n_mats = 8, int n_used = 2, bool b = false,- int64_t m = 32, int64_t n = 32, int64_t k = 32)+ int64_t m = 32, int64_t n = 32, int64_t k = 32,+ float amax = 1.0f): type_a(type_a), type_b(type_b), n_mats(n_mats), n_used(n_used), b(b),- m(m), n(n), k(k) {+ m(m), n(n), k(k), amax(amax) {GGML_ASSERT(n_used <= n_mats);}@@ -5097,7 +5103,7 @@ struct test_mul_mat_id : public test_case {}void initialize_tensors(ggml_context * ctx) override {- init_mul_mat_id_tensors(ctx, n_mats);+ init_mul_mat_id_tensors(ctx, n_mats, amax);}void reinit_perf_iter(ggml_context * ctx) override {@@ -10069,6 +10075,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 16, 10, 256));}+ // test src1 f16 overflow+ // TODO: https://github.com/ggml-org/llama.cpp/pull/26223#issuecomment-5585815365+ //for (int n : {16, 32, 64}) {+ // test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 4, false, 4096, n, 2048, 1e5f));+ // test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8, 2, false, 512, n, 256, 1e5f));+ //}+for (ggml_type type_a : base_types) {for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) {for (int n_mats : {4, 8}) {