LLM inference in C/C++
C++ ggml
ggml : update ggml_prec specification (#26675)
Georgi Gerganov5a6caa05fc80
diff --git a/ggml/include/ggml.h b/ggml/include/ggml.hindex b88b7e54a5..85a1ae7ae2 100644--- a/ggml/include/ggml.h+++ b/ggml/include/ggml.h@@ -433,10 +433,21 @@ extern "C" {GGML_TYPE_COUNT = 43,};- // precision+ // [TAG_GGML_PREC]+ // this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op+ // the declared types can be:+ // - result accumulation type+ // - source tensor data representation type+ // - etc.+ // the precision parameters are stored as ggml_tensor.op_params to the respective opsenum ggml_prec {- GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default- GGML_PREC_F32 = 10,+ GGML_PREC_UNDEFINED = 0,+ GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED+ GGML_PREC_F32 = 10,+ GGML_PREC_BF16 = 15,+ GGML_PREC_F16 = 20,+ GGML_PREC_Q8 = 30,+ GGML_PREC_Q4 = 40,};// op hint@@ -1429,6 +1440,42 @@ extern "C" {struct ggml_tensor * b,float eps);+ // [TAG_GGML_PREC]+ // set the minimum required accumulator type for the implementation to use during the compute+ // for example:+ // - GGML_PREC_F32 - requires accumulation of the results in F32+ // - GGML_PREC_BF16 - can accumulate the results in BF16, F32+ // - GGML_PREC_F16 - can accumulate the results in F16, F32+ // - GGML_PREC_Q8 - not allowed+ // - GGML_PREC_Q4 - not allowed+ //+ // return false on faliure+ GGML_API bool ggml_prec_set_acc(+ struct ggml_tensor * a,+ enum ggml_prec prec);++ // [TAG_GGML_PREC]+ // set the smallest rank that the implementation can use to internally convert the src[idx] data to+ // ranks in decreasing order:+ // - GGML_PREC_F32 - GGML_TYPE_F32+ // - GGML_PREC_BF16 - GGML_TYPE_BF16+ // - GGML_PREC_F16 - GGML_TYPE_F16,+ // - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.+ // - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.+ //+ // for example:+ // - ggml_prec_set_src(a, GGML_PREC_Q8, 1):+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0+ // - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4+ // - ggml_prec_set_src(a, GGML_PREC_Q4, 1):+ // - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.+ //+ // return false on faliure+ GGML_API bool ggml_prec_set_src(+ struct ggml_tensor * a,+ enum ggml_prec prec,+ int idx);+// A: k columns, n rows => [ne03, ne02, n, k]// B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]// result is n columns, m rows => [ne03 * x, ne02 * y, m, n]@@ -1439,9 +1486,10 @@ extern "C" {// change the precision of a matrix multiplication// set to GGML_PREC_F32 for higher precision (useful for phi-2)- GGML_API void ggml_mul_mat_set_prec(+ GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(struct ggml_tensor * a,- enum ggml_prec prec);+ enum ggml_prec prec),+ "use ggml_prec_set_acc() instead");// change the hint of a matrix multiplicationGGML_API void ggml_mul_mat_set_hint(@@ -2446,9 +2494,10 @@ extern "C" {float max_bias,float logit_softcap);- GGML_API void ggml_flash_attn_ext_set_prec(+ GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(struct ggml_tensor * a,- enum ggml_prec prec);+ enum ggml_prec prec),+ "use ggml_prec_set_acc() instead");GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(const struct ggml_tensor * a);diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.hindex 62b76abbce..ae26e0c23b 100644--- a/ggml/src/ggml-impl.h+++ b/ggml/src/ggml-impl.h@@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_treturn ((const float *)(tensor->op_params))[i];}+// [TAG_GGML_PREC]+// - GGML_OP_MUL_MAT+// 0 - acc+// 1 - hint+// 2 - src0 precision+// 3 - src1 precision+//+// - GGML_OP_MUL_MAT_ID+// 0 - acc+// 1 - hint+// 2 - src0 precision+// 3 - src1 precisionstatic void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));((int32_t *)(tensor->op_params))[i] = value;diff --git a/ggml/src/ggml.c b/ggml/src/ggml.cindex 6257cdbe58..5ef03e190e 100644--- a/ggml/src/ggml.c+++ b/ggml/src/ggml.c@@ -3277,6 +3277,57 @@ struct ggml_tensor * ggml_l2_norm_inplace(return ggml_l2_norm_impl(ctx, a, eps, true);}+// ggml_prec++bool ggml_prec_set_acc(+ struct ggml_tensor * a,+ enum ggml_prec prec) {+ switch (a->op) {+ case GGML_OP_MUL_MAT:+ case GGML_OP_MUL_MAT_ID:+ {+ const int32_t prec_i32 = (int32_t) prec;+ ggml_set_op_params_i32(a, 0, prec_i32);+ }+ break;+ case GGML_OP_FLASH_ATTN_EXT:+ {+ const int32_t prec_i32 = (int32_t) prec;+ ggml_set_op_params_i32(a, 3, prec_i32);+ }+ break;+ default:+ return false;+ };++ return true;+}++bool ggml_prec_set_src(+ struct ggml_tensor * a,+ enum ggml_prec prec,+ int idx) {+ GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC);++ switch (a->op) {+ case GGML_OP_MUL_MAT:+ case GGML_OP_MUL_MAT_ID:+ {+ if (idx != 1) {+ return false;+ }++ const int32_t prec_i32 = (int32_t) prec;+ ggml_set_op_params_i32(a, 2 + idx, prec_i32);+ }+ break;+ default:+ return false;+ };++ return true;+}+// ggml_mul_matstatic inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex 4cbd5fe218..5855393ef7 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -1926,7 +1926,7 @@ ggml_tensor * llm_graph_context::build_ffn(cur = build_lora_mm(down, cur);if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators- ggml_mul_mat_set_prec(cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);}}@@ -2024,7 +2024,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(if (probs_in == nullptr) {logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {- ggml_mul_mat_set_prec(logits, GGML_PREC_F32);+ ggml_prec_set_acc(logits, GGML_PREC_F32);}cb(logits, "ffn_moe_logits", il);} else {@@ -2636,7 +2636,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(ggml_flash_attn_ext_add_sinks(cur, sinks);GGML_ASSERT(n_kv_max >= 0 && n_kv_max <= INT32_MAX);ggml_flash_attn_ext_set_n_kv_max(cur, static_cast<int32_t>(n_kv_max));- ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);if (v_mla) {#if 0@@ -2662,7 +2662,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(// note: this op tends to require high floating point range// while for some models F16 is enough, for others it is not, so we default to F32 here- ggml_mul_mat_set_prec(kq, GGML_PREC_F32);+ ggml_prec_set_acc(kq, GGML_PREC_F32);if (arch == LLM_ARCH_GROK) {// need to do the following:@@ -2895,7 +2895,7 @@ ggml_tensor * llm_graph_context::build_attn(if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulatorscur = build_lora_mm(wo, cur);- ggml_mul_mat_set_prec(cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);if (wo_s) {cur = ggml_mul(ctx0, cur, wo_s);}@@ -2982,7 +2982,7 @@ ggml_tensor * llm_graph_context::build_attn(if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulatorscur = build_lora_mm(wo, cur);- ggml_mul_mat_set_prec(cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);if (wo_s) {cur = ggml_mul(ctx0, cur, wo_s);}diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cppindex 80260a6295..f3b64b210d 100644--- a/src/models/minimax-m3.cpp+++ b/src/models/minimax-m3.cpp@@ -191,7 +191,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa(ggml_tensor * o = ggml_flash_attn_ext(ctx0, q, k, v, mask, kq_scale,hparams.f_max_alibi_bias, 0.0f);- ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32);+ ggml_prec_set_acc(o, GGML_PREC_F32);cb(o, "msa_fattn", il);// [D, Gp, R, C] -> [D, Gp, C, R] -> [n_embd, T]@@ -389,7 +389,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns);ggml_tensor * sc = ggml_mul_mat(ctx0,ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4);- ggml_mul_mat_set_prec(sc, GGML_PREC_F32);+ ggml_prec_set_acc(sc, GGML_PREC_F32);// unmapped positions come out -inf, so they can never rank into the top-ksc = ggml_add_inplace(ctx0, sc,ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns));@@ -471,7 +471,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ggml_tensor * sc = ggml_mul_mat(ctx0, ikp,ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps));// indexer scores run in F32- ggml_mul_mat_set_prec(sc, GGML_PREC_F32);+ ggml_prec_set_acc(sc, GGML_PREC_F32);sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps);// unmapped positions (holes, padding, empty cells) come out -infsc = ggml_add_inplace(ctx0, sc, pm_s);diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cppindex c335e793b9..5121578ff6 100644--- a/tests/test-backend-ops.cpp+++ b/tests/test-backend-ops.cpp@@ -7659,7 +7659,7 @@ struct test_flash_attn_ext : public test_case {ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap);ggml_flash_attn_ext_add_sinks(out, s);ggml_flash_attn_ext_set_n_kv_max(out, n_kv_max);- ggml_flash_attn_ext_set_prec (out, prec);+ ggml_prec_set_acc(out, prec);ggml_set_name(out, "out");return out;diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cppindex 74f4e2b5a4..cd6421def5 100644--- a/tools/mtmd/clip.cpp+++ b/tools/mtmd/clip.cpp@@ -780,7 +780,7 @@ ggml_tensor * clip_graph::build_attn(}cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);- ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);if (sinks != nullptr) {ggml_flash_attn_ext_add_sinks(cur, sinks);}@@ -793,7 +793,7 @@ ggml_tensor * clip_graph::build_attn(ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);// F32 may not needed for vision encoders?- // ggml_mul_mat_set_prec(kq, GGML_PREC_F32);+ // ggml_prec_set_acc(kq, GGML_PREC_F32);kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);if (sinks != nullptr) {diff --git a/tools/mtmd/models/mimovl.cpp b/tools/mtmd/models/mimovl.cppindex 6ff1124a02..e1fbe2671d 100644--- a/tools/mtmd/models/mimovl.cpp+++ b/tools/mtmd/models/mimovl.cpp@@ -2,7 +2,7 @@ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {ggml_tensor * cur = ggml_mul_mat(ctx0, w, x);- ggml_mul_mat_set_prec(cur, GGML_PREC_F32);+ ggml_prec_set_acc(cur, GGML_PREC_F32);return cur;}diff --git a/tools/mtmd/models/qwen3tts-spkenc.cpp b/tools/mtmd/models/qwen3tts-spkenc.cppindex d4659fd63d..405fbb9cbc 100644--- a/tools/mtmd/models/qwen3tts-spkenc.cpp+++ b/tools/mtmd/models/qwen3tts-spkenc.cpp@@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tensggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC);ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out]- ggml_mul_mat_set_prec(y, GGML_PREC_F32);+ ggml_prec_set_acc(y, GGML_PREC_F32);ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1);y = ggml_add(ctx0, y, b2d);diff --git a/tools/tuning/fa-vec.cpp b/tools/tuning/fa-vec.cppindex f904379695..3d6cbeb2c1 100644--- a/tools/tuning/fa-vec.cpp+++ b/tools/tuning/fa-vec.cpp@@ -56,7 +56,7 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {ggml_set_name(m, "m");ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f);- ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32);+ ggml_prec_set_acc(out, GGML_PREC_F32);ggml_set_name(out, "out");return out;