LLM inference in C/C++
C++ ggml
metal : dequantize q8_0 using packed types (#27370)
Georgi Gerganov8ef78e644f55
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metalindex 243c997fc4..c3a95dd6e5 100644--- a/ggml/src/ggml-metal/ggml-metal.metal+++ b/ggml/src/ggml-metal/ggml-metal.metal@@ -656,13 +656,13 @@ void dequantize_q5_1_t4(device const block_q5_1 * xb, short il, thread type4 & rtemplate <typename type4x4>void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {- device const int8_t * qs = ((device const int8_t *)xb->qs);+ device const packed_char4 * qs = (device const packed_char4 *) xb->qs;const float d = xb->d;float4x4 reg_f;- for (int i = 0; i < 16; i++) {- reg_f[i/4][i%4] = (qs[i + 16*il] * d);+ for (int i = 0; i < 4; ++i) {+ reg_f[i] = float4(qs[4*il + i]) * d;}reg = (type4x4) reg_f;@@ -670,12 +670,10 @@ void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & regtemplate <typename type4>void dequantize_q8_0_t4(device const block_q8_0 *xb, short il, thread type4 & reg) {- device const int8_t * qs = ((device const int8_t *)xb->qs);+ device const packed_char4 * qs = (device const packed_char4 *) xb->qs;const float d = xb->d;- for (int i = 0; i < 4; i++) {- reg[i] = (qs[4*(il%4) + i + 16*(il/4)] * d);- }+ reg = (type4) (float4(qs[il]) * d);}template <typename type4x4>