Skip to content

LLM inference in C/C++

C++ ggml

metal : dequantize q8_0 using packed types (#27370)

Georgi Gerganov8ef78e644f55
diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal
index 243c997fc4..c3a95dd6e5 100644
--- a/ggml/src/ggml-metal/ggml-metal.metal
+++ b/ggml/src/ggml-metal/ggml-metal.metal
@@ -656,13 +656,13 @@ void dequantize_q5_1_t4(device const block_q5_1 * xb, short il, thread type4 & r
template <typename type4x4>
void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg) {
- device const int8_t * qs = ((device const int8_t *)xb->qs);
+ device const packed_char4 * qs = (device const packed_char4 *) xb->qs;
const float d = xb->d;
float4x4 reg_f;
- for (int i = 0; i < 16; i++) {
- reg_f[i/4][i%4] = (qs[i + 16*il] * d);
+ for (int i = 0; i < 4; ++i) {
+ reg_f[i] = float4(qs[4*il + i]) * d;
}
reg = (type4x4) reg_f;
@@ -670,12 +670,10 @@ void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & reg
template <typename type4>
void dequantize_q8_0_t4(device const block_q8_0 *xb, short il, thread type4 & reg) {
- device const int8_t * qs = ((device const int8_t *)xb->qs);
+ device const packed_char4 * qs = (device const packed_char4 *) xb->qs;
const float d = xb->d;
- for (int i = 0; i < 4; i++) {
- reg[i] = (qs[4*(il%4) + i + 16*(il/4)] * d);
- }
+ reg = (type4) (float4(qs[il]) * d);
}
template <typename type4x4>