Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add support for bfloat16
Justine Tunney6dbc507c2ef8
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex e68b7287..38ebc5a9 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -1,3 +1,5 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#pragma once#include "ggml.h"@@ -9,7 +11,7 @@#include <stddef.h>#include <stdbool.h>#include <string.h> // memcpy-#include <math.h> // fabsf+#include <math.h> // fabsf, fpclassify#ifdef __cplusplusextern "C" {@@ -260,6 +262,9 @@ size_t ggml_hash_insert ( struct ggml_hash_set hash_set, struct ggml// return index, asserts if table is fullsize_t ggml_hash_find_or_insert( struct ggml_hash_set hash_set, struct ggml_tensor * key);+#define GGML_FP32_TO_BF16(x) ggml_fp32_to_bf16(x)+#define GGML_BF16_TO_FP32(x) ggml_bf16_to_fp32(x)+#ifdef __cplusplus}#endifdiff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex a1b67a66..589c8394 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -262,6 +262,12 @@ static ggml_fp16_t ggml_table_exp_f16[1 << 16];// precomputed f32 table for f16 (256 KB) (ggml-impl.h)float ggml_table_f32_f16[1 << 16];+// precomputed gelu table for bf16 (128 KB)+static ggml_bf16_t ggml_table_gelu_bf16[1 << 16];++// precomputed exp table for bf16 (128 KB)+static ggml_bf16_t ggml_table_exp_bf16[1 << 16];+GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {switch (status) {case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";@@ -289,6 +295,12 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int n) {}}+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int n) {+ for (int i = 0; i < n; i++) {+ y[i] = GGML_BF16_TO_FP32(x[i]);+ }+}+#ifdef __x86_64__#pragma GCC push_options#pragma GCC target("f16c")@@ -322,6 +334,32 @@ void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) {}}+#ifdef __x86_64__+#pragma GCC push_options+#pragma GCC target("avx512bf16")+static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, int n) {+ int i = 0;+ for (; i + 32 <= n; i += 32)+ _mm512_storeu_ps(+ (__m512 *)(y + i),+ (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),+ _mm512_loadu_ps(x + i)));+ for (; i < n; i++)+ y[i] = GGML_FP32_TO_BF16(x[i]);+}+#pragma GCC pop_options+#endif++void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n) {+#ifdef __x86_64__+ if (have_avx512bf16)+ return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);+#endif+ for (int i = 0; i < n; i++) {+ y[i] = GGML_FP32_TO_BF16(x[i]);+ }+}+bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;}@@ -382,6 +420,7 @@ static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {[GGML_TYPE_I8] = {@@ -712,6 +751,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.type_size = sizeof(block_q8_K),.is_quantized = true,.from_float = quantize_row_q8_K,+ },+ [GGML_TYPE_BF16] = {+ .type_name = "bf16",+ .blck_size = 1,+ .type_size = sizeof(ggml_bf16_t),+ .is_quantized = false,+ .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,+ .from_float = (ggml_from_float_t) ggml_fp32_to_bf16_row,+ .from_float_reference = (ggml_from_float_t) ggml_fp32_to_bf16_row,+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,+ .vec_dot_type = GGML_TYPE_BF16,+ .nrows = 1,}};@@ -1361,6 +1412,8 @@ inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) {inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }+inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }@@ -1415,6 +1468,19 @@ static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float **s = sumf;}+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {+ assert(nrc == 1);+ UNUSED(nrc);+ UNUSED(bx);+ UNUSED(by);+ UNUSED(bs);+ ggml_float sumf = 0;+ for (int i = 0; i < n; ++i) {+ sumf += GGML_BF16_TO_FP32(x[i]) * GGML_BF16_TO_FP32(y[i]);+ }+ *s = sumf;+}+#ifdef __x86_64__#pragma GCC push_options#pragma GCC target("avx")@@ -1690,6 +1756,16 @@ inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp}}+inline static void ggml_vec_gelu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {+ for (int i = 0; i < n; ++i) {+ union {+ ggml_bf16_t f;+ uint16_t i;+ } u = {x[i]};+ y[i] = ggml_table_gelu_bf16[u.i];+ }+}+#ifdef GGML_GELU_FP16inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {uint16_t t;@@ -1821,6 +1897,14 @@ inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_*s = sum;}+inline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {+ float sum = 0.0f;+ for (int i = 0; i < n; ++i) {+ sum += GGML_BF16_TO_FP32(x[i]);+ }+ *s = sum;+}+inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {#ifndef GGML_USE_ACCELERATEfloat max = -INFINITY;@@ -2442,6 +2526,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {switch (ftype) {case GGML_FTYPE_ALL_F32: wtype = GGML_TYPE_F32; break;case GGML_FTYPE_MOSTLY_F16: wtype = GGML_TYPE_F16; break;+ case GGML_FTYPE_MOSTLY_BF16: wtype = GGML_TYPE_BF16; break;case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break;case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;@@ -2572,15 +2657,20 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {{const uint64_t t_start = ggml_time_us(); UNUSED(t_start);- ggml_fp16_t ii;for (int i = 0; i < (1 << 16); ++i) {- uint16_t ui = i;- memcpy(&ii, &ui, sizeof(ii));- const float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(ii);+ union {+ uint16_t i;+ ggml_fp16_t fp16;+ ggml_bf16_t bf16;+ } u = {i};+ float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(u.fp16);ggml_table_gelu_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_f32(f));ggml_table_gelu_quick_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_quick_f32(f));ggml_table_silu_f16[i] = GGML_FP32_TO_FP16(ggml_silu_f32(f));- ggml_table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f));+ ggml_table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f));+ f = GGML_BF16_TO_FP32(u.bf16);+ ggml_table_gelu_bf16[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(f));+ ggml_table_exp_bf16[i] = GGML_FP32_TO_BF16(expf(f));}const uint64_t t_end = ggml_time_us(); UNUSED(t_end);@@ -3100,6 +3190,13 @@ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));}} break;+ case GGML_TYPE_BF16:+ {+ assert(tensor->nb[0] == sizeof(ggml_bf16_t));+ for (int i = 0; i < n; i++) {+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));+ }+ } break;case GGML_TYPE_F32:{assert(tensor->nb[0] == sizeof(float));@@ -3167,6 +3264,11 @@ int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);}+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);+ }case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -3209,6 +3311,11 @@ void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -3232,6 +3339,8 @@ int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int ireturn ((int32_t *) data)[0];case GGML_TYPE_F16:return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);+ case GGML_TYPE_BF16:+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);case GGML_TYPE_F32:return ((float *) data)[0];default:@@ -3260,6 +3369,10 @@ void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,{((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{((float *)(data))[0] = value;@@ -3298,6 +3411,11 @@ float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);}+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);+ }case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -3340,6 +3458,11 @@ void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{GGML_ASSERT(tensor->nb[0] == sizeof(float));@@ -3363,6 +3486,8 @@ float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,return ((int32_t *) data)[0];case GGML_TYPE_F16:return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);+ case GGML_TYPE_BF16:+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);case GGML_TYPE_F32:return ((float *) data)[0];default:@@ -3391,6 +3516,10 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,{((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);} break;+ case GGML_TYPE_BF16:+ {+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);+ } break;case GGML_TYPE_F32:{((float *)(data))[0] = value;@@ -3585,7 +3714,11 @@ static struct ggml_tensor * ggml_add_cast_impl(// TODO: support less-strict constraint// GGML_ASSERT(ggml_can_repeat(b, a));GGML_ASSERT(ggml_can_repeat_rows(b, a));- GGML_ASSERT(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16); // currently only supported for quantized input and f16++ // currently only supported for quantized input and f16+ GGML_ASSERT(ggml_is_quantized(a->type) ||+ a->type == GGML_TYPE_F16 ||+ a->type == GGML_TYPE_BF16);bool is_node = false;@@ -7401,6 +7534,24 @@ static void ggml_compute_forward_dup_f32(id += ne00 * (ne01 - ir1);}}+ } else if (dst->type == GGML_TYPE_BF16) {+ size_t id = 0;+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;++ for (int i03 = 0; i03 < ne03; i03++) {+ for (int i02 = 0; i02 < ne02; i02++) {+ id += ne00 * ir0;+ for (int i01 = ir0; i01 < ir1; i01++) {+ for (int i00 = 0; i00 < ne00; i00++) {+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);++ dst_ptr[id] = GGML_FP32_TO_BF16(*src0_ptr);+ id++;+ }+ }+ id += ne00 * (ne01 - ir1);+ }+ }} else {GGML_ASSERT(false); // TODO: implement}@@ -7520,6 +7671,58 @@ static void ggml_compute_forward_dup_f32(}}}+ } else if (dst->type == GGML_TYPE_BF16) {+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ *(ggml_bf16_t *) dst_ptr = GGML_FP32_TO_BF16(*(const float *) src0_ptr);++ if (++i10 == ne0) {+ i10 = 0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }} else {GGML_ASSERT(false); // TODO: implement}@@ -7875,6 +8078,85 @@ static void ggml_compute_forward_add_f16_f32(}}+static void ggml_compute_forward_add_bf16_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_BINARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);++ if (dst->type == GGML_TYPE_F32) {+ GGML_ASSERT( nb0 == sizeof(float));+ }+ else {+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ }++ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (nb10 == sizeof(float)) {+ if (dst->type == GGML_TYPE_BF16) {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);++ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);+ }+ }+ } else {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);++ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i];+ }+ }+ }+ }+ else {+ // src1 is not contiguous+ GGML_ASSERT(false);+ }+}+static void ggml_compute_forward_add_f16_f16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -7931,6 +8213,62 @@ static void ggml_compute_forward_add_f16_f16(}}+static void ggml_compute_forward_add_bf16_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_BINARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);++ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (nb10 == sizeof(ggml_bf16_t)) {+ for (int ir = ir0; ir < ir1; ++ir) {+ // src0, src1 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ ggml_bf16_t * src1_ptr = (ggml_bf16_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);++ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + GGML_BF16_TO_FP32(src1_ptr[i]));+ }+ }+ }+ else {+ // src1 is not contiguous+ GGML_ASSERT(false);+ }+}+static void ggml_compute_forward_add_q_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -8040,6 +8378,18 @@ static void ggml_compute_forward_add(GGML_ASSERT(false);}} break;+ case GGML_TYPE_BF16:+ {+ if (src1->type == GGML_TYPE_BF16) {+ ggml_compute_forward_add_bf16_bf16(params, dst);+ }+ else if (src1->type == GGML_TYPE_F32) {+ ggml_compute_forward_add_bf16_f32(params, dst);+ }+ else {+ GGML_ASSERT(false);+ }+ } break;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -8176,7 +8526,7 @@ static void ggml_compute_forward_add1_f16_f32(}}-static void ggml_compute_forward_add1_f16_f16(+static void ggml_compute_forward_add1_bf16_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -8191,7 +8541,7 @@ static void ggml_compute_forward_add1_f16_f16(}// scalar to add- const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);+ const float v = *(float *) src1->data;const int ith = params->ith;const int nth = params->nth;@@ -8200,12 +8550,64 @@ static void ggml_compute_forward_add1_f16_f16(GGML_TENSOR_UNARY_OP_LOCALS- GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F16);- GGML_ASSERT(dst->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);- GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int ir = ir0; ir < ir1; ++ir) {+ // src0 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);+ }+ }+}++static void ggml_compute_forward_add1_f16_f16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_is_scalar(src1));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // scalar to add+ const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_UNARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F16);+ GGML_ASSERT(dst->type == GGML_TYPE_F16);++ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));// rows per threadconst int dr = (nr + nth - 1)/nth;@@ -8228,6 +8630,58 @@ static void ggml_compute_forward_add1_f16_f16(}}+static void ggml_compute_forward_add1_bf16_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(ggml_are_same_shape(src0, dst));+ GGML_ASSERT(ggml_is_scalar(src1));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // scalar to add+ const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(src0);++ GGML_TENSOR_UNARY_OP_LOCALS++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);++ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int ir = ir0; ir < ir1; ++ir) {+ // src0 and dst are same shape => same indices+ const int i3 = ir/(ne2*ne1);+ const int i2 = (ir - i3*ne2*ne1)/ne1;+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);++ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);+ for (int i = 0; i < ne0; i++) {+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);+ }+ }+}+static void ggml_compute_forward_add1_q_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -8321,6 +8775,18 @@ static void ggml_compute_forward_add1(GGML_ASSERT(false);}} break;+ case GGML_TYPE_BF16:+ {+ if (src1->type == GGML_TYPE_BF16) {+ ggml_compute_forward_add1_bf16_bf16(params, dst);+ }+ else if (src1->type == GGML_TYPE_F32) {+ ggml_compute_forward_add1_bf16_f32(params, dst);+ }+ else {+ GGML_ASSERT(false);+ }+ } break;case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -8448,6 +8914,7 @@ static void ggml_compute_forward_acc(ggml_compute_forward_acc_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -8968,6 +9435,40 @@ static void ggml_compute_forward_sum_f16(((ggml_fp16_t *) dst->data)[0] = GGML_FP32_TO_FP16(sum);}+static void ggml_compute_forward_sum_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ assert(params->ith == 0);+ assert(ggml_is_scalar(dst));++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ assert(src0->nb[0] == sizeof(ggml_bf16_t));++ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)++ float sum = 0;+ float row_sum = 0;++ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ ggml_vec_sum_bf16_ggf(ne00,+ &row_sum,+ (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));+ sum += row_sum;+ }+ }+ }+ ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);+}+static void ggml_compute_forward_sum(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -8983,6 +9484,10 @@ static void ggml_compute_forward_sum({ggml_compute_forward_sum_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_sum_bf16(params, dst);+ } break;default:{GGML_ASSERT(false);@@ -9257,6 +9762,7 @@ static void ggml_compute_forward_repeat(switch (src0->type) {case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_I16:{ggml_compute_forward_repeat_f16(params, dst);@@ -10717,7 +11223,8 @@ static void ggml_compute_forward_mul_mat(#endifif (src1_cont && (src1->type == vec_dot_type ||- src0->type == GGML_TYPE_F16)) {+ src0->type == GGML_TYPE_F16 ||+ src0->type == GGML_TYPE_BF16)) {for (int64_t j = 0; j < ne13; j++)for (int64_t i = 0; i < ne12; i++)if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),@@ -11603,6 +12110,7 @@ static void ggml_compute_forward_set(ggml_compute_forward_set_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -11776,6 +12284,49 @@ static void ggml_compute_forward_get_rows_f16(}}+static void ggml_compute_forward_get_rows_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ GGML_TENSOR_BINARY_OP_LOCALS++ const int64_t nc = ne00;+ const int64_t nr = ggml_nelements(src1);++ assert(ne0 == nc);+ assert(ne02 == ne11);+ assert(nb00 == sizeof(ggml_bf16_t));+ assert(ggml_nrows(dst) == nr);++ const int ith = params->ith;+ const int nth = params->nth;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int64_t i = ir0; i < ir1; ++i) {+ const int64_t i12 = i/(ne11*ne10);+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);++ ggml_bf16_to_fp32_row(+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);+ }+}+static void ggml_compute_forward_get_rows_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -11852,6 +12403,10 @@ static void ggml_compute_forward_get_rows({ggml_compute_forward_get_rows_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_get_rows_bf16(params, dst);+ } break;case GGML_TYPE_F32:case GGML_TYPE_I32:{@@ -11923,6 +12478,45 @@ static void ggml_compute_forward_get_rows_back_f32_f16(}}+static void ggml_compute_forward_get_rows_back_f32_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(params->ith == 0);+ GGML_ASSERT(ggml_is_contiguous(dst));++ // ggml_compute_forward_dup_same_cont(params, opt0, dst);++ if (params->type == GGML_TASK_TYPE_INIT) {+ if (params->ith != 0) {+ return;+ }+ memset(dst->data, 0, ggml_nbytes(dst));+ }++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int nc = src0->ne[0];+ const int nr = ggml_nelements(src1);++ GGML_ASSERT( dst->ne[0] == nc);+ GGML_ASSERT(src0->nb[0] == sizeof(ggml_bf16_t));++ for (int i = 0; i < nr; ++i) {+ const int r = ((int32_t *) src1->data)[i];++ for (int j = 0; j < nc; ++j) {+ ggml_bf16_t v = ((ggml_bf16_t *) ((char *) src0->data + i*src0->nb[1]))[j];+ ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_BF16_TO_FP32(v);+ }+ }+}+static void ggml_compute_forward_get_rows_back_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -11973,6 +12567,10 @@ static void ggml_compute_forward_get_rows_back({ggml_compute_forward_get_rows_back_f32_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_get_rows_back_f32_bf16(params, dst);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_get_rows_back_f32(params, dst);@@ -12514,57 +13112,123 @@ static void ggml_compute_forward_alibi_f16(}}-static void ggml_compute_forward_alibi(+static void ggml_compute_forward_alibi_bf16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];- switch (src0->type) {- case GGML_TYPE_F16:- {- ggml_compute_forward_alibi_f16(params, dst);- } break;- case GGML_TYPE_F32:- {- ggml_compute_forward_alibi_f32(params, dst);- } break;- case GGML_TYPE_Q4_0:- case GGML_TYPE_Q4_1:- case GGML_TYPE_Q5_0:- case GGML_TYPE_Q5_1:- case GGML_TYPE_Q8_0:- case GGML_TYPE_Q8_1:- case GGML_TYPE_Q2_K:- case GGML_TYPE_Q3_K:- case GGML_TYPE_Q4_K:- case GGML_TYPE_Q5_K:- case GGML_TYPE_Q6_K:- case GGML_TYPE_IQ2_XXS:- case GGML_TYPE_IQ2_XS:- case GGML_TYPE_IQ3_XXS:- case GGML_TYPE_IQ1_S:- case GGML_TYPE_IQ4_NL:- case GGML_TYPE_IQ4_XS:- case GGML_TYPE_IQ3_S:- case GGML_TYPE_IQ2_S:- case GGML_TYPE_Q8_K:- case GGML_TYPE_I8:- case GGML_TYPE_I16:- case GGML_TYPE_I32:- case GGML_TYPE_I64:- case GGML_TYPE_F64:- case GGML_TYPE_COUNT:- {- GGML_ASSERT(false);- } break;- }-}+ assert(params->ith == 0);-// ggml_compute_forward_clamp+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }-static void ggml_compute_forward_clamp_f32(- const struct ggml_compute_params * params,+ //const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_head = ((int32_t *) dst->op_params)[1];+ float max_bias;+ memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));++ const int ne0 = src0->ne[0]; // all_seq_len = n_past + ne1+ const int ne1 = src0->ne[1]; // seq_len_without_past+ const int ne2 = src0->ne[2]; // n_head -> this is k+ //const int ne3 = src0->ne[3]; // 1 -> bsz++ const int n = ggml_nrows(src0);+ const int ne2_ne3 = n/ne1; // ne2*ne3++ const int nb0 = src0->nb[0];+ const int nb1 = src0->nb[1];+ const int nb2 = src0->nb[2];+ //const int nb3 = src0->nb[3];++ GGML_ASSERT(nb0 == sizeof(ggml_bf16_t));+ //GGML_ASSERT(ne1 + n_past == ne0); (void) n_past;+ GGML_ASSERT(n_head == ne2);++ // add alibi to src0 (KQ_scaled)+ const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));++ const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);++ for (int k = 0; k < ne2_ne3; k++) {+ // TODO: k*nb2 or k*nb3+ float m_k;++ if (k < n_heads_log2_floor) {+ m_k = powf(m0, k + 1);+ } else {+ m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);+ }++ for (int i = 0; i < ne0; i++) {+ for (int j = 0; j < ne1; j++) {+ ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);+ float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);++ // we return F32+ pdst[0] = i * m_k + GGML_BF16_TO_FP32(src[0]);+ }+ }+ }+}++static void ggml_compute_forward_alibi(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];++ switch (src0->type) {+ case GGML_TYPE_F16:+ {+ ggml_compute_forward_alibi_f16(params, dst);+ } break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_alibi_bf16(params, dst);+ } break;+ case GGML_TYPE_F32:+ {+ ggml_compute_forward_alibi_f32(params, dst);+ } break;+ case GGML_TYPE_Q4_0:+ case GGML_TYPE_Q4_1:+ case GGML_TYPE_Q5_0:+ case GGML_TYPE_Q5_1:+ case GGML_TYPE_Q8_0:+ case GGML_TYPE_Q8_1:+ case GGML_TYPE_Q2_K:+ case GGML_TYPE_Q3_K:+ case GGML_TYPE_Q4_K:+ case GGML_TYPE_Q5_K:+ case GGML_TYPE_Q6_K:+ case GGML_TYPE_IQ2_XXS:+ case GGML_TYPE_IQ2_XS:+ case GGML_TYPE_IQ3_XXS:+ case GGML_TYPE_IQ1_S:+ case GGML_TYPE_IQ4_NL:+ case GGML_TYPE_IQ4_XS:+ case GGML_TYPE_IQ3_S:+ case GGML_TYPE_IQ2_S:+ case GGML_TYPE_Q8_K:+ case GGML_TYPE_I8:+ case GGML_TYPE_I16:+ case GGML_TYPE_I32:+ case GGML_TYPE_I64:+ case GGML_TYPE_F64:+ case GGML_TYPE_COUNT:+ {+ GGML_ASSERT(false);+ } break;+ }+}++// ggml_compute_forward_clamp++static void ggml_compute_forward_clamp_f32(+ const struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];@@ -12617,6 +13281,7 @@ static void ggml_compute_forward_clamp(ggml_compute_forward_clamp_f32(params, dst);} break;case GGML_TYPE_F16:+ case GGML_TYPE_BF16:case GGML_TYPE_Q4_0:case GGML_TYPE_Q4_1:case GGML_TYPE_Q5_0:@@ -13052,6 +13717,173 @@ static void ggml_compute_forward_rope_f16(}}+static void ggml_compute_forward_rope_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst,+ const bool forward) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;++ //const int n_past = ((int32_t *) dst->op_params)[0];+ const int n_dims = ((int32_t *) dst->op_params)[1];+ const int mode = ((int32_t *) dst->op_params)[2];+ const int n_ctx = ((int32_t *) dst->op_params)[3];+ const int n_orig_ctx = ((int32_t *) dst->op_params)[4];+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));++ GGML_TENSOR_UNARY_OP_LOCALS++ //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);+ //printf("n_past = %d, ne2 = %d\n", n_past, ne2);++ GGML_ASSERT(nb0 == sizeof(ggml_bf16_t));++ const int ith = params->ith;+ const int nth = params->nth;++ const int nr = ggml_nrows(dst);++ GGML_ASSERT(n_dims <= ne0);+ GGML_ASSERT(n_dims % 2 == 0);++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ // row index used to determine which thread to use+ int ir = 0;++ const float theta_scale = powf(freq_base, -2.0f/n_dims);+ const float inv_ndims = -1.f/n_dims;+ float corr_dims[2];+ ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);++ const bool is_neox = mode & 2;+ const bool is_glm = mode & 4;++ // backward process uses inverse rotation by cos and sin.+ // cos and sin build a rotation matrix, where the inverse is the transpose.+ // this essentially just switches the sign of sin.+ const float sin_sign = forward ? 1.0f : -1.0f;++ const int32_t * pos = (const int32_t *) src1->data;++ for (int64_t i3 = 0; i3 < ne3; i3++) {+ for (int64_t i2 = 0; i2 < ne2; i2++) {+ const int64_t p = pos[i2];++ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;+ if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox+ ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);+ }++ for (int64_t i1 = 0; i1 < ne1; i1++) {+ if (ir++ < ir0) continue;+ if (ir > ir1) break;++ float theta_base = (float)p;++ if (is_glm) {+ theta_base = MIN(p, n_ctx - 2);+ float block_theta = MAX(p - (n_ctx - 2), 0);+ for (int64_t i0 = 0; i0 < ne0 / 4; i0++) {+ const float cos_theta = cosf(theta_base);+ const float sin_theta = sinf(theta_base) * sin_sign;+ const float cos_block_theta = cosf(block_theta);+ const float sin_block_theta = sinf(block_theta) * sin_sign;++ theta_base *= theta_scale;+ block_theta *= theta_scale;++ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ const float x0 = GGML_BF16_TO_FP32(src[0]);+ const float x1 = GGML_BF16_TO_FP32(src[n_dims/2]);+ const float x2 = GGML_BF16_TO_FP32(src[n_dims]);+ const float x3 = GGML_BF16_TO_FP32(src[n_dims/2*3]);++ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);+ dst_data[n_dims/2] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);+ dst_data[n_dims] = GGML_FP32_TO_BF16(x2*cos_block_theta - x3*sin_block_theta);+ dst_data[n_dims/2*3] = GGML_FP32_TO_BF16(x2*sin_block_theta + x3*cos_block_theta);+ }+ } else if (!is_neox) {+ for (int64_t i0 = 0; i0 < ne0; i0 += 2) {+ const float cos_theta = cache[i0 + 0];+ const float sin_theta = cache[i0 + 1];++ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ const float x0 = GGML_BF16_TO_FP32(src[0]);+ const float x1 = GGML_BF16_TO_FP32(src[1]);++ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);+ dst_data[1] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);+ }+ } else {+ // TODO: this might be wrong for ne0 != n_dims - need double check+ // it seems we have to rope just the first n_dims elements and do nothing with the rest+ // ref: https://github.com/ml-explore/mlx/blob/dc2edc762c797e3b8de50b1dad4dc0a131691033/benchmarks/python/llama_jax_bench.py#L11-L26+ theta_base *= freq_scale;+ for (int64_t ic = 0; ic < ne0; ic += 2) {+ if (ic < n_dims) {+ const int64_t ib = 0;++ // simplified from `(ib * n_dims + ic) * inv_ndims`+ float cur_rot = inv_ndims * ic - ib;++ float cos_theta, sin_theta;+ rope_yarn(+ theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,+ &cos_theta, &sin_theta+ );+ sin_theta *= sin_sign;++ theta_base *= theta_scale;++ const int64_t i0 = ib*n_dims + ic/2;++ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ const float x0 = GGML_BF16_TO_FP32(src[0]);+ const float x1 = GGML_BF16_TO_FP32(src[n_dims/2]);++ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);+ dst_data[n_dims/2] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);+ } else {+ const int64_t i0 = ic;++ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);++ dst_data[0] = src[0];+ dst_data[1] = src[1];+ }+ }+ }+ }+ }+ }+}+static void ggml_compute_forward_rope(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -13063,6 +13895,10 @@ static void ggml_compute_forward_rope({ggml_compute_forward_rope_f16(params, dst, true);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_rope_bf16(params, dst, true);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_rope_f32(params, dst, true);@@ -13087,6 +13923,10 @@ static void ggml_compute_forward_rope_back({ggml_compute_forward_rope_f16(params, dst, false);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_rope_bf16(params, dst, false);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_rope_f32(params, dst, false);@@ -13199,6 +14039,105 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(}}+static void ggml_compute_forward_conv_transpose_1d_bf16_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(src0->type == GGML_TYPE_BF16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_BINARY_OP_LOCALS++ const int ith = params->ith;+ const int nth = params->nth;++ const int nk = ne00*ne01*ne02;++ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nb10 == sizeof(float));++ if (params->type == GGML_TASK_TYPE_INIT) {+ if (ith != 0) {+ return;+ }+ memset(params->wdata, 0, params->wsize);++ // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)+ {+ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + 0;++ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = 0; i01 < ne01; i01++) {+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i02*nb02 + i01*nb01);+ ggml_bf16_t * dst_data = wdata + i01*ne00*ne02;+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ dst_data[i00*ne02 + i02] = src[i00];+ }+ }+ }+ }++ // permute source data (src1) from (L x Cin) to (Cin x L)+ {+ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + nk;+ ggml_bf16_t * dst_data = wdata;++ for (int64_t i11 = 0; i11 < ne11; i11++) {+ const float * const src = (float *)((char *) src1->data + i11*nb11);+ for (int64_t i10 = 0; i10 < ne10; i10++) {+ dst_data[i10*ne11 + i11] = GGML_FP32_TO_BF16(src[i10]);+ }+ }+ }++ // need to zero dst since we are accumulating into it+ memset(dst->data, 0, ggml_nbytes(dst));++ return;+ }++ if (params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];++ // total rows in dst+ const int nr = ne1;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + 0;+ ggml_bf16_t * const wdata_src = wdata + nk;++ for (int i1 = ir0; i1 < ir1; i1++) {+ float * dst_data = (float *)((char *) dst->data + i1*nb1);+ ggml_bf16_t * wdata_kernel = wdata + i1*ne02*ne00;+ for (int i10 = 0; i10 < ne10; i10++) {+ const int i1n = i10*ne11;+ for (int i00 = 0; i00 < ne00; i00++) {+ float v = 0;+ ggml_vec_dot_bf16(ne02, &v, 0,+ (ggml_bf16_t *) wdata_src + i1n, 0,+ (ggml_bf16_t *) wdata_kernel + i00*ne02, 0, 1);+ dst_data[i10*s0 + i00] += v;+ }+ }+ }+}+static void ggml_compute_forward_conv_transpose_1d_f32(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -13309,6 +14248,10 @@ static void ggml_compute_forward_conv_transpose_1d({ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_conv_transpose_1d_bf16_f32(params, dst);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_conv_transpose_1d_f32(params, dst);@@ -13323,7 +14266,95 @@ static void ggml_compute_forward_conv_transpose_1d(// src0: kernel [OC, IC, KH, KW]// src1: image [N, IC, IH, IW]// dst: result [N, OH, OW, IC*KH*KW]-static void ggml_compute_forward_im2col_f32(+static void ggml_compute_forward_im2col_f32(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * src0 = dst->src[0];+ const struct ggml_tensor * src1 = dst->src[1];++ GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F32);++ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_BINARY_OP_LOCALS;++ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t N = is_2D ? ne13 : ne12;+ const int64_t IC = is_2D ? ne12 : ne11;+ const int64_t IH = is_2D ? ne11 : 1;+ const int64_t IW = ne10;++ const int64_t KH = is_2D ? ne01 : 1;+ const int64_t KW = ne00;++ const int64_t OH = is_2D ? ne2 : 1;+ const int64_t OW = ne1;++ int ofs0 = is_2D ? nb13 : nb12;+ int ofs1 = is_2D ? nb12 : nb11;++ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nb10 == sizeof(float));++ if (params->type == GGML_TASK_TYPE_INIT) {+ return;+ }++ if (params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]+ {+ float * const wdata = (float *) dst->data;++ for (int64_t in = 0; in < N; in++) {+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1+ for (int64_t iow = 0; iow < OW; iow++) {+ for (int64_t iic = ith; iic < IC; iic += nth) {++ // micro kernel+ float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]++ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1+ for (int64_t ikw = 0; ikw < KW; ikw++) {+ const int64_t iiw = iow*s0 + ikw*d0 - p0;+ const int64_t iih = ioh*s1 + ikh*d1 - p1;++ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;+ } else {+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);+ }+ }+ }+ }+ }+ }+ }+ }+}+++// src0: kernel [OC, IC, KH, KW]+// src1: image [N, IC, IH, IW]+// dst: result [N, OH, OW, IC*KH*KW]+static void ggml_compute_forward_im2col_f16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -13332,7 +14363,7 @@ static void ggml_compute_forward_im2col_f32(GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F32);+ GGML_ASSERT( dst->type == GGML_TYPE_F16);int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13377,7 +14408,7 @@ static void ggml_compute_forward_im2col_f32(// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]{- float * const wdata = (float *) dst->data;+ ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;for (int64_t in = 0; in < N; in++) {for (int64_t ioh = 0; ioh < OH; ioh++) { // 1@@ -13385,7 +14416,7 @@ static void ggml_compute_forward_im2col_f32(for (int64_t iic = ith; iic < IC; iic += nth) {// micro kernel- float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]+ ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]for (int64_t ikh = 0; ikh < KH; ikh++) { // 1@@ -13396,7 +14427,7 @@ static void ggml_compute_forward_im2col_f32(if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;} else {- dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);}}}@@ -13407,20 +14438,19 @@ static void ggml_compute_forward_im2col_f32(}}-// src0: kernel [OC, IC, KH, KW]// src1: image [N, IC, IH, IW]// dst: result [N, OH, OW, IC*KH*KW]-static void ggml_compute_forward_im2col_f16(+static void ggml_compute_forward_im2col_bf16(const struct ggml_compute_params * params,struct ggml_tensor * dst) {const struct ggml_tensor * src0 = dst->src[0];const struct ggml_tensor * src1 = dst->src[1];- GGML_ASSERT(src0->type == GGML_TYPE_F16);+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);GGML_ASSERT(src1->type == GGML_TYPE_F32);- GGML_ASSERT( dst->type == GGML_TYPE_F16);+ GGML_ASSERT( dst->type == GGML_TYPE_BF16);int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -13452,7 +14482,7 @@ static void ggml_compute_forward_im2col_f16(int ofs0 = is_2D ? nb13 : nb12;int ofs1 = is_2D ? nb12 : nb11;- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));GGML_ASSERT(nb10 == sizeof(float));if (params->type == GGML_TASK_TYPE_INIT) {@@ -13465,7 +14495,7 @@ static void ggml_compute_forward_im2col_f16(// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]{- ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;+ ggml_bf16_t * const wdata = (ggml_bf16_t *) dst->data;for (int64_t in = 0; in < N; in++) {for (int64_t ioh = 0; ioh < OH; ioh++) { // 1@@ -13473,7 +14503,7 @@ static void ggml_compute_forward_im2col_f16(for (int64_t iic = ith; iic < IC; iic += nth) {// micro kernel- ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]+ ggml_bf16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]for (int64_t ikh = 0; ikh < KH; ikh++) { // 1@@ -13482,9 +14512,9 @@ static void ggml_compute_forward_im2col_f16(const int64_t iih = ioh*s1 + ikh*d1 - p1;if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {- dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_BF16(0);} else {- dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_BF16(src_data[iih*IW + iiw]);}}}@@ -13503,6 +14533,10 @@ static void ggml_compute_forward_im2col({ggml_compute_forward_im2col_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_im2col_bf16(params, dst);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_im2col_f32(params, dst);@@ -14229,25 +15263,253 @@ static void ggml_compute_forward_flash_attn_f32(#endif}- for (int64_t ic = 0; ic < nev1; ++ic) {- // dst indices- const int i1 = iq1;- const int i2 = iq2;- const int i3 = iq3;+ for (int64_t ic = 0; ic < nev1; ++ic) {+ // dst indices+ const int i1 = iq1;+ const int i2 = iq2;+ const int i3 = iq3;++ // v indices+ const int iv2 = iq2 % nev2;+ const int iv3 = iq3;++ ggml_vec_dot_f32(masked_begin,+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,+ S, 0, 1);+ }+ }+}++static void ggml_compute_forward_flash_attn_f16(+ const struct ggml_compute_params * params,+ const bool masked,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * q = dst->src[0];+ const struct ggml_tensor * k = dst->src[1];+ const struct ggml_tensor * v = dst->src[2];++ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t D = neq0;+ const int64_t N = neq1;+ const int64_t P = nek1 - N;+ const int64_t M = P + N;++ const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL);++ GGML_ASSERT(ne0 == D);+ GGML_ASSERT(ne1 == N);+ GGML_ASSERT(P >= 0);++ GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));++ GGML_ASSERT(neq0 == D);+ GGML_ASSERT(nek0 == D);+ GGML_ASSERT(nev1 == D);++ GGML_ASSERT(neq1 == N);+ GGML_ASSERT(nek1 == N + P);+ GGML_ASSERT(nev1 == D);++ // dst cannot be transposed or permuted+ GGML_ASSERT(nb0 == sizeof(float));+ GGML_ASSERT(nb0 <= nb1);+ GGML_ASSERT(nb1 <= nb2);+ GGML_ASSERT(nb2 <= nb3);++ if (params->type == GGML_TASK_TYPE_INIT) {+ return;+ }++ if (params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // parallelize by q rows using ggml_vec_dot_f32++ // total rows in q+ const int nr = neq1*neq2*neq3;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ const float scale = 1.0f/sqrtf(D);++ //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);++ for (int ir = ir0; ir < ir1; ++ir) {+ // q indices+ const int iq3 = ir/(neq2*neq1);+ const int iq2 = (ir - iq3*neq2*neq1)/neq1;+ const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);++ float * S = (float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32);++ for (int i = M; i < Mup; ++i) {+ S[i] = -INFINITY;+ }++ if (GGML_VEC_DOT_UNROLL > 2 || nek1 % GGML_VEC_DOT_UNROLL != 0) {+ for (int64_t ic = 0; ic < nek1; ++ic) {+ // k indices+ const int ik3 = iq3;+ const int ik2 = iq2 % nek2;+ const int ik1 = ic;++ // S indices+ const int i1 = ik1;++ ggml_vec_dot_f16(neq0,+ S + i1, 0,+ (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);+ }+ } else {+ for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) {+ // k indices+ const int ik3 = iq3;+ const int ik2 = iq2 % nek2;+ const int ik1 = ic;++ // S indices+ const int i1 = ik1;++ ggml_vec_dot_f16_unroll(neq0, nbk1,+ S + i1,+ ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),+ (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));+ }+ }++ // scale+ ggml_vec_scale_f32(nek1, S, scale);++ if (masked) {+ for (int64_t i = P; i < M; i++) {+ if (i > P + iq1) {+ S[i] = -INFINITY;+ }+ }+ }++ // softmax+ // todo: exclude known -INF S[..] values from max and loop, assuming their results to be zero.+ // dont forget to set their S values to zero+ {+ float max = -INFINITY;+ ggml_vec_max_f32(M, &max, S);++ ggml_float sum = 0.0;+ {+#ifdef GGML_SOFT_MAX_ACCELERATE+ max = -max;+ vDSP_vsadd(S, 1, &max, S, 1, Mup);+ vvexpf(S, S, &Mup);+ ggml_vec_sum_f32(Mup, &sum, S);+#else+ uint16_t scvt[GGML_SOFT_MAX_UNROLL];+ ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };++ for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) {+ float * SS = S + i;++ for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) {+ if (SS[j] == -INFINITY) {+ SS[j] = 0.0f;+ } else {+ ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);+ memcpy(&scvt[j], &s, sizeof(uint16_t));+ const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]);+ sump[j] += (ggml_float)val;+ SS[j] = val;+ }+ }+ }++ for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {+ sum += sump[i];+ }+#endif+ }++ assert(sum > 0.0);++ sum = 1.0/sum;+ ggml_vec_scale_f32(M, S, sum);++#ifndef NDEBUG+ for (int i = 0; i < M; ++i) {+ assert(!isnan(S[i]));+ assert(!isinf(S[i]));+ }+#endif+ }++ ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);++ for (int64_t i = 0; i < M; i++) {+ S16[i] = GGML_FP32_TO_FP16(S[i]);+ }++ // todo: exclude known zero S[..] values from dot (reducing nev0 and increasing begin of v and S16).+ if (GGML_VEC_DOT_UNROLL == 1 || (nev1 % GGML_VEC_DOT_UNROLL != 0)) {+ for (int64_t ic = 0; ic < nev1; ++ic) {+ // dst indices+ const int i1 = iq1;+ const int i2 = iq2;+ const int i3 = iq3;++ // v indices+ const int iv2 = iq2 % nev2;+ const int iv3 = iq3;- // v indices- const int iv2 = iq2 % nev2;- const int iv3 = iq3;+ ggml_vec_dot_f16(nev0,+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,+ S16, 0, 1);+ }+ } else {+ for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) {+ // dst indices+ const int i1 = iq1;+ const int i2 = iq2;+ const int i3 = iq3;- ggml_vec_dot_f32(masked_begin,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,- (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,- S, 0, 1);+ // v indices+ const int iv2 = iq2 % nev2;+ const int iv3 = iq3;++ ggml_vec_dot_f16_unroll(nev0, nbv1,+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),+ ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),+ S16);+ }}}}-static void ggml_compute_forward_flash_attn_f16(+static void ggml_compute_forward_flash_attn_bf16(const struct ggml_compute_params * params,const bool masked,struct ggml_tensor * dst) {@@ -14282,9 +15544,9 @@ static void ggml_compute_forward_flash_attn_f16(GGML_ASSERT(ne1 == N);GGML_ASSERT(P >= 0);- GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t));- GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));- GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));+ GGML_ASSERT(nbq0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nbk0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nbv0 == sizeof(ggml_bf16_t));GGML_ASSERT(neq0 == D);GGML_ASSERT(nek0 == D);@@ -14336,36 +15598,19 @@ static void ggml_compute_forward_flash_attn_f16(S[i] = -INFINITY;}- if (GGML_VEC_DOT_UNROLL > 2 || nek1 % GGML_VEC_DOT_UNROLL != 0) {- for (int64_t ic = 0; ic < nek1; ++ic) {- // k indices- const int ik3 = iq3;- const int ik2 = iq2 % nek2;- const int ik1 = ic;-- // S indices- const int i1 = ik1;-- ggml_vec_dot_f16(neq0,- S + i1, 0,- (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,- (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);- }- } else {- for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) {- // k indices- const int ik3 = iq3;- const int ik2 = iq2 % nek2;- const int ik1 = ic;+ for (int64_t ic = 0; ic < nek1; ++ic) {+ // k indices+ const int ik3 = iq3;+ const int ik2 = iq2 % nek2;+ const int ik1 = ic;- // S indices- const int i1 = ik1;+ // S indices+ const int i1 = ik1;- ggml_vec_dot_f16_unroll(neq0, nbk1,- S + i1,- ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),- (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));- }+ ggml_vec_dot_bf16(neq0,+ S + i1, 0,+ (ggml_bf16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,+ (ggml_bf16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);}// scale@@ -14388,12 +15633,6 @@ static void ggml_compute_forward_flash_attn_f16(ggml_float sum = 0.0;{-#ifdef GGML_SOFT_MAX_ACCELERATE- max = -max;- vDSP_vsadd(S, 1, &max, S, 1, Mup);- vvexpf(S, S, &Mup);- ggml_vec_sum_f32(Mup, &sum, S);-#elseuint16_t scvt[GGML_SOFT_MAX_UNROLL];ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };@@ -14404,9 +15643,9 @@ static void ggml_compute_forward_flash_attn_f16(if (SS[j] == -INFINITY) {SS[j] = 0.0f;} else {- ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);+ ggml_bf16_t s = GGML_FP32_TO_BF16(SS[j] - max);memcpy(&scvt[j], &s, sizeof(uint16_t));- const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]);+ const float val = GGML_BF16_TO_FP32(ggml_table_exp_bf16[scvt[j]]);sump[j] += (ggml_float)val;SS[j] = val;}@@ -14416,7 +15655,6 @@ static void ggml_compute_forward_flash_attn_f16(for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {sum += sump[i];}-#endif}assert(sum > 0.0);@@ -14432,45 +15670,26 @@ static void ggml_compute_forward_flash_attn_f16(#endif}- ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);+ ggml_bf16_t * S16 = (ggml_bf16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);for (int64_t i = 0; i < M; i++) {- S16[i] = GGML_FP32_TO_FP16(S[i]);+ S16[i] = GGML_FP32_TO_BF16(S[i]);}- // todo: exclude known zero S[..] values from dot (reducing nev0 and increasing begin of v and S16).- if (GGML_VEC_DOT_UNROLL == 1 || (nev1 % GGML_VEC_DOT_UNROLL != 0)) {- for (int64_t ic = 0; ic < nev1; ++ic) {- // dst indices- const int i1 = iq1;- const int i2 = iq2;- const int i3 = iq3;-- // v indices- const int iv2 = iq2 % nev2;- const int iv3 = iq3;-- ggml_vec_dot_f16(nev0,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,- (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,- S16, 0, 1);- }- } else {- for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) {- // dst indices- const int i1 = iq1;- const int i2 = iq2;- const int i3 = iq3;+ for (int64_t ic = 0; ic < nev1; ++ic) {+ // dst indices+ const int i1 = iq1;+ const int i2 = iq2;+ const int i3 = iq3;- // v indices- const int iv2 = iq2 % nev2;- const int iv3 = iq3;+ // v indices+ const int iv2 = iq2 % nev2;+ const int iv3 = iq3;- ggml_vec_dot_f16_unroll(nev0, nbv1,- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),- ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),- S16);- }+ ggml_vec_dot_bf16(nev0,+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (ggml_bf16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,+ S16, 0, 1);}}}@@ -14487,6 +15706,10 @@ static void ggml_compute_forward_flash_attn({ggml_compute_forward_flash_attn_f16(params, masked, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_flash_attn_bf16(params, masked, dst);+ } break;case GGML_TYPE_F32:{ggml_compute_forward_flash_attn_f32(params, masked, dst);@@ -14635,6 +15858,141 @@ static void ggml_compute_forward_flash_ff_f16(}}+static void ggml_compute_forward_flash_ff_bf16(+ const struct ggml_compute_params * params,+ struct ggml_tensor * dst) {++ const struct ggml_tensor * a = dst->src[0]; // BF16+ const struct ggml_tensor * b0 = dst->src[1]; // BF16 fc_w+ const struct ggml_tensor * b1 = dst->src[2]; // F32 fc_b+ const struct ggml_tensor * c0 = dst->src[3]; // BF16 proj_w+ const struct ggml_tensor * c1 = dst->src[4]; // F32 proj_b++ int64_t t0 = ggml_perf_time_us();+ UNUSED(t0);++ GGML_TENSOR_LOCALS(int64_t, nea, a, ne)+ GGML_TENSOR_LOCALS(size_t, nba, a, nb)+ GGML_TENSOR_LOCALS(int64_t, neb0, b0, ne)+ GGML_TENSOR_LOCALS(size_t, nbb0, b0, nb)+ GGML_TENSOR_LOCALS(int64_t, neb1, b1, ne)+ GGML_TENSOR_LOCALS(size_t, nbb1, b1, nb)+ GGML_TENSOR_LOCALS(int64_t, nec0, c0, ne)+ GGML_TENSOR_LOCALS(size_t, nbc0, c0, nb)+ GGML_TENSOR_LOCALS(int64_t, nec1, c1, ne)+ GGML_TENSOR_LOCALS(size_t, nbc1, c1, nb)+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)++ const int ith = params->ith;+ const int nth = params->nth;++ const int64_t D = nea0;+ //const int64_t N = nea1;+ const int64_t M = neb01;++ GGML_ASSERT(ne0 == nea0);+ GGML_ASSERT(ne1 == nea1);+ GGML_ASSERT(ne2 == nea2);++ GGML_ASSERT(nba0 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nbb00 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nbb10 == sizeof(float));+ GGML_ASSERT(nbc00 == sizeof(ggml_bf16_t));+ GGML_ASSERT(nbc10 == sizeof(float));++ GGML_ASSERT(neb00 == D);+ GGML_ASSERT(neb01 == M);+ GGML_ASSERT(neb10 == M);+ GGML_ASSERT(neb11 == 1);++ GGML_ASSERT(nec00 == M);+ GGML_ASSERT(nec01 == D);+ GGML_ASSERT(nec10 == D);+ GGML_ASSERT(nec11 == 1);++ // dst cannot be transposed or permuted+ GGML_ASSERT(nb0 == sizeof(float));+ GGML_ASSERT(nb0 <= nb1);+ GGML_ASSERT(nb1 <= nb2);+ GGML_ASSERT(nb2 <= nb3);++ if (params->type == GGML_TASK_TYPE_INIT) {+ return;+ }++ if (params->type == GGML_TASK_TYPE_FINALIZE) {+ return;+ }++ // parallelize by a rows using ggml_vec_dot_f32++ // total rows in a+ const int nr = nea1*nea2*nea3;++ // rows per thread+ const int dr = (nr + nth - 1)/nth;++ // row range for this thread+ const int ir0 = dr*ith;+ const int ir1 = MIN(ir0 + dr, nr);++ for (int ir = ir0; ir < ir1; ++ir) {+ // a indices+ const int ia3 = ir/(nea2*nea1);+ const int ia2 = (ir - ia3*nea2*nea1)/nea1;+ const int ia1 = (ir - ia3*nea2*nea1 - ia2*nea1);++ float * S = (float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32);++ for (int64_t ic = 0; ic < neb01; ++ic) {+ // b0 indices+ const int ib03 = ia3;+ const int ib02 = ia2;+ const int ib01 = ic;++ // S indices+ const int i1 = ib01;++ ggml_vec_dot_bf16(nea0,+ S + i1, 0,+ (ggml_bf16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), 0,+ (ggml_bf16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)), 0, 1);+ }++ ggml_vec_add_f32(neb01, S, S, (float *) b1->data);+ //ggml_vec_gelu_f32(neb01, S, S);++ ggml_bf16_t * S16 = (ggml_bf16_t *) ((float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32) + M);++ for (int64_t i = 0; i < M; i++) {+ S16[i] = GGML_FP32_TO_BF16(S[i]);+ }++ ggml_vec_gelu_bf16(neb01, S16, S16);++ {+ // dst indices+ const int i1 = ia1;+ const int i2 = ia2;+ const int i3 = ia3;++ for (int64_t ic = 0; ic < nec01; ++ic) {++ ggml_vec_dot_bf16(neb01,+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,+ (ggml_bf16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), 0,+ S16, 0, 1);+ }++ ggml_vec_add_f32(nec01,+ (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),+ (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),+ (float *) c1->data);+ }+ }+}+static void ggml_compute_forward_flash_ff(const struct ggml_compute_params * params,struct ggml_tensor * dst) {@@ -14646,6 +16004,10 @@ static void ggml_compute_forward_flash_ff({ggml_compute_forward_flash_ff_f16(params, dst);} break;+ case GGML_TYPE_BF16:+ {+ ggml_compute_forward_flash_ff_bf16(params, dst);+ } break;case GGML_TYPE_F32:{GGML_ASSERT(false); // TODO@@ -15517,6 +16879,7 @@ static void ggml_compute_forward_get_rel_pos(switch (src0->type) {case GGML_TYPE_F16:+ case GGML_TYPE_BF16:{ggml_compute_forward_get_rel_pos_f16(params, dst);} break;@@ -18468,7 +19831,8 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threaconst int64_t ne10 = node->src[1]->ne[0]; // Lconst int64_t ne11 = node->src[1]->ne[1]; // Cin- if (node->src[0]->type == GGML_TYPE_F16 &&+ if ((node->src[0]->type == GGML_TYPE_F16 ||+ node->src[0]->type == GGML_TYPE_BF16) &&node->src[1]->type == GGML_TYPE_F32) {cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;cur += sizeof(ggml_fp16_t)*ne10*ne11;@@ -18504,6 +19868,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa} else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2+ } else if (node->src[1]->type == GGML_TYPE_BF16) {+ cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)+ cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2}} break;case GGML_OP_FLASH_FF:@@ -18514,6 +19881,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa} else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2+ } else if (node->src[1]->type == GGML_TYPE_BF16) {+ cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)+ cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2}} break;case GGML_OP_FLASH_ATTN_BACK:@@ -18527,6 +19897,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa} else if (node->src[1]->type == GGML_TYPE_F16) {cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2+ } else if (node->src[1]->type == GGML_TYPE_BF16) {+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2}} break;@@ -19318,7 +20691,9 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraphif (node->type == GGML_TYPE_I8 || node->type == GGML_TYPE_I16 || node->type == GGML_TYPE_I32) {fprintf(fp, "%d", ggml_get_i32_1d(node, j));}- else if (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16) {+ else if (node->type == GGML_TYPE_F32 ||+ node->type == GGML_TYPE_F16 ||+ node->type == GGML_TYPE_BF16) {fprintf(fp, "%.1e", (double)ggml_get_f32_1d(node, j));}else {@@ -20373,6 +21748,12 @@ size_t ggml_quantize_chunk(ggml_fp32_to_fp16_row(src + start, (ggml_fp16_t *)dst + start, n);result = n * elemsize;} break;+ case GGML_TYPE_BF16:+ {+ size_t elemsize = sizeof(ggml_bf16_t);+ ggml_fp32_to_bf16_row(src + start, (ggml_bf16_t *)dst + start, n);+ result = n * elemsize;+ } break;case GGML_TYPE_F32:{size_t elemsize = sizeof(float);diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 8977645f..99d61411 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -371,6 +371,7 @@ extern "C" {GGML_TYPE_I32 = 26,GGML_TYPE_I64 = 27,GGML_TYPE_F64 = 28,+ GGML_TYPE_BF16 = 29,GGML_TYPE_COUNT,};@@ -410,6 +411,7 @@ extern "C" {GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensorsGGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors+ GGML_FTYPE_MOSTLY_BF16 = 23, // except 1d tensors};// available tensor operations:@@ -2364,6 +2366,90 @@ extern "C" {GGML_API int ggml_cpu_has_vsx (void);GGML_API int ggml_cpu_has_matmul_int8(void);+ /**+ * Google Brain 16-bit floating point number.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───┐+ * 0b0000000000000000 bfloat16+ *+ * Since bf16 has the same number of exponent bits as a 32bit float,+ * encoding and decoding numbers becomes relatively straightforward.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌──┴───┐┌─┴───────────────────┐+ * 0b00000000000000000000000000000000 IEEE binary32+ *+ * For comparison, the standard fp16 format has fewer exponent bits.+ *+ * ┌sign+ * │+ * │ ┌exponent+ * │ │+ * │ │ ┌mantissa+ * │ │ │+ * │┌─┴─┐┌─┴──────┐+ * 0b0000000000000000 IEEE binary16+ *+ * So be warned that converting between them yields a ten bit float.+ *+ * @see IEEE 754-2008+ */+ typedef struct {+ uint16_t x;+ } ggml_bf16_t;++ /**+ * Converts bfloat16 to float32.+ */+ static inline float ggml_bf16_to_fp32(ggml_bf16_t h) {+ union {+ float f;+ uint32_t i;+ } u;+ u.i = (uint32_t)h.x << 16;+ return u.f;+ }++ /**+ * Converts float32 to bfloat16.+ *+ * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.+ * Subnormals shall be flushed to zero, and NANs will be quiet.+ * This code should vectorize nicely if using modern compilers.+ */+ static inline ggml_bf16_t ggml_fp32_to_bf16(float s) {+ ggml_bf16_t h;+ union {+ float f;+ uint32_t i;+ } u;+ u.f = s;+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */+ h.x = (u.i >> 16) | 64; /* force to quiet */+ return h;+ }+ if (!(u.i & 0x7f800000)) { /* subnormal */+ h.x = (u.i & 0x80000000) >> 16; /* flush to zero */+ return h;+ }+ h.x = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;+ return h;+ }++ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int n);+ GGML_API void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n);+//// Internal types and functions exposed for tests and benchmarks//diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex cefe2f2a..1d2cf233 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -2683,6 +2683,7 @@ struct llama_model_loader {switch (type_max) {case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;+ case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;@@ -3034,6 +3035,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {switch (ftype) {case LLAMA_FTYPE_ALL_F32: return "all F32";case LLAMA_FTYPE_MOSTLY_F16: return "F16";+ case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:@@ -4961,6 +4963,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam|| !(model.ftype == LLAMA_FTYPE_ALL_F32 ||model.ftype == LLAMA_FTYPE_MOSTLY_F16 ||+ model.ftype == LLAMA_FTYPE_MOSTLY_BF16 ||model.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||model.ftype == LLAMA_FTYPE_MOSTLY_Q4_1)@@ -12063,6 +12066,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const scase LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break;case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break;case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break;+ case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break;case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break;case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break;diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 9d6f458d..230f820b 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -119,6 +119,7 @@ extern "C" {LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensorsLLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensorsLLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors+ LLAMA_FTYPE_MOSTLY_BF16 = 31, // except 1d tensorsLLAMA_FTYPE_GUESSED = 1024, // not specified in the model file};diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1index 7efd9ea9..e92f10da 100644--- a/llama.cpp/quantize/quantize.1+++ b/llama.cpp/quantize/quantize.1@@ -76,6 +76,8 @@ The following quantization types are available:.It7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B.It+ 31 BF16 Google Brain Floating Point+.It1 F16 13.00G @ 7B.It0 F32 26.00G @ 7Bdiff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex 76ed165b..940cd991 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -51,6 +51,7 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {{ "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 4.45G, +0.0122 ppl @ LLaMA-v1-7B", },{ "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 5.15G, +0.0008 ppl @ LLaMA-v1-7B", },{ "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 6.70G, +0.0004 ppl @ LLaMA-v1-7B", },+ { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "Google Brain Floating Point", },{ "F16", LLAMA_FTYPE_MOSTLY_F16, "13.00G @ 7B", },{ "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },// Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 6934e553..a9e95b51 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -45,6 +45,9 @@ o/$(MODE)/llamafile: \o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavxo/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_bss_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma+o/$(MODE)/llamafile/sgemm_bss_avx512.o: private TARGET_ARCH += -Xx86_64-mavx512f+o/$(MODE)/llamafile/sgemm_bss_avx512bf16.o: private TARGET_ARCH += -Xx86_64-mavx512bf16 -Xx86_64-mavx512vlo/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512fo/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16co/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512fdiff --git a/llamafile/float.h b/llamafile/float.hindex 76a52b6d..82aaa35e 100644--- a/llamafile/float.h+++ b/llamafile/float.h@@ -12,6 +12,14 @@ inline unsigned toint(float f) {return u.i;}+inline float tofloat(unsigned i) {+ union {+ unsigned i;+ float f;+ } u = {i};+ return u.f;+}+inline bool isnan(float f) {return (toint(f) & 0x7fffffff) > 0x7f800000;}@@ -20,7 +28,7 @@ inline bool isinf(float f) {return (toint(f) & 0x7fffffff) == 0x7f800000;}-inline unsigned signbit(float f) {+inline unsigned sign(float f) {return toint(f) & 0x80000000u;}diff --git a/llamafile/half.h b/llamafile/half.hindex ac6545bb..acfeb55c 100644--- a/llamafile/half.h+++ b/llamafile/half.h@@ -26,7 +26,7 @@ inline bool isinf(half f) {return (toint(f) & 0x7fff) == 0x7c00;}-inline int signbit(half f) {+inline int sign(half f) {return toint(f) & 0x8000;}diff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 1f5694a4..bb6046e9 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -31,6 +31,7 @@ void llamafile_launch_browser(const char *);int llamafile_get_math_cpu_count(void);extern bool FLAG_unsecure;+extern const bool have_avx512bf16;#define LLAMAFILE_GPU_ERROR -2#define LLAMAFILE_GPU_DISABLE -1diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cppindex 5f62b721..1656bca8 100644--- a/llamafile/sgemm.cpp+++ b/llamafile/sgemm.cpp@@ -19,10 +19,26 @@#include "llamafile.h"#include <cassert>#include <cosmo.h>+#include <cpuid.h>#include <libc/sysv/consts/hwcap.h>#include <sys/auxv.h>+// TODO(jart): Delete when cosmocc 3.3.3 goes live.+static bool check_avx512bf16(void) {+#ifdef __x86_64__+ // available on cooperlake, zen4, etc.+ int abcd[4];+ __cpuidex(abcd, 1, 0);+ if (abcd[0] >= 7) {+ __cpuidex(abcd, 7, 1);+ return !!(abcd[0] & (1u << 5));+ }+#endif+ return false;+}+static const long hwcap = getauxval(AT_HWCAP);+const bool have_avx512bf16 = check_avx512bf16();/*** Performs optimized matrix multiplication on CPU.@@ -112,6 +128,24 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,#endifreturn false;+ case GGML_TYPE_BF16:+#ifdef __x86_64__+ if (Btype != GGML_TYPE_F32)+ return false;+ if (have_avx512bf16 && !(k % 16))+ return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,+ (const float *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ if (X86_HAVE(AVX512F) && !(k % 16))+ return llamafile_sgemm_bss_avx512(m, n, k, (const ggml_bf16_t *)A, lda,+ (const float *)B, ldb, (float *)C, ldc, ith, nth,+ task);+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && !(k % 8))+ return llamafile_sgemm_bss_avx2(m, n, k, (const ggml_bf16_t *)A, lda, (const float *)B,+ ldb, (float *)C, ldc, ith, nth, task);+#endif+ return false;+case GGML_TYPE_Q8_0:if (Btype != GGML_TYPE_Q8_0)return false;diff --git a/llamafile/sgemm.h b/llamafile/sgemm.hindex b7b45f8d..15e0f883 100644--- a/llamafile/sgemm.h+++ b/llamafile/sgemm.h@@ -23,6 +23,13 @@ bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, constbool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,int, float *, int, int, int, int);+bool llamafile_sgemm_bss_avx2(int, int, int, const ggml_bf16_t *, int, const float *, int, float *,+ int, int, int, int);+bool llamafile_sgemm_bss_avx512(int, int, int, const ggml_bf16_t *, int, const float *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_bss_avx512bf16(int, int, int, const ggml_bf16_t *, int, const float *, int,+ float *, int, int, int, int);+bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,int, float *, int, int, int, int);bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cppnew file mode 100644index 00000000..9d22305d--- /dev/null+++ b/llamafile/sgemm_bss_avx2.cpp@@ -0,0 +1,55 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"+#include <immintrin.h>++#define KN 8++typedef __m256 V;+typedef __m256 D;+typedef ggml_bf16_t TA;+typedef float TB;+typedef float TC;++static inline __m256 zero() {+ return _mm256_setzero_ps();+}++static inline __m256 load(const float *p) {+ return _mm256_loadu_ps(p);+}++static inline __m256 load(const ggml_bf16_t *p) {+ return _mm256_castsi256_ps(+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));+}++#include "sgemmer.inc"++bool llamafile_sgemm_bss_avx2(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cppnew file mode 100644index 00000000..9fcebf40--- /dev/null+++ b/llamafile/sgemm_bss_avx512.cpp@@ -0,0 +1,55 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"+#include <immintrin.h>++#define KN 16++typedef __m512 V;+typedef __m512 D;+typedef ggml_bf16_t TA;+typedef float TB;+typedef float TC;++static inline __m512 zero() {+ return _mm512_setzero_ps();+}++static inline __m512 load(const float *p) {+ return _mm512_loadu_ps(p);+}++static inline __m512 load(const ggml_bf16_t *p) {+ return _mm512_castsi512_ps(+ _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));+}++#include "sgemmer.inc"++bool llamafile_sgemm_bss_avx512(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cppnew file mode 100644index 00000000..5e924903--- /dev/null+++ b/llamafile/sgemm_bss_avx512bf16.cpp@@ -0,0 +1,58 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#ifdef __x86_64__++#include "sgemm.h"+#include <immintrin.h>++#define KN 16++typedef __m256 D;+typedef __m256bh V;+typedef ggml_bf16_t TA;+typedef float TB;+typedef float TC;++static inline __m256 zero() {+ return _mm256_setzero_ps();+}++static inline __m256bh load(const ggml_bf16_t *p) {+ return (__m256bh)_mm256_loadu_ps((const float *)p);+}++static inline __m256bh load(const float *p) {+ return _mm512_cvtneps_pbh(_mm512_loadu_ps(p));+}++static inline __m256 madd(__m256bh x, __m256bh y, __m256 z) {+ return _mm256_dpbf16_ps(z, x, y);+}++#include "sgemmer.inc"++bool llamafile_sgemm_bss_avx512bf16(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,+ TC *C, int ldc, int ith, int nth, int task) {+ if (task != GGML_TASK_TYPE_COMPUTE)+ return true;+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};+ tb.matmul(m, n);+ return true;+}++#endif // __x86_64__diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cppindex f1f31c2b..f93e1e89 100644--- a/llamafile/sgemm_hhs_neon.cpp+++ b/llamafile/sgemm_hhs_neon.cpp@@ -23,6 +23,7 @@#define KN 8typedef float16x8_t V;+typedef float16x8_t D;typedef unsigned short TA;typedef unsigned short TB;typedef float TC;diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cppindex d1584919..7abd5f52 100644--- a/llamafile/sgemm_hss_avx512f.cpp+++ b/llamafile/sgemm_hss_avx512f.cpp@@ -23,6 +23,7 @@#define KN 16typedef __m512 V;+typedef __m512 D;typedef unsigned short TA;typedef float TB;typedef float TC;@@ -36,7 +37,7 @@ static inline V load(const float *p) {}static inline V load(const unsigned short *p) {- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)));+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));}#include "sgemmer.inc"diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cppindex 87bc04cd..14859edc 100644--- a/llamafile/sgemm_hss_f16c.cpp+++ b/llamafile/sgemm_hss_f16c.cpp@@ -23,6 +23,7 @@#define KN 8typedef __m256 V;+typedef __m256 D;typedef unsigned short TA;typedef float TB;typedef float TC;@@ -36,7 +37,7 @@ static inline V load(const float *p) {}static inline V load(const unsigned short *p) {- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)));+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));}#include "sgemmer.inc"diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cppindex aec6c67d..9ae6e68e 100644--- a/llamafile/sgemm_hss_neon.cpp+++ b/llamafile/sgemm_hss_neon.cpp@@ -23,6 +23,7 @@#define KN 4typedef float32x4_t V;+typedef float32x4_t D;typedef unsigned short TA;typedef float TB;typedef float TC;diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cppindex 73c4b90f..a7ea9055 100644--- a/llamafile/sgemm_sss_avx.cpp+++ b/llamafile/sgemm_sss_avx.cpp@@ -23,6 +23,7 @@#define KN 8typedef __m256 V;+typedef __m256 D;typedef float TA;typedef float TB;typedef float TC;diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cppindex 94b4e234..09eb708f 100644--- a/llamafile/sgemm_sss_avx512f.cpp+++ b/llamafile/sgemm_sss_avx512f.cpp@@ -23,6 +23,7 @@#define KN 16typedef __m512 V;+typedef __m512 D;typedef float TA;typedef float TB;typedef float TC;diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cppindex 1e925b80..b32e6567 100644--- a/llamafile/sgemm_sss_fma.cpp+++ b/llamafile/sgemm_sss_fma.cpp@@ -23,6 +23,7 @@#define KN 8typedef __m256 V;+typedef __m256 D;typedef float TA;typedef float TB;typedef float TC;diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cppindex 16f7f0a3..f744654f 100644--- a/llamafile/sgemm_sss_neon.cpp+++ b/llamafile/sgemm_sss_neon.cpp@@ -23,6 +23,7 @@#define KN 4typedef float32x4_t V;+typedef float32x4_t D;typedef float TA;typedef float TB;typedef float TC;diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.incindex c7a0f9c4..c44d3852 100644--- a/llamafile/sgemmer.inc+++ b/llamafile/sgemmer.inc@@ -68,31 +68,31 @@ class SGEMMER {dontinline void gemm5x5(int m0, int m, int n0, int n) {BEGIN_KERNEL(5, 5)- V c00 = zero();- V c01 = zero();- V c02 = zero();- V c03 = zero();- V c04 = zero();- V c10 = zero();- V c11 = zero();- V c12 = zero();- V c13 = zero();- V c14 = zero();- V c20 = zero();- V c21 = zero();- V c22 = zero();- V c23 = zero();- V c24 = zero();- V c30 = zero();- V c31 = zero();- V c32 = zero();- V c33 = zero();- V c34 = zero();- V c40 = zero();- V c41 = zero();- V c42 = zero();- V c43 = zero();- V c44 = zero();+ D c00 = zero();+ D c01 = zero();+ D c02 = zero();+ D c03 = zero();+ D c04 = zero();+ D c10 = zero();+ D c11 = zero();+ D c12 = zero();+ D c13 = zero();+ D c14 = zero();+ D c20 = zero();+ D c21 = zero();+ D c22 = zero();+ D c23 = zero();+ D c24 = zero();+ D c30 = zero();+ D c31 = zero();+ D c32 = zero();+ D c33 = zero();+ D c34 = zero();+ D c40 = zero();+ D c41 = zero();+ D c42 = zero();+ D c43 = zero();+ D c44 = zero();for (int l = 0; l < k; l += KN) {V k0 = load(B + ldb * (j + 0) + l);V k1 = load(B + ldb * (j + 1) + l);@@ -160,18 +160,18 @@ class SGEMMER {dontinline void gemm3x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(3, 4)- V c00 = zero();- V c01 = zero();- V c02 = zero();- V c03 = zero();- V c10 = zero();- V c11 = zero();- V c12 = zero();- V c13 = zero();- V c20 = zero();- V c21 = zero();- V c22 = zero();- V c23 = zero();+ D c00 = zero();+ D c01 = zero();+ D c02 = zero();+ D c03 = zero();+ D c10 = zero();+ D c11 = zero();+ D c12 = zero();+ D c13 = zero();+ D c20 = zero();+ D c21 = zero();+ D c22 = zero();+ D c23 = zero();for (int l = 0; l < k; l += KN) {V k0 = load(B + ldb * (j + 0) + l);V k1 = load(B + ldb * (j + 1) + l);@@ -210,10 +210,10 @@ class SGEMMER {dontinline void gemm1x4(int m0, int m, int n0, int n) {BEGIN_KERNEL(1, 4)- V c00 = zero();- V c01 = zero();- V c02 = zero();- V c03 = zero();+ D c00 = zero();+ D c01 = zero();+ D c02 = zero();+ D c03 = zero();for (int l = 0; l < k; l += KN) {V a0 = load(A + lda * (i + 0) + l);V k0 = load(B + ldb * (j + 0) + l);@@ -234,18 +234,18 @@ class SGEMMER {dontinline void gemm4x1(int m0, int m, int n0, int n) {BEGIN_KERNEL(4, 1)- V c00 = zero();- V c01 = zero();- V c02 = zero();- V c10 = zero();- V c11 = zero();- V c12 = zero();- V c20 = zero();- V c21 = zero();- V c22 = zero();- V c30 = zero();- V c31 = zero();- V c32 = zero();+ D c00 = zero();+ D c01 = zero();+ D c02 = zero();+ D c10 = zero();+ D c11 = zero();+ D c12 = zero();+ D c20 = zero();+ D c21 = zero();+ D c22 = zero();+ D c30 = zero();+ D c31 = zero();+ D c32 = zero();int l = 0;while (l + KN * 3 <= k) {{@@ -289,10 +289,10 @@ class SGEMMER {dontinline void gemm1x1(int m0, int m, int n0, int n) {BEGIN_KERNEL(1, 1)- V c0 = zero();- V c1 = zero();- V c2 = zero();- V c3 = zero();+ D c0 = zero();+ D c1 = zero();+ D c2 = zero();+ D c3 = zero();int l = 0;while (l + KN * 4 <= k) {c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);diff --git a/llamafile/tester.h b/llamafile/tester.hindex f8de5e08..96c49d49 100644--- a/llamafile/tester.h+++ b/llamafile/tester.h@@ -128,10 +128,10 @@ template <typename T> ErrorReport diff(int m, int n, const T *Wan, int lda, consinfs += flt::isinf(y);denormals -= flt::isdenormal(x);denormals += flt::isdenormal(y);- flips += flt::signbit(x) != flt::signbit(y);+ flips += flt::sign(x) != flt::sign(y);long long xi = flt::toint(x);long long yi = flt::toint(y);- if (flt::signbit(x) == flt::signbit(y)) {+ if (flt::sign(x) == flt::sign(y)) {++considered;long long bad = std::abs(xi - yi);ulp += bad;