Skip to content

Distribute and run LLMs with a single file.

Add support for bfloat16

Justine Tunney6dbc507c2ef8
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.h
index e68b7287..38ebc5a9 100644
--- a/llama.cpp/ggml-impl.h
+++ b/llama.cpp/ggml-impl.h
@@ -1,3 +1,5 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
#pragma once
#include "ggml.h"
@@ -9,7 +11,7 @@
#include <stddef.h>
#include <stdbool.h>
#include <string.h> // memcpy
-#include <math.h> // fabsf
+#include <math.h> // fabsf, fpclassify
#ifdef __cplusplus
extern "C" {
@@ -260,6 +262,9 @@ size_t ggml_hash_insert ( struct ggml_hash_set hash_set, struct ggml
// return index, asserts if table is full
size_t ggml_hash_find_or_insert( struct ggml_hash_set hash_set, struct ggml_tensor * key);
+#define GGML_FP32_TO_BF16(x) ggml_fp32_to_bf16(x)
+#define GGML_BF16_TO_FP32(x) ggml_bf16_to_fp32(x)
+
#ifdef __cplusplus
}
#endif
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index a1b67a66..589c8394 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -262,6 +262,12 @@ static ggml_fp16_t ggml_table_exp_f16[1 << 16];
// precomputed f32 table for f16 (256 KB) (ggml-impl.h)
float ggml_table_f32_f16[1 << 16];
+// precomputed gelu table for bf16 (128 KB)
+static ggml_bf16_t ggml_table_gelu_bf16[1 << 16];
+
+// precomputed exp table for bf16 (128 KB)
+static ggml_bf16_t ggml_table_exp_bf16[1 << 16];
+
GGML_CALL const char * ggml_status_to_string(enum ggml_status status) {
switch (status) {
case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";
@@ -289,6 +295,12 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int n) {
}
}
+void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int n) {
+ for (int i = 0; i < n; i++) {
+ y[i] = GGML_BF16_TO_FP32(x[i]);
+ }
+}
+
#ifdef __x86_64__
#pragma GCC push_options
#pragma GCC target("f16c")
@@ -322,6 +334,32 @@ void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int n) {
}
}
+#ifdef __x86_64__
+#pragma GCC push_options
+#pragma GCC target("avx512bf16")
+static void ggml_fp32_to_bf16_row_avx512bf16(const float * x, ggml_bf16_t * y, int n) {
+ int i = 0;
+ for (; i + 32 <= n; i += 32)
+ _mm512_storeu_ps(
+ (__m512 *)(y + i),
+ (__m512)_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),
+ _mm512_loadu_ps(x + i)));
+ for (; i < n; i++)
+ y[i] = GGML_FP32_TO_BF16(x[i]);
+}
+#pragma GCC pop_options
+#endif
+
+void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n) {
+#ifdef __x86_64__
+ if (have_avx512bf16)
+ return ggml_fp32_to_bf16_row_avx512bf16(x, y, n);
+#endif
+ for (int i = 0; i < n; i++) {
+ y[i] = GGML_FP32_TO_BF16(x[i]);
+ }
+}
+
bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {
return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;
}
@@ -382,6 +420,7 @@ static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * restrict x, size_t bx, const float * restrict y, size_t by, int nrc);
static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * restrict x, size_t bx, ggml_fp16_t * restrict y, size_t by, int nrc);
+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc);
static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
[GGML_TYPE_I8] = {
@@ -712,6 +751,18 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
.type_size = sizeof(block_q8_K),
.is_quantized = true,
.from_float = quantize_row_q8_K,
+ },
+ [GGML_TYPE_BF16] = {
+ .type_name = "bf16",
+ .blck_size = 1,
+ .type_size = sizeof(ggml_bf16_t),
+ .is_quantized = false,
+ .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,
+ .from_float = (ggml_from_float_t) ggml_fp32_to_bf16_row,
+ .from_float_reference = (ggml_from_float_t) ggml_fp32_to_bf16_row,
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
+ .vec_dot_type = GGML_TYPE_BF16,
+ .nrows = 1,
}
};
@@ -1361,6 +1412,8 @@ inline static void ggml_vec_set_i32(const int n, int32_t * x, const int32_t v) {
inline static void ggml_vec_set_f16(const int n, ggml_fp16_t * x, const int32_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+inline static void ggml_vec_set_bf16(const int n, ggml_bf16_t * x, const ggml_bf16_t v) { for (int i = 0; i < n; ++i) x[i] = v; }
+
inline static void ggml_vec_add_f32 (const int n, float * z, const float * x, const float * y) { for (int i = 0; i < n; ++i) z[i] = x[i] + y[i]; }
inline static void ggml_vec_add1_f32(const int n, float * z, const float * x, const float v) { for (int i = 0; i < n; ++i) z[i] = x[i] + v; }
inline static void ggml_vec_acc_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] += x[i]; }
@@ -1415,6 +1468,19 @@ static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float *
*s = sumf;
}
+static void ggml_vec_dot_bf16(int n, float * restrict s, size_t bs, ggml_bf16_t * restrict x, size_t bx, ggml_bf16_t * restrict y, size_t by, int nrc) {
+ assert(nrc == 1);
+ UNUSED(nrc);
+ UNUSED(bx);
+ UNUSED(by);
+ UNUSED(bs);
+ ggml_float sumf = 0;
+ for (int i = 0; i < n; ++i) {
+ sumf += GGML_BF16_TO_FP32(x[i]) * GGML_BF16_TO_FP32(y[i]);
+ }
+ *s = sumf;
+}
+
#ifdef __x86_64__
#pragma GCC push_options
#pragma GCC target("avx")
@@ -1690,6 +1756,16 @@ inline static void ggml_vec_gelu_f16(const int n, ggml_fp16_t * y, const ggml_fp
}
}
+inline static void ggml_vec_gelu_bf16(const int n, ggml_bf16_t * y, const ggml_bf16_t * x) {
+ for (int i = 0; i < n; ++i) {
+ union {
+ ggml_bf16_t f;
+ uint16_t i;
+ } u = {x[i]};
+ y[i] = ggml_table_gelu_bf16[u.i];
+ }
+}
+
#ifdef GGML_GELU_FP16
inline static void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
uint16_t t;
@@ -1821,6 +1897,14 @@ inline static void ggml_vec_sum_f16_ggf(const int n, float * s, const ggml_fp16_
*s = sum;
}
+inline static void ggml_vec_sum_bf16_ggf(const int n, float * s, const ggml_bf16_t * x) {
+ float sum = 0.0f;
+ for (int i = 0; i < n; ++i) {
+ sum += GGML_BF16_TO_FP32(x[i]);
+ }
+ *s = sum;
+}
+
inline static void ggml_vec_max_f32(const int n, float * s, const float * x) {
#ifndef GGML_USE_ACCELERATE
float max = -INFINITY;
@@ -2442,6 +2526,7 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) {
switch (ftype) {
case GGML_FTYPE_ALL_F32: wtype = GGML_TYPE_F32; break;
case GGML_FTYPE_MOSTLY_F16: wtype = GGML_TYPE_F16; break;
+ case GGML_FTYPE_MOSTLY_BF16: wtype = GGML_TYPE_BF16; break;
case GGML_FTYPE_MOSTLY_Q4_0: wtype = GGML_TYPE_Q4_0; break;
case GGML_FTYPE_MOSTLY_Q4_1: wtype = GGML_TYPE_Q4_1; break;
case GGML_FTYPE_MOSTLY_Q5_0: wtype = GGML_TYPE_Q5_0; break;
@@ -2572,15 +2657,20 @@ struct ggml_context * ggml_init(struct ggml_init_params params) {
{
const uint64_t t_start = ggml_time_us(); UNUSED(t_start);
- ggml_fp16_t ii;
for (int i = 0; i < (1 << 16); ++i) {
- uint16_t ui = i;
- memcpy(&ii, &ui, sizeof(ii));
- const float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(ii);
+ union {
+ uint16_t i;
+ ggml_fp16_t fp16;
+ ggml_bf16_t bf16;
+ } u = {i};
+ float f = ggml_table_f32_f16[i] = GGML_COMPUTE_FP16_TO_FP32(u.fp16);
ggml_table_gelu_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_f32(f));
ggml_table_gelu_quick_f16[i] = GGML_FP32_TO_FP16(ggml_gelu_quick_f32(f));
ggml_table_silu_f16[i] = GGML_FP32_TO_FP16(ggml_silu_f32(f));
- ggml_table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f));
+ ggml_table_exp_f16[i] = GGML_FP32_TO_FP16(expf(f));
+ f = GGML_BF16_TO_FP32(u.bf16);
+ ggml_table_gelu_bf16[i] = GGML_FP32_TO_BF16(ggml_gelu_f32(f));
+ ggml_table_exp_bf16[i] = GGML_FP32_TO_BF16(expf(f));
}
const uint64_t t_end = ggml_time_us(); UNUSED(t_end);
@@ -3100,6 +3190,13 @@ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) {
ggml_vec_set_f16(nc, (ggml_fp16_t *)(data + i*n1), GGML_FP32_TO_FP16(value));
}
} break;
+ case GGML_TYPE_BF16:
+ {
+ assert(tensor->nb[0] == sizeof(ggml_bf16_t));
+ for (int i = 0; i < n; i++) {
+ ggml_vec_set_bf16(nc, (ggml_bf16_t *)(data + i*n1), GGML_FP32_TO_BF16(value));
+ }
+ } break;
case GGML_TYPE_F32:
{
assert(tensor->nb[0] == sizeof(float));
@@ -3167,6 +3264,11 @@ int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i) {
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
}
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
+ }
case GGML_TYPE_F32:
{
GGML_ASSERT(tensor->nb[0] == sizeof(float));
@@ -3209,6 +3311,11 @@ void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value) {
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);
} break;
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
+ } break;
case GGML_TYPE_F32:
{
GGML_ASSERT(tensor->nb[0] == sizeof(float));
@@ -3232,6 +3339,8 @@ int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i
return ((int32_t *) data)[0];
case GGML_TYPE_F16:
return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
+ case GGML_TYPE_BF16:
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
case GGML_TYPE_F32:
return ((float *) data)[0];
default:
@@ -3260,6 +3369,10 @@ void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,
{
((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
+ } break;
case GGML_TYPE_F32:
{
((float *)(data))[0] = value;
@@ -3298,6 +3411,11 @@ float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i) {
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
return GGML_FP16_TO_FP32(((ggml_fp16_t *)(tensor->data))[i]);
}
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *)(tensor->data))[i]);
+ }
case GGML_TYPE_F32:
{
GGML_ASSERT(tensor->nb[0] == sizeof(float));
@@ -3340,6 +3458,11 @@ void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value) {
GGML_ASSERT(tensor->nb[0] == sizeof(ggml_fp16_t));
((ggml_fp16_t *)(tensor->data))[i] = GGML_FP32_TO_FP16(value);
} break;
+ case GGML_TYPE_BF16:
+ {
+ GGML_ASSERT(tensor->nb[0] == sizeof(ggml_bf16_t));
+ ((ggml_bf16_t *)(tensor->data))[i] = GGML_FP32_TO_BF16(value);
+ } break;
case GGML_TYPE_F32:
{
GGML_ASSERT(tensor->nb[0] == sizeof(float));
@@ -3363,6 +3486,8 @@ float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,
return ((int32_t *) data)[0];
case GGML_TYPE_F16:
return GGML_FP16_TO_FP32(((ggml_fp16_t *) data)[0]);
+ case GGML_TYPE_BF16:
+ return GGML_BF16_TO_FP32(((ggml_bf16_t *) data)[0]);
case GGML_TYPE_F32:
return ((float *) data)[0];
default:
@@ -3391,6 +3516,10 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,
{
((ggml_fp16_t *)(data))[0] = GGML_FP32_TO_FP16(value);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ((ggml_bf16_t *)(data))[0] = GGML_FP32_TO_BF16(value);
+ } break;
case GGML_TYPE_F32:
{
((float *)(data))[0] = value;
@@ -3585,7 +3714,11 @@ static struct ggml_tensor * ggml_add_cast_impl(
// TODO: support less-strict constraint
// GGML_ASSERT(ggml_can_repeat(b, a));
GGML_ASSERT(ggml_can_repeat_rows(b, a));
- GGML_ASSERT(ggml_is_quantized(a->type) || a->type == GGML_TYPE_F16); // currently only supported for quantized input and f16
+
+ // currently only supported for quantized input and f16
+ GGML_ASSERT(ggml_is_quantized(a->type) ||
+ a->type == GGML_TYPE_F16 ||
+ a->type == GGML_TYPE_BF16);
bool is_node = false;
@@ -7401,6 +7534,24 @@ static void ggml_compute_forward_dup_f32(
id += ne00 * (ne01 - ir1);
}
}
+ } else if (dst->type == GGML_TYPE_BF16) {
+ size_t id = 0;
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) dst->data;
+
+ for (int i03 = 0; i03 < ne03; i03++) {
+ for (int i02 = 0; i02 < ne02; i02++) {
+ id += ne00 * ir0;
+ for (int i01 = ir0; i01 < ir1; i01++) {
+ for (int i00 = 0; i00 < ne00; i00++) {
+ const float * src0_ptr = (float *) ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+
+ dst_ptr[id] = GGML_FP32_TO_BF16(*src0_ptr);
+ id++;
+ }
+ }
+ id += ne00 * (ne01 - ir1);
+ }
+ }
} else {
GGML_ASSERT(false); // TODO: implement
}
@@ -7520,6 +7671,58 @@ static void ggml_compute_forward_dup_f32(
}
}
}
+ } else if (dst->type == GGML_TYPE_BF16) {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ i10 += ne00 * ir0;
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ for (int64_t i01 = ir0; i01 < ir1; i01++) {
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);
+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);
+
+ *(ggml_bf16_t *) dst_ptr = GGML_FP32_TO_BF16(*(const float *) src0_ptr);
+
+ if (++i10 == ne0) {
+ i10 = 0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
+ i10 += ne00 * (ne01 - ir1);
+ while (i10 >= ne0) {
+ i10 -= ne0;
+ if (++i11 == ne1) {
+ i11 = 0;
+ if (++i12 == ne2) {
+ i12 = 0;
+ if (++i13 == ne3) {
+ i13 = 0;
+ }
+ }
+ }
+ }
+ }
+ }
} else {
GGML_ASSERT(false); // TODO: implement
}
@@ -7875,6 +8078,85 @@ static void ggml_compute_forward_add_f16_f32(
}
}
+static void ggml_compute_forward_add_bf16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+
+ if (dst->type == GGML_TYPE_F32) {
+ GGML_ASSERT( nb0 == sizeof(float));
+ }
+ else {
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ }
+
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(float)) {
+ if (dst->type == GGML_TYPE_BF16) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i]);
+ }
+ }
+ } else {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ float * dst_ptr = (float *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ float * src1_ptr = (float *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_BF16_TO_FP32(src0_ptr[i]) + src1_ptr[i];
+ }
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ASSERT(false);
+ }
+}
+
static void ggml_compute_forward_add_f16_f16(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -7931,6 +8213,62 @@ static void ggml_compute_forward_add_f16_f16(
}
}
+static void ggml_compute_forward_add_bf16_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, src1) && ggml_are_same_shape(src0, dst));
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ if (nb10 == sizeof(ggml_bf16_t)) {
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0, src1 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ ggml_bf16_t * src1_ptr = (ggml_bf16_t *) ((char *) src1->data + i3*nb13 + i2*nb12 + i1*nb11);
+
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + GGML_BF16_TO_FP32(src1_ptr[i]));
+ }
+ }
+ }
+ else {
+ // src1 is not contiguous
+ GGML_ASSERT(false);
+ }
+}
+
static void ggml_compute_forward_add_q_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -8040,6 +8378,18 @@ static void ggml_compute_forward_add(
GGML_ASSERT(false);
}
} break;
+ case GGML_TYPE_BF16:
+ {
+ if (src1->type == GGML_TYPE_BF16) {
+ ggml_compute_forward_add_bf16_bf16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add_bf16_f32(params, dst);
+ }
+ else {
+ GGML_ASSERT(false);
+ }
+ } break;
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -8176,7 +8526,7 @@ static void ggml_compute_forward_add1_f16_f32(
}
}
-static void ggml_compute_forward_add1_f16_f16(
+static void ggml_compute_forward_add1_bf16_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -8191,7 +8541,7 @@ static void ggml_compute_forward_add1_f16_f16(
}
// scalar to add
- const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);
+ const float v = *(float *) src1->data;
const int ith = params->ith;
const int nth = params->nth;
@@ -8200,12 +8550,64 @@ static void ggml_compute_forward_add1_f16_f16(
GGML_TENSOR_UNARY_OP_LOCALS
- GGML_ASSERT(src0->type == GGML_TYPE_F16);
- GGML_ASSERT(src1->type == GGML_TYPE_F16);
- GGML_ASSERT(dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
- GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
+static void ggml_compute_forward_add1_f16_f16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // scalar to add
+ const float v = GGML_FP16_TO_FP32(*(ggml_fp16_t *) src1->data);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F16);
+ GGML_ASSERT(dst->type == GGML_TYPE_F16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
// rows per thread
const int dr = (nr + nth - 1)/nth;
@@ -8228,6 +8630,58 @@ static void ggml_compute_forward_add1_f16_f16(
}
}
+static void ggml_compute_forward_add1_bf16_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
+ GGML_ASSERT(ggml_is_scalar(src1));
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // scalar to add
+ const float v = GGML_BF16_TO_FP32(*(ggml_bf16_t *) src1->data);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(src0);
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_BF16);
+ GGML_ASSERT(dst->type == GGML_TYPE_BF16);
+
+ GGML_ASSERT( nb0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // src0 and dst are same shape => same indices
+ const int i3 = ir/(ne2*ne1);
+ const int i2 = (ir - i3*ne2*ne1)/ne1;
+ const int i1 = (ir - i3*ne2*ne1 - i2*ne1);
+
+ ggml_bf16_t * dst_ptr = (ggml_bf16_t *) ((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 );
+ ggml_bf16_t * src0_ptr = (ggml_bf16_t *) ((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01);
+ for (int i = 0; i < ne0; i++) {
+ dst_ptr[i] = GGML_FP32_TO_BF16(GGML_BF16_TO_FP32(src0_ptr[i]) + v);
+ }
+ }
+}
+
static void ggml_compute_forward_add1_q_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -8321,6 +8775,18 @@ static void ggml_compute_forward_add1(
GGML_ASSERT(false);
}
} break;
+ case GGML_TYPE_BF16:
+ {
+ if (src1->type == GGML_TYPE_BF16) {
+ ggml_compute_forward_add1_bf16_bf16(params, dst);
+ }
+ else if (src1->type == GGML_TYPE_F32) {
+ ggml_compute_forward_add1_bf16_f32(params, dst);
+ }
+ else {
+ GGML_ASSERT(false);
+ }
+ } break;
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -8448,6 +8914,7 @@ static void ggml_compute_forward_acc(
ggml_compute_forward_acc_f32(params, dst);
} break;
case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -8968,6 +9435,40 @@ static void ggml_compute_forward_sum_f16(
((ggml_fp16_t *) dst->data)[0] = GGML_FP32_TO_FP16(sum);
}
+static void ggml_compute_forward_sum_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ assert(params->ith == 0);
+ assert(ggml_is_scalar(dst));
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ assert(src0->nb[0] == sizeof(ggml_bf16_t));
+
+ GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
+ GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
+
+ float sum = 0;
+ float row_sum = 0;
+
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_bf16_ggf(ne00,
+ &row_sum,
+ (ggml_bf16_t *) ((char *) src0->data + i01 * nb01 + i02 * nb02 + i03 * nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ ((ggml_bf16_t *) dst->data)[0] = GGML_FP32_TO_BF16(sum);
+}
+
static void ggml_compute_forward_sum(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -8983,6 +9484,10 @@ static void ggml_compute_forward_sum(
{
ggml_compute_forward_sum_f16(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_sum_bf16(params, dst);
+ } break;
default:
{
GGML_ASSERT(false);
@@ -9257,6 +9762,7 @@ static void ggml_compute_forward_repeat(
switch (src0->type) {
case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
case GGML_TYPE_I16:
{
ggml_compute_forward_repeat_f16(params, dst);
@@ -10717,7 +11223,8 @@ static void ggml_compute_forward_mul_mat(
#endif
if (src1_cont && (src1->type == vec_dot_type ||
- src0->type == GGML_TYPE_F16)) {
+ src0->type == GGML_TYPE_F16 ||
+ src0->type == GGML_TYPE_BF16)) {
for (int64_t j = 0; j < ne13; j++)
for (int64_t i = 0; i < ne12; i++)
if (!llamafile_sgemm(ne01, ne11, ne00/ggml_blck_size(src0->type),
@@ -11603,6 +12110,7 @@ static void ggml_compute_forward_set(
ggml_compute_forward_set_f32(params, dst);
} break;
case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -11776,6 +12284,49 @@ static void ggml_compute_forward_get_rows_f16(
}
}
+static void ggml_compute_forward_get_rows_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int64_t nc = ne00;
+ const int64_t nr = ggml_nelements(src1);
+
+ assert(ne0 == nc);
+ assert(ne02 == ne11);
+ assert(nb00 == sizeof(ggml_bf16_t));
+ assert(ggml_nrows(dst) == nr);
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int64_t i = ir0; i < ir1; ++i) {
+ const int64_t i12 = i/(ne11*ne10);
+ const int64_t i11 = (i - i12*ne11*ne10)/ne10;
+ const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
+ const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
+
+ ggml_bf16_to_fp32_row(
+ (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03),
+ (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc);
+ }
+}
+
static void ggml_compute_forward_get_rows_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -11852,6 +12403,10 @@ static void ggml_compute_forward_get_rows(
{
ggml_compute_forward_get_rows_f16(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_get_rows_bf16(params, dst);
+ } break;
case GGML_TYPE_F32:
case GGML_TYPE_I32:
{
@@ -11923,6 +12478,45 @@ static void ggml_compute_forward_get_rows_back_f32_f16(
}
}
+static void ggml_compute_forward_get_rows_back_f32_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(params->ith == 0);
+ GGML_ASSERT(ggml_is_contiguous(dst));
+
+ // ggml_compute_forward_dup_same_cont(params, opt0, dst);
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ if (params->ith != 0) {
+ return;
+ }
+ memset(dst->data, 0, ggml_nbytes(dst));
+ }
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ const int nc = src0->ne[0];
+ const int nr = ggml_nelements(src1);
+
+ GGML_ASSERT( dst->ne[0] == nc);
+ GGML_ASSERT(src0->nb[0] == sizeof(ggml_bf16_t));
+
+ for (int i = 0; i < nr; ++i) {
+ const int r = ((int32_t *) src1->data)[i];
+
+ for (int j = 0; j < nc; ++j) {
+ ggml_bf16_t v = ((ggml_bf16_t *) ((char *) src0->data + i*src0->nb[1]))[j];
+ ((float *) ((char *) dst->data + r*dst->nb[1]))[j] += GGML_BF16_TO_FP32(v);
+ }
+ }
+}
+
static void ggml_compute_forward_get_rows_back_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -11973,6 +12567,10 @@ static void ggml_compute_forward_get_rows_back(
{
ggml_compute_forward_get_rows_back_f32_f16(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_get_rows_back_f32_bf16(params, dst);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_get_rows_back_f32(params, dst);
@@ -12514,57 +13112,123 @@ static void ggml_compute_forward_alibi_f16(
}
}
-static void ggml_compute_forward_alibi(
+static void ggml_compute_forward_alibi_bf16(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
- switch (src0->type) {
- case GGML_TYPE_F16:
- {
- ggml_compute_forward_alibi_f16(params, dst);
- } break;
- case GGML_TYPE_F32:
- {
- ggml_compute_forward_alibi_f32(params, dst);
- } break;
- case GGML_TYPE_Q4_0:
- case GGML_TYPE_Q4_1:
- case GGML_TYPE_Q5_0:
- case GGML_TYPE_Q5_1:
- case GGML_TYPE_Q8_0:
- case GGML_TYPE_Q8_1:
- case GGML_TYPE_Q2_K:
- case GGML_TYPE_Q3_K:
- case GGML_TYPE_Q4_K:
- case GGML_TYPE_Q5_K:
- case GGML_TYPE_Q6_K:
- case GGML_TYPE_IQ2_XXS:
- case GGML_TYPE_IQ2_XS:
- case GGML_TYPE_IQ3_XXS:
- case GGML_TYPE_IQ1_S:
- case GGML_TYPE_IQ4_NL:
- case GGML_TYPE_IQ4_XS:
- case GGML_TYPE_IQ3_S:
- case GGML_TYPE_IQ2_S:
- case GGML_TYPE_Q8_K:
- case GGML_TYPE_I8:
- case GGML_TYPE_I16:
- case GGML_TYPE_I32:
- case GGML_TYPE_I64:
- case GGML_TYPE_F64:
- case GGML_TYPE_COUNT:
- {
- GGML_ASSERT(false);
- } break;
- }
-}
+ assert(params->ith == 0);
-// ggml_compute_forward_clamp
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
-static void ggml_compute_forward_clamp_f32(
- const struct ggml_compute_params * params,
+ //const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_head = ((int32_t *) dst->op_params)[1];
+ float max_bias;
+ memcpy(&max_bias, (int32_t *) dst->op_params + 2, sizeof(float));
+
+ const int ne0 = src0->ne[0]; // all_seq_len = n_past + ne1
+ const int ne1 = src0->ne[1]; // seq_len_without_past
+ const int ne2 = src0->ne[2]; // n_head -> this is k
+ //const int ne3 = src0->ne[3]; // 1 -> bsz
+
+ const int n = ggml_nrows(src0);
+ const int ne2_ne3 = n/ne1; // ne2*ne3
+
+ const int nb0 = src0->nb[0];
+ const int nb1 = src0->nb[1];
+ const int nb2 = src0->nb[2];
+ //const int nb3 = src0->nb[3];
+
+ GGML_ASSERT(nb0 == sizeof(ggml_bf16_t));
+ //GGML_ASSERT(ne1 + n_past == ne0); (void) n_past;
+ GGML_ASSERT(n_head == ne2);
+
+ // add alibi to src0 (KQ_scaled)
+ const int n_heads_log2_floor = 1 << (int) floor(log2(n_head));
+
+ const float m0 = powf(2.0f, -(max_bias) / n_heads_log2_floor);
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_heads_log2_floor);
+
+ for (int k = 0; k < ne2_ne3; k++) {
+ // TODO: k*nb2 or k*nb3
+ float m_k;
+
+ if (k < n_heads_log2_floor) {
+ m_k = powf(m0, k + 1);
+ } else {
+ m_k = powf(m1, 2 * (k - n_heads_log2_floor) + 1);
+ }
+
+ for (int i = 0; i < ne0; i++) {
+ for (int j = 0; j < ne1; j++) {
+ ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i*nb0 + j*nb1 + k*nb2);
+ float * pdst = (float *)((char *) dst->data + i*nb0 + j*nb1 + k*nb2);
+
+ // we return F32
+ pdst[0] = i * m_k + GGML_BF16_TO_FP32(src[0]);
+ }
+ }
+ }
+}
+
+static void ggml_compute_forward_alibi(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+
+ switch (src0->type) {
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_alibi_f16(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_alibi_bf16(params, dst);
+ } break;
+ case GGML_TYPE_F32:
+ {
+ ggml_compute_forward_alibi_f32(params, dst);
+ } break;
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q8_1:
+ case GGML_TYPE_Q2_K:
+ case GGML_TYPE_Q3_K:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ2_XXS:
+ case GGML_TYPE_IQ2_XS:
+ case GGML_TYPE_IQ3_XXS:
+ case GGML_TYPE_IQ1_S:
+ case GGML_TYPE_IQ4_NL:
+ case GGML_TYPE_IQ4_XS:
+ case GGML_TYPE_IQ3_S:
+ case GGML_TYPE_IQ2_S:
+ case GGML_TYPE_Q8_K:
+ case GGML_TYPE_I8:
+ case GGML_TYPE_I16:
+ case GGML_TYPE_I32:
+ case GGML_TYPE_I64:
+ case GGML_TYPE_F64:
+ case GGML_TYPE_COUNT:
+ {
+ GGML_ASSERT(false);
+ } break;
+ }
+}
+
+// ggml_compute_forward_clamp
+
+static void ggml_compute_forward_clamp_f32(
+ const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
@@ -12617,6 +13281,7 @@ static void ggml_compute_forward_clamp(
ggml_compute_forward_clamp_f32(params, dst);
} break;
case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
case GGML_TYPE_Q4_0:
case GGML_TYPE_Q4_1:
case GGML_TYPE_Q5_0:
@@ -13052,6 +13717,173 @@ static void ggml_compute_forward_rope_f16(
}
}
+static void ggml_compute_forward_rope_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst,
+ const bool forward) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ if (params->type == GGML_TASK_TYPE_INIT || params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
+
+ //const int n_past = ((int32_t *) dst->op_params)[0];
+ const int n_dims = ((int32_t *) dst->op_params)[1];
+ const int mode = ((int32_t *) dst->op_params)[2];
+ const int n_ctx = ((int32_t *) dst->op_params)[3];
+ const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
+ memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
+ memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
+ memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
+ memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
+ memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
+ memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
+
+ GGML_TENSOR_UNARY_OP_LOCALS
+
+ //printf("ne0: %d, ne1: %d, ne2: %d, ne3: %d\n", ne0, ne1, ne2, ne3);
+ //printf("n_past = %d, ne2 = %d\n", n_past, ne2);
+
+ GGML_ASSERT(nb0 == sizeof(ggml_bf16_t));
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nr = ggml_nrows(dst);
+
+ GGML_ASSERT(n_dims <= ne0);
+ GGML_ASSERT(n_dims % 2 == 0);
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ // row index used to determine which thread to use
+ int ir = 0;
+
+ const float theta_scale = powf(freq_base, -2.0f/n_dims);
+ const float inv_ndims = -1.f/n_dims;
+ float corr_dims[2];
+ ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims);
+
+ const bool is_neox = mode & 2;
+ const bool is_glm = mode & 4;
+
+ // backward process uses inverse rotation by cos and sin.
+ // cos and sin build a rotation matrix, where the inverse is the transpose.
+ // this essentially just switches the sign of sin.
+ const float sin_sign = forward ? 1.0f : -1.0f;
+
+ const int32_t * pos = (const int32_t *) src1->data;
+
+ for (int64_t i3 = 0; i3 < ne3; i3++) {
+ for (int64_t i2 = 0; i2 < ne2; i2++) {
+ const int64_t p = pos[i2];
+
+ float * cache = (float *) params->wdata + (ne0 + CACHE_LINE_SIZE_F32)*ith;
+ if (!is_glm && !is_neox) { // TODO: cache sin/cos for glm, neox
+ ggml_rope_cache_init(p, freq_scale, corr_dims, ne0, ext_factor, attn_factor, cache, sin_sign, theta_scale);
+ }
+
+ for (int64_t i1 = 0; i1 < ne1; i1++) {
+ if (ir++ < ir0) continue;
+ if (ir > ir1) break;
+
+ float theta_base = (float)p;
+
+ if (is_glm) {
+ theta_base = MIN(p, n_ctx - 2);
+ float block_theta = MAX(p - (n_ctx - 2), 0);
+ for (int64_t i0 = 0; i0 < ne0 / 4; i0++) {
+ const float cos_theta = cosf(theta_base);
+ const float sin_theta = sinf(theta_base) * sin_sign;
+ const float cos_block_theta = cosf(block_theta);
+ const float sin_block_theta = sinf(block_theta) * sin_sign;
+
+ theta_base *= theta_scale;
+ block_theta *= theta_scale;
+
+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = GGML_BF16_TO_FP32(src[0]);
+ const float x1 = GGML_BF16_TO_FP32(src[n_dims/2]);
+ const float x2 = GGML_BF16_TO_FP32(src[n_dims]);
+ const float x3 = GGML_BF16_TO_FP32(src[n_dims/2*3]);
+
+ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);
+ dst_data[n_dims/2] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);
+ dst_data[n_dims] = GGML_FP32_TO_BF16(x2*cos_block_theta - x3*sin_block_theta);
+ dst_data[n_dims/2*3] = GGML_FP32_TO_BF16(x2*sin_block_theta + x3*cos_block_theta);
+ }
+ } else if (!is_neox) {
+ for (int64_t i0 = 0; i0 < ne0; i0 += 2) {
+ const float cos_theta = cache[i0 + 0];
+ const float sin_theta = cache[i0 + 1];
+
+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = GGML_BF16_TO_FP32(src[0]);
+ const float x1 = GGML_BF16_TO_FP32(src[1]);
+
+ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);
+ dst_data[1] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);
+ }
+ } else {
+ // TODO: this might be wrong for ne0 != n_dims - need double check
+ // it seems we have to rope just the first n_dims elements and do nothing with the rest
+ // ref: https://github.com/ml-explore/mlx/blob/dc2edc762c797e3b8de50b1dad4dc0a131691033/benchmarks/python/llama_jax_bench.py#L11-L26
+ theta_base *= freq_scale;
+ for (int64_t ic = 0; ic < ne0; ic += 2) {
+ if (ic < n_dims) {
+ const int64_t ib = 0;
+
+ // simplified from `(ib * n_dims + ic) * inv_ndims`
+ float cur_rot = inv_ndims * ic - ib;
+
+ float cos_theta, sin_theta;
+ rope_yarn(
+ theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor,
+ &cos_theta, &sin_theta
+ );
+ sin_theta *= sin_sign;
+
+ theta_base *= theta_scale;
+
+ const int64_t i0 = ib*n_dims + ic/2;
+
+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ const float x0 = GGML_BF16_TO_FP32(src[0]);
+ const float x1 = GGML_BF16_TO_FP32(src[n_dims/2]);
+
+ dst_data[0] = GGML_FP32_TO_BF16(x0*cos_theta - x1*sin_theta);
+ dst_data[n_dims/2] = GGML_FP32_TO_BF16(x0*sin_theta + x1*cos_theta);
+ } else {
+ const int64_t i0 = ic;
+
+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i3*nb03 + i2*nb02 + i1*nb01 + i0*nb00);
+ ggml_bf16_t * dst_data = (ggml_bf16_t *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1 + i0*nb0);
+
+ dst_data[0] = src[0];
+ dst_data[1] = src[1];
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
static void ggml_compute_forward_rope(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -13063,6 +13895,10 @@ static void ggml_compute_forward_rope(
{
ggml_compute_forward_rope_f16(params, dst, true);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_rope_bf16(params, dst, true);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_f32(params, dst, true);
@@ -13087,6 +13923,10 @@ static void ggml_compute_forward_rope_back(
{
ggml_compute_forward_rope_f16(params, dst, false);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_rope_bf16(params, dst, false);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_f32(params, dst, false);
@@ -13199,6 +14039,105 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
}
}
+static void ggml_compute_forward_conv_transpose_1d_bf16_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ int64_t t0 = ggml_perf_time_us();
+ UNUSED(t0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int nk = ne00*ne01*ne02;
+
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ if (ith != 0) {
+ return;
+ }
+ memset(params->wdata, 0, params->wsize);
+
+ // permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
+ {
+ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + 0;
+
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const ggml_bf16_t * const src = (ggml_bf16_t *)((char *) src0->data + i02*nb02 + i01*nb01);
+ ggml_bf16_t * dst_data = wdata + i01*ne00*ne02;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ dst_data[i00*ne02 + i02] = src[i00];
+ }
+ }
+ }
+ }
+
+ // permute source data (src1) from (L x Cin) to (Cin x L)
+ {
+ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + nk;
+ ggml_bf16_t * dst_data = wdata;
+
+ for (int64_t i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i11*nb11);
+ for (int64_t i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne11 + i11] = GGML_FP32_TO_BF16(src[i10]);
+ }
+ }
+ }
+
+ // need to zero dst since we are accumulating into it
+ memset(dst->data, 0, ggml_nbytes(dst));
+
+ return;
+ }
+
+ if (params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
+
+ // total rows in dst
+ const int nr = ne1;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ ggml_bf16_t * const wdata = (ggml_bf16_t *) params->wdata + 0;
+ ggml_bf16_t * const wdata_src = wdata + nk;
+
+ for (int i1 = ir0; i1 < ir1; i1++) {
+ float * dst_data = (float *)((char *) dst->data + i1*nb1);
+ ggml_bf16_t * wdata_kernel = wdata + i1*ne02*ne00;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ const int i1n = i10*ne11;
+ for (int i00 = 0; i00 < ne00; i00++) {
+ float v = 0;
+ ggml_vec_dot_bf16(ne02, &v, 0,
+ (ggml_bf16_t *) wdata_src + i1n, 0,
+ (ggml_bf16_t *) wdata_kernel + i00*ne02, 0, 1);
+ dst_data[i10*s0 + i00] += v;
+ }
+ }
+ }
+}
+
static void ggml_compute_forward_conv_transpose_1d_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -13309,6 +14248,10 @@ static void ggml_compute_forward_conv_transpose_1d(
{
ggml_compute_forward_conv_transpose_1d_f16_f32(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_conv_transpose_1d_bf16_f32(params, dst);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_conv_transpose_1d_f32(params, dst);
@@ -13323,7 +14266,95 @@ static void ggml_compute_forward_conv_transpose_1d(
// src0: kernel [OC, IC, KH, KW]
// src1: image [N, IC, IH, IW]
// dst: result [N, OH, OW, IC*KH*KW]
-static void ggml_compute_forward_im2col_f32(
+static void ggml_compute_forward_im2col_f32(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * src0 = dst->src[0];
+ const struct ggml_tensor * src1 = dst->src[1];
+
+ GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
+
+ int64_t t0 = ggml_perf_time_us();
+ UNUSED(t0);
+
+ GGML_TENSOR_BINARY_OP_LOCALS;
+
+ const int32_t s0 = ((const int32_t *)(dst->op_params))[0];
+ const int32_t s1 = ((const int32_t *)(dst->op_params))[1];
+ const int32_t p0 = ((const int32_t *)(dst->op_params))[2];
+ const int32_t p1 = ((const int32_t *)(dst->op_params))[3];
+ const int32_t d0 = ((const int32_t *)(dst->op_params))[4];
+ const int32_t d1 = ((const int32_t *)(dst->op_params))[5];
+ const bool is_2D = ((const int32_t *)(dst->op_params))[6] == 1;
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t N = is_2D ? ne13 : ne12;
+ const int64_t IC = is_2D ? ne12 : ne11;
+ const int64_t IH = is_2D ? ne11 : 1;
+ const int64_t IW = ne10;
+
+ const int64_t KH = is_2D ? ne01 : 1;
+ const int64_t KW = ne00;
+
+ const int64_t OH = is_2D ? ne2 : 1;
+ const int64_t OW = ne1;
+
+ int ofs0 = is_2D ? nb13 : nb12;
+ int ofs1 = is_2D ? nb12 : nb11;
+
+ GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb10 == sizeof(float));
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ return;
+ }
+
+ if (params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
+ {
+ float * const wdata = (float *) dst->data;
+
+ for (int64_t in = 0; in < N; in++) {
+ for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
+ for (int64_t iow = 0; iow < OW; iow++) {
+ for (int64_t iic = ith; iic < IC; iic += nth) {
+
+ // micro kernel
+ float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
+
+ for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
+ for (int64_t ikw = 0; ikw < KW; ikw++) {
+ const int64_t iiw = iow*s0 + ikw*d0 - p0;
+ const int64_t iih = ioh*s1 + ikh*d1 - p1;
+
+ if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
+ } else {
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+
+// src0: kernel [OC, IC, KH, KW]
+// src1: image [N, IC, IH, IW]
+// dst: result [N, OH, OW, IC*KH*KW]
+static void ggml_compute_forward_im2col_f16(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -13332,7 +14363,7 @@ static void ggml_compute_forward_im2col_f32(
GGML_ASSERT(src0->type == GGML_TYPE_F16);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
- GGML_ASSERT( dst->type == GGML_TYPE_F32);
+ GGML_ASSERT( dst->type == GGML_TYPE_F16);
int64_t t0 = ggml_perf_time_us();
UNUSED(t0);
@@ -13377,7 +14408,7 @@ static void ggml_compute_forward_im2col_f32(
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
{
- float * const wdata = (float *) dst->data;
+ ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
for (int64_t in = 0; in < N; in++) {
for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
@@ -13385,7 +14416,7 @@ static void ggml_compute_forward_im2col_f32(
for (int64_t iic = ith; iic < IC; iic += nth) {
// micro kernel
- float * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
@@ -13396,7 +14427,7 @@ static void ggml_compute_forward_im2col_f32(
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
} else {
- dst_data[iic*(KH*KW) + ikh*KW + ikw] = (src_data[iih*IW + iiw]);
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);
}
}
}
@@ -13407,20 +14438,19 @@ static void ggml_compute_forward_im2col_f32(
}
}
-
// src0: kernel [OC, IC, KH, KW]
// src1: image [N, IC, IH, IW]
// dst: result [N, OH, OW, IC*KH*KW]
-static void ggml_compute_forward_im2col_f16(
+static void ggml_compute_forward_im2col_bf16(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];
- GGML_ASSERT(src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == GGML_TYPE_BF16);
GGML_ASSERT(src1->type == GGML_TYPE_F32);
- GGML_ASSERT( dst->type == GGML_TYPE_F16);
+ GGML_ASSERT( dst->type == GGML_TYPE_BF16);
int64_t t0 = ggml_perf_time_us();
UNUSED(t0);
@@ -13452,7 +14482,7 @@ static void ggml_compute_forward_im2col_f16(
int ofs0 = is_2D ? nb13 : nb12;
int ofs1 = is_2D ? nb12 : nb11;
- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nb00 == sizeof(ggml_bf16_t));
GGML_ASSERT(nb10 == sizeof(float));
if (params->type == GGML_TASK_TYPE_INIT) {
@@ -13465,7 +14495,7 @@ static void ggml_compute_forward_im2col_f16(
// im2col: [N, IC, IH, IW] => [N, OH, OW, IC*KH*KW]
{
- ggml_fp16_t * const wdata = (ggml_fp16_t *) dst->data;
+ ggml_bf16_t * const wdata = (ggml_bf16_t *) dst->data;
for (int64_t in = 0; in < N; in++) {
for (int64_t ioh = 0; ioh < OH; ioh++) { // 1
@@ -13473,7 +14503,7 @@ static void ggml_compute_forward_im2col_f16(
for (int64_t iic = ith; iic < IC; iic += nth) {
// micro kernel
- ggml_fp16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
+ ggml_bf16_t * dst_data = wdata + (in*OH*OW + ioh*OW + iow)*(IC*KH*KW); // [IC, KH, KW]
const float * const src_data = (float *)((char *) src1->data + in*ofs0 + iic*ofs1); // [IH, IW]
for (int64_t ikh = 0; ikh < KH; ikh++) { // 1
@@ -13482,9 +14512,9 @@ static void ggml_compute_forward_im2col_f16(
const int64_t iih = ioh*s1 + ikh*d1 - p1;
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
- dst_data[iic*(KH*KW) + ikh*KW + ikw] = 0;
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_BF16(0);
} else {
- dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_FP16(src_data[iih*IW + iiw]);
+ dst_data[iic*(KH*KW) + ikh*KW + ikw] = GGML_FP32_TO_BF16(src_data[iih*IW + iiw]);
}
}
}
@@ -13503,6 +14533,10 @@ static void ggml_compute_forward_im2col(
{
ggml_compute_forward_im2col_f16(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_im2col_bf16(params, dst);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_im2col_f32(params, dst);
@@ -14229,25 +15263,253 @@ static void ggml_compute_forward_flash_attn_f32(
#endif
}
- for (int64_t ic = 0; ic < nev1; ++ic) {
- // dst indices
- const int i1 = iq1;
- const int i2 = iq2;
- const int i3 = iq3;
+ for (int64_t ic = 0; ic < nev1; ++ic) {
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
+
+ // v indices
+ const int iv2 = iq2 % nev2;
+ const int iv3 = iq3;
+
+ ggml_vec_dot_f32(masked_begin,
+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
+ (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,
+ S, 0, 1);
+ }
+ }
+}
+
+static void ggml_compute_forward_flash_attn_f16(
+ const struct ggml_compute_params * params,
+ const bool masked,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * q = dst->src[0];
+ const struct ggml_tensor * k = dst->src[1];
+ const struct ggml_tensor * v = dst->src[2];
+
+ int64_t t0 = ggml_perf_time_us();
+ UNUSED(t0);
+
+ GGML_TENSOR_LOCALS(int64_t, neq, q, ne)
+ GGML_TENSOR_LOCALS(size_t, nbq, q, nb)
+ GGML_TENSOR_LOCALS(int64_t, nek, k, ne)
+ GGML_TENSOR_LOCALS(size_t, nbk, k, nb)
+ GGML_TENSOR_LOCALS(int64_t, nev, v, ne)
+ GGML_TENSOR_LOCALS(size_t, nbv, v, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t D = neq0;
+ const int64_t N = neq1;
+ const int64_t P = nek1 - N;
+ const int64_t M = P + N;
+
+ const int Mup = ggml_up(M, GGML_SOFT_MAX_UNROLL);
+
+ GGML_ASSERT(ne0 == D);
+ GGML_ASSERT(ne1 == N);
+ GGML_ASSERT(P >= 0);
+
+ GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));
+
+ GGML_ASSERT(neq0 == D);
+ GGML_ASSERT(nek0 == D);
+ GGML_ASSERT(nev1 == D);
+
+ GGML_ASSERT(neq1 == N);
+ GGML_ASSERT(nek1 == N + P);
+ GGML_ASSERT(nev1 == D);
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ return;
+ }
+
+ if (params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // parallelize by q rows using ggml_vec_dot_f32
+
+ // total rows in q
+ const int nr = neq1*neq2*neq3;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ const float scale = 1.0f/sqrtf(D);
+
+ //printf("P=%d N=%d D=%d ir0=%d ir1=%d scale = %f\n", P, N, D, ir0, ir1, scale);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // q indices
+ const int iq3 = ir/(neq2*neq1);
+ const int iq2 = (ir - iq3*neq2*neq1)/neq1;
+ const int iq1 = (ir - iq3*neq2*neq1 - iq2*neq1);
+
+ float * S = (float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32);
+
+ for (int i = M; i < Mup; ++i) {
+ S[i] = -INFINITY;
+ }
+
+ if (GGML_VEC_DOT_UNROLL > 2 || nek1 % GGML_VEC_DOT_UNROLL != 0) {
+ for (int64_t ic = 0; ic < nek1; ++ic) {
+ // k indices
+ const int ik3 = iq3;
+ const int ik2 = iq2 % nek2;
+ const int ik1 = ic;
+
+ // S indices
+ const int i1 = ik1;
+
+ ggml_vec_dot_f16(neq0,
+ S + i1, 0,
+ (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
+ (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);
+ }
+ } else {
+ for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) {
+ // k indices
+ const int ik3 = iq3;
+ const int ik2 = iq2 % nek2;
+ const int ik1 = ic;
+
+ // S indices
+ const int i1 = ik1;
+
+ ggml_vec_dot_f16_unroll(neq0, nbk1,
+ S + i1,
+ ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),
+ (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));
+ }
+ }
+
+ // scale
+ ggml_vec_scale_f32(nek1, S, scale);
+
+ if (masked) {
+ for (int64_t i = P; i < M; i++) {
+ if (i > P + iq1) {
+ S[i] = -INFINITY;
+ }
+ }
+ }
+
+ // softmax
+ // todo: exclude known -INF S[..] values from max and loop, assuming their results to be zero.
+ // dont forget to set their S values to zero
+ {
+ float max = -INFINITY;
+ ggml_vec_max_f32(M, &max, S);
+
+ ggml_float sum = 0.0;
+ {
+#ifdef GGML_SOFT_MAX_ACCELERATE
+ max = -max;
+ vDSP_vsadd(S, 1, &max, S, 1, Mup);
+ vvexpf(S, S, &Mup);
+ ggml_vec_sum_f32(Mup, &sum, S);
+#else
+ uint16_t scvt[GGML_SOFT_MAX_UNROLL];
+ ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };
+
+ for (int i = 0; i < Mup; i += GGML_SOFT_MAX_UNROLL) {
+ float * SS = S + i;
+
+ for (int j = 0; j < GGML_SOFT_MAX_UNROLL; ++j) {
+ if (SS[j] == -INFINITY) {
+ SS[j] = 0.0f;
+ } else {
+ ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
+ memcpy(&scvt[j], &s, sizeof(uint16_t));
+ const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]);
+ sump[j] += (ggml_float)val;
+ SS[j] = val;
+ }
+ }
+ }
+
+ for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {
+ sum += sump[i];
+ }
+#endif
+ }
+
+ assert(sum > 0.0);
+
+ sum = 1.0/sum;
+ ggml_vec_scale_f32(M, S, sum);
+
+#ifndef NDEBUG
+ for (int i = 0; i < M; ++i) {
+ assert(!isnan(S[i]));
+ assert(!isinf(S[i]));
+ }
+#endif
+ }
+
+ ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);
+
+ for (int64_t i = 0; i < M; i++) {
+ S16[i] = GGML_FP32_TO_FP16(S[i]);
+ }
+
+ // todo: exclude known zero S[..] values from dot (reducing nev0 and increasing begin of v and S16).
+ if (GGML_VEC_DOT_UNROLL == 1 || (nev1 % GGML_VEC_DOT_UNROLL != 0)) {
+ for (int64_t ic = 0; ic < nev1; ++ic) {
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
+
+ // v indices
+ const int iv2 = iq2 % nev2;
+ const int iv3 = iq3;
- // v indices
- const int iv2 = iq2 % nev2;
- const int iv3 = iq3;
+ ggml_vec_dot_f16(nev0,
+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
+ (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,
+ S16, 0, 1);
+ }
+ } else {
+ for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) {
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
- ggml_vec_dot_f32(masked_begin,
- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
- (float *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,
- S, 0, 1);
+ // v indices
+ const int iv2 = iq2 % nev2;
+ const int iv3 = iq3;
+
+ ggml_vec_dot_f16_unroll(nev0, nbv1,
+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
+ ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),
+ S16);
+ }
}
}
}
-static void ggml_compute_forward_flash_attn_f16(
+static void ggml_compute_forward_flash_attn_bf16(
const struct ggml_compute_params * params,
const bool masked,
struct ggml_tensor * dst) {
@@ -14282,9 +15544,9 @@ static void ggml_compute_forward_flash_attn_f16(
GGML_ASSERT(ne1 == N);
GGML_ASSERT(P >= 0);
- GGML_ASSERT(nbq0 == sizeof(ggml_fp16_t));
- GGML_ASSERT(nbk0 == sizeof(ggml_fp16_t));
- GGML_ASSERT(nbv0 == sizeof(ggml_fp16_t));
+ GGML_ASSERT(nbq0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nbk0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nbv0 == sizeof(ggml_bf16_t));
GGML_ASSERT(neq0 == D);
GGML_ASSERT(nek0 == D);
@@ -14336,36 +15598,19 @@ static void ggml_compute_forward_flash_attn_f16(
S[i] = -INFINITY;
}
- if (GGML_VEC_DOT_UNROLL > 2 || nek1 % GGML_VEC_DOT_UNROLL != 0) {
- for (int64_t ic = 0; ic < nek1; ++ic) {
- // k indices
- const int ik3 = iq3;
- const int ik2 = iq2 % nek2;
- const int ik1 = ic;
-
- // S indices
- const int i1 = ik1;
-
- ggml_vec_dot_f16(neq0,
- S + i1, 0,
- (ggml_fp16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
- (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);
- }
- } else {
- for (int64_t ic = 0; ic < nek1; ic += GGML_VEC_DOT_UNROLL) {
- // k indices
- const int ik3 = iq3;
- const int ik2 = iq2 % nek2;
- const int ik1 = ic;
+ for (int64_t ic = 0; ic < nek1; ++ic) {
+ // k indices
+ const int ik3 = iq3;
+ const int ik2 = iq2 % nek2;
+ const int ik1 = ic;
- // S indices
- const int i1 = ik1;
+ // S indices
+ const int i1 = ik1;
- ggml_vec_dot_f16_unroll(neq0, nbk1,
- S + i1,
- ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)),
- (ggml_fp16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)));
- }
+ ggml_vec_dot_bf16(neq0,
+ S + i1, 0,
+ (ggml_bf16_t *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0,
+ (ggml_bf16_t *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1);
}
// scale
@@ -14388,12 +15633,6 @@ static void ggml_compute_forward_flash_attn_f16(
ggml_float sum = 0.0;
{
-#ifdef GGML_SOFT_MAX_ACCELERATE
- max = -max;
- vDSP_vsadd(S, 1, &max, S, 1, Mup);
- vvexpf(S, S, &Mup);
- ggml_vec_sum_f32(Mup, &sum, S);
-#else
uint16_t scvt[GGML_SOFT_MAX_UNROLL];
ggml_float sump[GGML_SOFT_MAX_UNROLL] = { 0.0 };
@@ -14404,9 +15643,9 @@ static void ggml_compute_forward_flash_attn_f16(
if (SS[j] == -INFINITY) {
SS[j] = 0.0f;
} else {
- ggml_fp16_t s = GGML_FP32_TO_FP16(SS[j] - max);
+ ggml_bf16_t s = GGML_FP32_TO_BF16(SS[j] - max);
memcpy(&scvt[j], &s, sizeof(uint16_t));
- const float val = GGML_FP16_TO_FP32(ggml_table_exp_f16[scvt[j]]);
+ const float val = GGML_BF16_TO_FP32(ggml_table_exp_bf16[scvt[j]]);
sump[j] += (ggml_float)val;
SS[j] = val;
}
@@ -14416,7 +15655,6 @@ static void ggml_compute_forward_flash_attn_f16(
for (int i = 0; i < GGML_SOFT_MAX_UNROLL; i++) {
sum += sump[i];
}
-#endif
}
assert(sum > 0.0);
@@ -14432,45 +15670,26 @@ static void ggml_compute_forward_flash_attn_f16(
#endif
}
- ggml_fp16_t * S16 = (ggml_fp16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);
+ ggml_bf16_t * S16 = (ggml_bf16_t *) ((float *) params->wdata + ith*(2*Mup + CACHE_LINE_SIZE_F32) + Mup);
for (int64_t i = 0; i < M; i++) {
- S16[i] = GGML_FP32_TO_FP16(S[i]);
+ S16[i] = GGML_FP32_TO_BF16(S[i]);
}
- // todo: exclude known zero S[..] values from dot (reducing nev0 and increasing begin of v and S16).
- if (GGML_VEC_DOT_UNROLL == 1 || (nev1 % GGML_VEC_DOT_UNROLL != 0)) {
- for (int64_t ic = 0; ic < nev1; ++ic) {
- // dst indices
- const int i1 = iq1;
- const int i2 = iq2;
- const int i3 = iq3;
-
- // v indices
- const int iv2 = iq2 % nev2;
- const int iv3 = iq3;
-
- ggml_vec_dot_f16(nev0,
- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
- (ggml_fp16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,
- S16, 0, 1);
- }
- } else {
- for (int64_t ic = 0; ic < nev1; ic += GGML_VEC_DOT_UNROLL) {
- // dst indices
- const int i1 = iq1;
- const int i2 = iq2;
- const int i3 = iq3;
+ for (int64_t ic = 0; ic < nev1; ++ic) {
+ // dst indices
+ const int i1 = iq1;
+ const int i2 = iq2;
+ const int i3 = iq3;
- // v indices
- const int iv2 = iq2 % nev2;
- const int iv3 = iq3;
+ // v indices
+ const int iv2 = iq2 % nev2;
+ const int iv3 = iq3;
- ggml_vec_dot_f16_unroll(nev0, nbv1,
- (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)),
- ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)),
- S16);
- }
+ ggml_vec_dot_bf16(nev0,
+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
+ (ggml_bf16_t *) ((char *) v->data + ( ic*nbv1 + iv2*nbv2 + iv3*nbv3)), 0,
+ S16, 0, 1);
}
}
}
@@ -14487,6 +15706,10 @@ static void ggml_compute_forward_flash_attn(
{
ggml_compute_forward_flash_attn_f16(params, masked, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_flash_attn_bf16(params, masked, dst);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_flash_attn_f32(params, masked, dst);
@@ -14635,6 +15858,141 @@ static void ggml_compute_forward_flash_ff_f16(
}
}
+static void ggml_compute_forward_flash_ff_bf16(
+ const struct ggml_compute_params * params,
+ struct ggml_tensor * dst) {
+
+ const struct ggml_tensor * a = dst->src[0]; // BF16
+ const struct ggml_tensor * b0 = dst->src[1]; // BF16 fc_w
+ const struct ggml_tensor * b1 = dst->src[2]; // F32 fc_b
+ const struct ggml_tensor * c0 = dst->src[3]; // BF16 proj_w
+ const struct ggml_tensor * c1 = dst->src[4]; // F32 proj_b
+
+ int64_t t0 = ggml_perf_time_us();
+ UNUSED(t0);
+
+ GGML_TENSOR_LOCALS(int64_t, nea, a, ne)
+ GGML_TENSOR_LOCALS(size_t, nba, a, nb)
+ GGML_TENSOR_LOCALS(int64_t, neb0, b0, ne)
+ GGML_TENSOR_LOCALS(size_t, nbb0, b0, nb)
+ GGML_TENSOR_LOCALS(int64_t, neb1, b1, ne)
+ GGML_TENSOR_LOCALS(size_t, nbb1, b1, nb)
+ GGML_TENSOR_LOCALS(int64_t, nec0, c0, ne)
+ GGML_TENSOR_LOCALS(size_t, nbc0, c0, nb)
+ GGML_TENSOR_LOCALS(int64_t, nec1, c1, ne)
+ GGML_TENSOR_LOCALS(size_t, nbc1, c1, nb)
+ GGML_TENSOR_LOCALS(int64_t, ne, dst, ne)
+ GGML_TENSOR_LOCALS(size_t, nb, dst, nb)
+
+ const int ith = params->ith;
+ const int nth = params->nth;
+
+ const int64_t D = nea0;
+ //const int64_t N = nea1;
+ const int64_t M = neb01;
+
+ GGML_ASSERT(ne0 == nea0);
+ GGML_ASSERT(ne1 == nea1);
+ GGML_ASSERT(ne2 == nea2);
+
+ GGML_ASSERT(nba0 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nbb00 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nbb10 == sizeof(float));
+ GGML_ASSERT(nbc00 == sizeof(ggml_bf16_t));
+ GGML_ASSERT(nbc10 == sizeof(float));
+
+ GGML_ASSERT(neb00 == D);
+ GGML_ASSERT(neb01 == M);
+ GGML_ASSERT(neb10 == M);
+ GGML_ASSERT(neb11 == 1);
+
+ GGML_ASSERT(nec00 == M);
+ GGML_ASSERT(nec01 == D);
+ GGML_ASSERT(nec10 == D);
+ GGML_ASSERT(nec11 == 1);
+
+ // dst cannot be transposed or permuted
+ GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb0 <= nb1);
+ GGML_ASSERT(nb1 <= nb2);
+ GGML_ASSERT(nb2 <= nb3);
+
+ if (params->type == GGML_TASK_TYPE_INIT) {
+ return;
+ }
+
+ if (params->type == GGML_TASK_TYPE_FINALIZE) {
+ return;
+ }
+
+ // parallelize by a rows using ggml_vec_dot_f32
+
+ // total rows in a
+ const int nr = nea1*nea2*nea3;
+
+ // rows per thread
+ const int dr = (nr + nth - 1)/nth;
+
+ // row range for this thread
+ const int ir0 = dr*ith;
+ const int ir1 = MIN(ir0 + dr, nr);
+
+ for (int ir = ir0; ir < ir1; ++ir) {
+ // a indices
+ const int ia3 = ir/(nea2*nea1);
+ const int ia2 = (ir - ia3*nea2*nea1)/nea1;
+ const int ia1 = (ir - ia3*nea2*nea1 - ia2*nea1);
+
+ float * S = (float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32);
+
+ for (int64_t ic = 0; ic < neb01; ++ic) {
+ // b0 indices
+ const int ib03 = ia3;
+ const int ib02 = ia2;
+ const int ib01 = ic;
+
+ // S indices
+ const int i1 = ib01;
+
+ ggml_vec_dot_bf16(nea0,
+ S + i1, 0,
+ (ggml_bf16_t *) ((char *) b0->data + (ib01*nbb01 + ib02*nbb02 + ib03*nbb03)), 0,
+ (ggml_bf16_t *) ((char *) a->data + ( ia1*nba1 + ia2*nba2 + ia3*nba3)), 0, 1);
+ }
+
+ ggml_vec_add_f32(neb01, S, S, (float *) b1->data);
+ //ggml_vec_gelu_f32(neb01, S, S);
+
+ ggml_bf16_t * S16 = (ggml_bf16_t *) ((float *) params->wdata + ith*(2*M + CACHE_LINE_SIZE_F32) + M);
+
+ for (int64_t i = 0; i < M; i++) {
+ S16[i] = GGML_FP32_TO_BF16(S[i]);
+ }
+
+ ggml_vec_gelu_bf16(neb01, S16, S16);
+
+ {
+ // dst indices
+ const int i1 = ia1;
+ const int i2 = ia2;
+ const int i3 = ia3;
+
+ for (int64_t ic = 0; ic < nec01; ++ic) {
+
+ ggml_vec_dot_bf16(neb01,
+ (float *) ((char *) dst->data + (ic*nb0 + i1*nb1 + i2*nb2 + i3*nb3)), 0,
+ (ggml_bf16_t *) ((char *) c0->data + ( ic*nbc01 + i2*nbc02 + i3*nbc03)), 0,
+ S16, 0, 1);
+ }
+
+ ggml_vec_add_f32(nec01,
+ (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),
+ (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3)),
+ (float *) c1->data);
+ }
+ }
+}
+
static void ggml_compute_forward_flash_ff(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
@@ -14646,6 +16004,10 @@ static void ggml_compute_forward_flash_ff(
{
ggml_compute_forward_flash_ff_f16(params, dst);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_flash_ff_bf16(params, dst);
+ } break;
case GGML_TYPE_F32:
{
GGML_ASSERT(false); // TODO
@@ -15517,6 +16879,7 @@ static void ggml_compute_forward_get_rel_pos(
switch (src0->type) {
case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
{
ggml_compute_forward_get_rel_pos_f16(params, dst);
} break;
@@ -18468,7 +19831,8 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
const int64_t ne10 = node->src[1]->ne[0]; // L
const int64_t ne11 = node->src[1]->ne[1]; // Cin
- if (node->src[0]->type == GGML_TYPE_F16 &&
+ if ((node->src[0]->type == GGML_TYPE_F16 ||
+ node->src[0]->type == GGML_TYPE_BF16) &&
node->src[1]->type == GGML_TYPE_F32) {
cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
cur += sizeof(ggml_fp16_t)*ne10*ne11;
@@ -18504,6 +19868,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
} else if (node->src[1]->type == GGML_TYPE_F16) {
cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)
cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2
+ } else if (node->src[1]->type == GGML_TYPE_BF16) {
+ cur = sizeof(float)*ne11*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*ne11*n_tasks; // this is overestimated by x2
}
} break;
case GGML_OP_FLASH_FF:
@@ -18514,6 +19881,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
} else if (node->src[1]->type == GGML_TYPE_F16) {
cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)
cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2
+ } else if (node->src[1]->type == GGML_TYPE_BF16) {
+ cur = sizeof(float)*node->src[1]->ne[1]*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*node->src[1]->ne[1]*n_tasks; // this is overestimated by x2
}
} break;
case GGML_OP_FLASH_ATTN_BACK:
@@ -18527,6 +19897,9 @@ struct ggml_cplan ggml_graph_plan(const struct ggml_cgraph * cgraph, int n_threa
} else if (node->src[1]->type == GGML_TYPE_F16) {
cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
+ } else if (node->src[1]->type == GGML_TYPE_BF16) {
+ cur = sizeof(float)*mxDn*n_tasks; // TODO: this can become (n_tasks-1)
+ cur += sizeof(float)*mxDn*n_tasks; // this is overestimated by x2
}
} break;
@@ -19318,7 +20691,9 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph
if (node->type == GGML_TYPE_I8 || node->type == GGML_TYPE_I16 || node->type == GGML_TYPE_I32) {
fprintf(fp, "%d", ggml_get_i32_1d(node, j));
}
- else if (node->type == GGML_TYPE_F32 || node->type == GGML_TYPE_F16) {
+ else if (node->type == GGML_TYPE_F32 ||
+ node->type == GGML_TYPE_F16 ||
+ node->type == GGML_TYPE_BF16) {
fprintf(fp, "%.1e", (double)ggml_get_f32_1d(node, j));
}
else {
@@ -20373,6 +21748,12 @@ size_t ggml_quantize_chunk(
ggml_fp32_to_fp16_row(src + start, (ggml_fp16_t *)dst + start, n);
result = n * elemsize;
} break;
+ case GGML_TYPE_BF16:
+ {
+ size_t elemsize = sizeof(ggml_bf16_t);
+ ggml_fp32_to_bf16_row(src + start, (ggml_bf16_t *)dst + start, n);
+ result = n * elemsize;
+ } break;
case GGML_TYPE_F32:
{
size_t elemsize = sizeof(float);
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index 8977645f..99d61411 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -371,6 +371,7 @@ extern "C" {
GGML_TYPE_I32 = 26,
GGML_TYPE_I64 = 27,
GGML_TYPE_F64 = 28,
+ GGML_TYPE_BF16 = 29,
GGML_TYPE_COUNT,
};
@@ -410,6 +411,7 @@ extern "C" {
GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors
GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors
+ GGML_FTYPE_MOSTLY_BF16 = 23, // except 1d tensors
};
// available tensor operations:
@@ -2364,6 +2366,90 @@ extern "C" {
GGML_API int ggml_cpu_has_vsx (void);
GGML_API int ggml_cpu_has_matmul_int8(void);
+ /**
+ * Google Brain 16-bit floating point number.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───┐
+ * 0b0000000000000000 bfloat16
+ *
+ * Since bf16 has the same number of exponent bits as a 32bit float,
+ * encoding and decoding numbers becomes relatively straightforward.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌──┴───┐┌─┴───────────────────┐
+ * 0b00000000000000000000000000000000 IEEE binary32
+ *
+ * For comparison, the standard fp16 format has fewer exponent bits.
+ *
+ * ┌sign
+ * │
+ * │ ┌exponent
+ * │ │
+ * │ │ ┌mantissa
+ * │ │ │
+ * │┌─┴─┐┌─┴──────┐
+ * 0b0000000000000000 IEEE binary16
+ *
+ * So be warned that converting between them yields a ten bit float.
+ *
+ * @see IEEE 754-2008
+ */
+ typedef struct {
+ uint16_t x;
+ } ggml_bf16_t;
+
+ /**
+ * Converts bfloat16 to float32.
+ */
+ static inline float ggml_bf16_to_fp32(ggml_bf16_t h) {
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.i = (uint32_t)h.x << 16;
+ return u.f;
+ }
+
+ /**
+ * Converts float32 to bfloat16.
+ *
+ * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.
+ * Subnormals shall be flushed to zero, and NANs will be quiet.
+ * This code should vectorize nicely if using modern compilers.
+ */
+ static inline ggml_bf16_t ggml_fp32_to_bf16(float s) {
+ ggml_bf16_t h;
+ union {
+ float f;
+ uint32_t i;
+ } u;
+ u.f = s;
+ if ((u.i & 0x7fffffff) > 0x7f800000) { /* nan */
+ h.x = (u.i >> 16) | 64; /* force to quiet */
+ return h;
+ }
+ if (!(u.i & 0x7f800000)) { /* subnormal */
+ h.x = (u.i & 0x80000000) >> 16; /* flush to zero */
+ return h;
+ }
+ h.x = (u.i + (0x7fff + ((u.i >> 16) & 1))) >> 16;
+ return h;
+ }
+
+ GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int n);
+ GGML_API void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int n);
+
//
// Internal types and functions exposed for tests and benchmarks
//
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index cefe2f2a..1d2cf233 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -2683,6 +2683,7 @@ struct llama_model_loader {
switch (type_max) {
case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;
case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;
+ case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;
case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;
case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;
case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;
@@ -3034,6 +3035,7 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {
switch (ftype) {
case LLAMA_FTYPE_ALL_F32: return "all F32";
case LLAMA_FTYPE_MOSTLY_F16: return "F16";
+ case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:
@@ -4961,6 +4963,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
|| !(
model.ftype == LLAMA_FTYPE_ALL_F32 ||
model.ftype == LLAMA_FTYPE_MOSTLY_F16 ||
+ model.ftype == LLAMA_FTYPE_MOSTLY_BF16 ||
model.ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ||
model.ftype == LLAMA_FTYPE_MOSTLY_Q4_1
)
@@ -12063,6 +12066,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
case LLAMA_FTYPE_MOSTLY_Q5_0: default_type = GGML_TYPE_Q5_0; break;
case LLAMA_FTYPE_MOSTLY_Q5_1: default_type = GGML_TYPE_Q5_1; break;
case LLAMA_FTYPE_MOSTLY_Q8_0: default_type = GGML_TYPE_Q8_0; break;
+ case LLAMA_FTYPE_MOSTLY_BF16: default_type = GGML_TYPE_BF16; break;
case LLAMA_FTYPE_MOSTLY_F16: default_type = GGML_TYPE_F16; break;
case LLAMA_FTYPE_ALL_F32: default_type = GGML_TYPE_F32; break;
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 9d6f458d..230f820b 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -119,6 +119,7 @@ extern "C" {
LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors
LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors
LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors
+ LLAMA_FTYPE_MOSTLY_BF16 = 31, // except 1d tensors
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
};
diff --git a/llama.cpp/quantize/quantize.1 b/llama.cpp/quantize/quantize.1
index 7efd9ea9..e92f10da 100644
--- a/llama.cpp/quantize/quantize.1
+++ b/llama.cpp/quantize/quantize.1
@@ -76,6 +76,8 @@ The following quantization types are available:
.It
   7 Q8_0 6.70G +0.0004 ppl @ LLaMA-v1-7B
.It
+  31 BF16 Google Brain Floating Point
+.It
   1 F16 13.00G @ 7B
.It
   0 F32 26.00G @ 7B
diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cpp
index 76ed165b..940cd991 100644
--- a/llama.cpp/quantize/quantize.cpp
+++ b/llama.cpp/quantize/quantize.cpp
@@ -51,6 +51,7 @@ static const std::vector<struct quant_option> QUANT_OPTIONS = {
{ "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 4.45G, +0.0122 ppl @ LLaMA-v1-7B", },
{ "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 5.15G, +0.0008 ppl @ LLaMA-v1-7B", },
{ "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 6.70G, +0.0004 ppl @ LLaMA-v1-7B", },
+ { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "Google Brain Floating Point", },
{ "F16", LLAMA_FTYPE_MOSTLY_F16, "13.00G @ 7B", },
{ "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },
// Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 6934e553..a9e95b51 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -45,6 +45,9 @@ o/$(MODE)/llamafile: \
o/$(MODE)/llamafile/sgemm_sss_avx.o: private TARGET_ARCH += -Xx86_64-mavx
o/$(MODE)/llamafile/sgemm_sss_fma.o: private TARGET_ARCH += -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_bss_avx2.o: private TARGET_ARCH += -Xx86_64-mavx2 -Xx86_64-mfma
+o/$(MODE)/llamafile/sgemm_bss_avx512.o: private TARGET_ARCH += -Xx86_64-mavx512f
+o/$(MODE)/llamafile/sgemm_bss_avx512bf16.o: private TARGET_ARCH += -Xx86_64-mavx512bf16 -Xx86_64-mavx512vl
o/$(MODE)/llamafile/sgemm_sss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
o/$(MODE)/llamafile/sgemm_hss_f16c.o: private TARGET_ARCH += -Xx86_64-mfma -Xx86_64-mf16c
o/$(MODE)/llamafile/sgemm_hss_avx512f.o: private TARGET_ARCH += -Xx86_64-mavx512f
diff --git a/llamafile/float.h b/llamafile/float.h
index 76a52b6d..82aaa35e 100644
--- a/llamafile/float.h
+++ b/llamafile/float.h
@@ -12,6 +12,14 @@ inline unsigned toint(float f) {
return u.i;
}
+inline float tofloat(unsigned i) {
+ union {
+ unsigned i;
+ float f;
+ } u = {i};
+ return u.f;
+}
+
inline bool isnan(float f) {
return (toint(f) & 0x7fffffff) > 0x7f800000;
}
@@ -20,7 +28,7 @@ inline bool isinf(float f) {
return (toint(f) & 0x7fffffff) == 0x7f800000;
}
-inline unsigned signbit(float f) {
+inline unsigned sign(float f) {
return toint(f) & 0x80000000u;
}
diff --git a/llamafile/half.h b/llamafile/half.h
index ac6545bb..acfeb55c 100644
--- a/llamafile/half.h
+++ b/llamafile/half.h
@@ -26,7 +26,7 @@ inline bool isinf(half f) {
return (toint(f) & 0x7fff) == 0x7c00;
}
-inline int signbit(half f) {
+inline int sign(half f) {
return toint(f) & 0x8000;
}
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 1f5694a4..bb6046e9 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -31,6 +31,7 @@ void llamafile_launch_browser(const char *);
int llamafile_get_math_cpu_count(void);
extern bool FLAG_unsecure;
+extern const bool have_avx512bf16;
#define LLAMAFILE_GPU_ERROR -2
#define LLAMAFILE_GPU_DISABLE -1
diff --git a/llamafile/sgemm.cpp b/llamafile/sgemm.cpp
index 5f62b721..1656bca8 100644
--- a/llamafile/sgemm.cpp
+++ b/llamafile/sgemm.cpp
@@ -19,10 +19,26 @@
#include "llamafile.h"
#include <cassert>
#include <cosmo.h>
+#include <cpuid.h>
#include <libc/sysv/consts/hwcap.h>
#include <sys/auxv.h>
+// TODO(jart): Delete when cosmocc 3.3.3 goes live.
+static bool check_avx512bf16(void) {
+#ifdef __x86_64__
+ // available on cooperlake, zen4, etc.
+ int abcd[4];
+ __cpuidex(abcd, 1, 0);
+ if (abcd[0] >= 7) {
+ __cpuidex(abcd, 7, 1);
+ return !!(abcd[0] & (1u << 5));
+ }
+#endif
+ return false;
+}
+
static const long hwcap = getauxval(AT_HWCAP);
+const bool have_avx512bf16 = check_avx512bf16();
/**
* Performs optimized matrix multiplication on CPU.
@@ -112,6 +128,24 @@ bool llamafile_sgemm(int m, int n, int k, const void *A, int lda, const void *B,
#endif
return false;
+ case GGML_TYPE_BF16:
+#ifdef __x86_64__
+ if (Btype != GGML_TYPE_F32)
+ return false;
+ if (have_avx512bf16 && !(k % 16))
+ return llamafile_sgemm_bss_avx512bf16(m, n, k, (const ggml_bf16_t *)A, lda,
+ (const float *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ if (X86_HAVE(AVX512F) && !(k % 16))
+ return llamafile_sgemm_bss_avx512(m, n, k, (const ggml_bf16_t *)A, lda,
+ (const float *)B, ldb, (float *)C, ldc, ith, nth,
+ task);
+ if (X86_HAVE(AVX2) && X86_HAVE(FMA) && !(k % 8))
+ return llamafile_sgemm_bss_avx2(m, n, k, (const ggml_bf16_t *)A, lda, (const float *)B,
+ ldb, (float *)C, ldc, ith, nth, task);
+#endif
+ return false;
+
case GGML_TYPE_Q8_0:
if (Btype != GGML_TYPE_Q8_0)
return false;
diff --git a/llamafile/sgemm.h b/llamafile/sgemm.h
index b7b45f8d..15e0f883 100644
--- a/llamafile/sgemm.h
+++ b/llamafile/sgemm.h
@@ -23,6 +23,13 @@ bool llamafile_sgemm_hss_neon(int, int, int, const unsigned short *, int, const
bool llamafile_sgemm_hhs_neon(int, int, int, const unsigned short *, int, const unsigned short *,
int, float *, int, int, int, int);
+bool llamafile_sgemm_bss_avx2(int, int, int, const ggml_bf16_t *, int, const float *, int, float *,
+ int, int, int, int);
+bool llamafile_sgemm_bss_avx512(int, int, int, const ggml_bf16_t *, int, const float *, int,
+ float *, int, int, int, int);
+bool llamafile_sgemm_bss_avx512bf16(int, int, int, const ggml_bf16_t *, int, const float *, int,
+ float *, int, int, int, int);
+
bool llamafile_sgemm_q0q0s_avx512vnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *,
int, float *, int, int, int, int);
bool llamafile_sgemm_q0q0s_avxvnni(int, int, int, const block_q8_0 *, int, const block_q8_0 *, int,
diff --git a/llamafile/sgemm_bss_avx2.cpp b/llamafile/sgemm_bss_avx2.cpp
new file mode 100644
index 00000000..9d22305d
--- /dev/null
+++ b/llamafile/sgemm_bss_avx2.cpp
@@ -0,0 +1,55 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+#include <immintrin.h>
+
+#define KN 8
+
+typedef __m256 V;
+typedef __m256 D;
+typedef ggml_bf16_t TA;
+typedef float TB;
+typedef float TC;
+
+static inline __m256 zero() {
+ return _mm256_setzero_ps();
+}
+
+static inline __m256 load(const float *p) {
+ return _mm256_loadu_ps(p);
+}
+
+static inline __m256 load(const ggml_bf16_t *p) {
+ return _mm256_castsi256_ps(
+ _mm256_slli_epi32(_mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i *)p)), 16));
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_bss_avx2(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_bss_avx512.cpp b/llamafile/sgemm_bss_avx512.cpp
new file mode 100644
index 00000000..9fcebf40
--- /dev/null
+++ b/llamafile/sgemm_bss_avx512.cpp
@@ -0,0 +1,55 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+#include <immintrin.h>
+
+#define KN 16
+
+typedef __m512 V;
+typedef __m512 D;
+typedef ggml_bf16_t TA;
+typedef float TB;
+typedef float TC;
+
+static inline __m512 zero() {
+ return _mm512_setzero_ps();
+}
+
+static inline __m512 load(const float *p) {
+ return _mm512_loadu_ps(p);
+}
+
+static inline __m512 load(const ggml_bf16_t *p) {
+ return _mm512_castsi512_ps(
+ _mm512_slli_epi32(_mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i *)p)), 16));
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_bss_avx512(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_bss_avx512bf16.cpp b/llamafile/sgemm_bss_avx512bf16.cpp
new file mode 100644
index 00000000..5e924903
--- /dev/null
+++ b/llamafile/sgemm_bss_avx512bf16.cpp
@@ -0,0 +1,58 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef __x86_64__
+
+#include "sgemm.h"
+#include <immintrin.h>
+
+#define KN 16
+
+typedef __m256 D;
+typedef __m256bh V;
+typedef ggml_bf16_t TA;
+typedef float TB;
+typedef float TC;
+
+static inline __m256 zero() {
+ return _mm256_setzero_ps();
+}
+
+static inline __m256bh load(const ggml_bf16_t *p) {
+ return (__m256bh)_mm256_loadu_ps((const float *)p);
+}
+
+static inline __m256bh load(const float *p) {
+ return _mm512_cvtneps_pbh(_mm512_loadu_ps(p));
+}
+
+static inline __m256 madd(__m256bh x, __m256bh y, __m256 z) {
+ return _mm256_dpbf16_ps(z, x, y);
+}
+
+#include "sgemmer.inc"
+
+bool llamafile_sgemm_bss_avx512bf16(int m, int n, int k, const TA *A, int lda, const TB *B, int ldb,
+ TC *C, int ldc, int ith, int nth, int task) {
+ if (task != GGML_TASK_TYPE_COMPUTE)
+ return true;
+ SGEMMER tb{k, A, lda, B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+}
+
+#endif // __x86_64__
diff --git a/llamafile/sgemm_hhs_neon.cpp b/llamafile/sgemm_hhs_neon.cpp
index f1f31c2b..f93e1e89 100644
--- a/llamafile/sgemm_hhs_neon.cpp
+++ b/llamafile/sgemm_hhs_neon.cpp
@@ -23,6 +23,7 @@
#define KN 8
typedef float16x8_t V;
+typedef float16x8_t D;
typedef unsigned short TA;
typedef unsigned short TB;
typedef float TC;
diff --git a/llamafile/sgemm_hss_avx512f.cpp b/llamafile/sgemm_hss_avx512f.cpp
index d1584919..7abd5f52 100644
--- a/llamafile/sgemm_hss_avx512f.cpp
+++ b/llamafile/sgemm_hss_avx512f.cpp
@@ -23,6 +23,7 @@
#define KN 16
typedef __m512 V;
+typedef __m512 D;
typedef unsigned short TA;
typedef float TB;
typedef float TC;
@@ -36,7 +37,7 @@ static inline V load(const float *p) {
}
static inline V load(const unsigned short *p) {
- return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(p)));
+ return _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)p));
}
#include "sgemmer.inc"
diff --git a/llamafile/sgemm_hss_f16c.cpp b/llamafile/sgemm_hss_f16c.cpp
index 87bc04cd..14859edc 100644
--- a/llamafile/sgemm_hss_f16c.cpp
+++ b/llamafile/sgemm_hss_f16c.cpp
@@ -23,6 +23,7 @@
#define KN 8
typedef __m256 V;
+typedef __m256 D;
typedef unsigned short TA;
typedef float TB;
typedef float TC;
@@ -36,7 +37,7 @@ static inline V load(const float *p) {
}
static inline V load(const unsigned short *p) {
- return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(p)));
+ return _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)p));
}
#include "sgemmer.inc"
diff --git a/llamafile/sgemm_hss_neon.cpp b/llamafile/sgemm_hss_neon.cpp
index aec6c67d..9ae6e68e 100644
--- a/llamafile/sgemm_hss_neon.cpp
+++ b/llamafile/sgemm_hss_neon.cpp
@@ -23,6 +23,7 @@
#define KN 4
typedef float32x4_t V;
+typedef float32x4_t D;
typedef unsigned short TA;
typedef float TB;
typedef float TC;
diff --git a/llamafile/sgemm_sss_avx.cpp b/llamafile/sgemm_sss_avx.cpp
index 73c4b90f..a7ea9055 100644
--- a/llamafile/sgemm_sss_avx.cpp
+++ b/llamafile/sgemm_sss_avx.cpp
@@ -23,6 +23,7 @@
#define KN 8
typedef __m256 V;
+typedef __m256 D;
typedef float TA;
typedef float TB;
typedef float TC;
diff --git a/llamafile/sgemm_sss_avx512f.cpp b/llamafile/sgemm_sss_avx512f.cpp
index 94b4e234..09eb708f 100644
--- a/llamafile/sgemm_sss_avx512f.cpp
+++ b/llamafile/sgemm_sss_avx512f.cpp
@@ -23,6 +23,7 @@
#define KN 16
typedef __m512 V;
+typedef __m512 D;
typedef float TA;
typedef float TB;
typedef float TC;
diff --git a/llamafile/sgemm_sss_fma.cpp b/llamafile/sgemm_sss_fma.cpp
index 1e925b80..b32e6567 100644
--- a/llamafile/sgemm_sss_fma.cpp
+++ b/llamafile/sgemm_sss_fma.cpp
@@ -23,6 +23,7 @@
#define KN 8
typedef __m256 V;
+typedef __m256 D;
typedef float TA;
typedef float TB;
typedef float TC;
diff --git a/llamafile/sgemm_sss_neon.cpp b/llamafile/sgemm_sss_neon.cpp
index 16f7f0a3..f744654f 100644
--- a/llamafile/sgemm_sss_neon.cpp
+++ b/llamafile/sgemm_sss_neon.cpp
@@ -23,6 +23,7 @@
#define KN 4
typedef float32x4_t V;
+typedef float32x4_t D;
typedef float TA;
typedef float TB;
typedef float TC;
diff --git a/llamafile/sgemmer.inc b/llamafile/sgemmer.inc
index c7a0f9c4..c44d3852 100644
--- a/llamafile/sgemmer.inc
+++ b/llamafile/sgemmer.inc
@@ -68,31 +68,31 @@ class SGEMMER {
dontinline void gemm5x5(int m0, int m, int n0, int n) {
BEGIN_KERNEL(5, 5)
- V c00 = zero();
- V c01 = zero();
- V c02 = zero();
- V c03 = zero();
- V c04 = zero();
- V c10 = zero();
- V c11 = zero();
- V c12 = zero();
- V c13 = zero();
- V c14 = zero();
- V c20 = zero();
- V c21 = zero();
- V c22 = zero();
- V c23 = zero();
- V c24 = zero();
- V c30 = zero();
- V c31 = zero();
- V c32 = zero();
- V c33 = zero();
- V c34 = zero();
- V c40 = zero();
- V c41 = zero();
- V c42 = zero();
- V c43 = zero();
- V c44 = zero();
+ D c00 = zero();
+ D c01 = zero();
+ D c02 = zero();
+ D c03 = zero();
+ D c04 = zero();
+ D c10 = zero();
+ D c11 = zero();
+ D c12 = zero();
+ D c13 = zero();
+ D c14 = zero();
+ D c20 = zero();
+ D c21 = zero();
+ D c22 = zero();
+ D c23 = zero();
+ D c24 = zero();
+ D c30 = zero();
+ D c31 = zero();
+ D c32 = zero();
+ D c33 = zero();
+ D c34 = zero();
+ D c40 = zero();
+ D c41 = zero();
+ D c42 = zero();
+ D c43 = zero();
+ D c44 = zero();
for (int l = 0; l < k; l += KN) {
V k0 = load(B + ldb * (j + 0) + l);
V k1 = load(B + ldb * (j + 1) + l);
@@ -160,18 +160,18 @@ class SGEMMER {
dontinline void gemm3x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(3, 4)
- V c00 = zero();
- V c01 = zero();
- V c02 = zero();
- V c03 = zero();
- V c10 = zero();
- V c11 = zero();
- V c12 = zero();
- V c13 = zero();
- V c20 = zero();
- V c21 = zero();
- V c22 = zero();
- V c23 = zero();
+ D c00 = zero();
+ D c01 = zero();
+ D c02 = zero();
+ D c03 = zero();
+ D c10 = zero();
+ D c11 = zero();
+ D c12 = zero();
+ D c13 = zero();
+ D c20 = zero();
+ D c21 = zero();
+ D c22 = zero();
+ D c23 = zero();
for (int l = 0; l < k; l += KN) {
V k0 = load(B + ldb * (j + 0) + l);
V k1 = load(B + ldb * (j + 1) + l);
@@ -210,10 +210,10 @@ class SGEMMER {
dontinline void gemm1x4(int m0, int m, int n0, int n) {
BEGIN_KERNEL(1, 4)
- V c00 = zero();
- V c01 = zero();
- V c02 = zero();
- V c03 = zero();
+ D c00 = zero();
+ D c01 = zero();
+ D c02 = zero();
+ D c03 = zero();
for (int l = 0; l < k; l += KN) {
V a0 = load(A + lda * (i + 0) + l);
V k0 = load(B + ldb * (j + 0) + l);
@@ -234,18 +234,18 @@ class SGEMMER {
dontinline void gemm4x1(int m0, int m, int n0, int n) {
BEGIN_KERNEL(4, 1)
- V c00 = zero();
- V c01 = zero();
- V c02 = zero();
- V c10 = zero();
- V c11 = zero();
- V c12 = zero();
- V c20 = zero();
- V c21 = zero();
- V c22 = zero();
- V c30 = zero();
- V c31 = zero();
- V c32 = zero();
+ D c00 = zero();
+ D c01 = zero();
+ D c02 = zero();
+ D c10 = zero();
+ D c11 = zero();
+ D c12 = zero();
+ D c20 = zero();
+ D c21 = zero();
+ D c22 = zero();
+ D c30 = zero();
+ D c31 = zero();
+ D c32 = zero();
int l = 0;
while (l + KN * 3 <= k) {
{
@@ -289,10 +289,10 @@ class SGEMMER {
dontinline void gemm1x1(int m0, int m, int n0, int n) {
BEGIN_KERNEL(1, 1)
- V c0 = zero();
- V c1 = zero();
- V c2 = zero();
- V c3 = zero();
+ D c0 = zero();
+ D c1 = zero();
+ D c2 = zero();
+ D c3 = zero();
int l = 0;
while (l + KN * 4 <= k) {
c0 = madd(load(A + lda * i + l), load(B + ldb * j + l), c0);
diff --git a/llamafile/tester.h b/llamafile/tester.h
index f8de5e08..96c49d49 100644
--- a/llamafile/tester.h
+++ b/llamafile/tester.h
@@ -128,10 +128,10 @@ template <typename T> ErrorReport diff(int m, int n, const T *Wan, int lda, cons
infs += flt::isinf(y);
denormals -= flt::isdenormal(x);
denormals += flt::isdenormal(y);
- flips += flt::signbit(x) != flt::signbit(y);
+ flips += flt::sign(x) != flt::sign(y);
long long xi = flt::toint(x);
long long yi = flt::toint(y);
- if (flt::signbit(x) == flt::signbit(y)) {
+ if (flt::sign(x) == flt::sign(y)) {
++considered;
long long bad = std::abs(xi - yi);
ulp += bad;