Skip to content

Distribute and run LLMs with a single file.

Sync with upstream projects

Justine Tunney1ca6e4292c81
diff --git a/Makefile b/Makefile
index e026b9ba..997a35db 100644
--- a/Makefile
+++ b/Makefile
@@ -23,7 +23,7 @@ include stb/BUILD.mk
.PHONY: o/$(MODE)/
o/$(MODE)/: o/$(MODE)/llamafile \
o/$(MODE)/llama.cpp \
- o/$(MODE)/stable-diffusion.cpp \
+ o/$(MODE)/stable-diffusion.cpp \
o/$(MODE)/whisper.cpp \
o/$(MODE)/stb \
o/$(MODE)/depend.test
@@ -44,6 +44,7 @@ install: llamafile/zipalign.1 \
o/$(MODE)/llama.cpp/llama-bench/llama-bench \
o/$(MODE)/llama.cpp/perplexity/perplexity \
o/$(MODE)/llama.cpp/llava/llava-quantize \
+ o/$(MODE)/whisper.cpp/main \
o/$(MODE)/llamafile/server/main
mkdir -p $(PREFIX)/bin
$(INSTALL) o/$(MODE)/llamafile/zipalign $(PREFIX)/bin/zipalign
@@ -57,6 +58,8 @@ install: llamafile/zipalign.1 \
$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity
$(INSTALL) o/$(MODE)/llama.cpp/llava/llava-quantize $(PREFIX)/bin/llava-quantize
$(INSTALL) o/$(MODE)/llamafile/server/main $(PREFIX)/bin/llamafiler
+ $(INSTALL) o/$(MODE)/stable-diffusion.cpp/main $(PREFIX)/bin/sdfile
+ $(INSTALL) o/$(MODE)/whisper.cpp/main $(PREFIX)/bin/whisperfile
mkdir -p $(PREFIX)/share/man/man1
$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1
$(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafile
index 68d7e732..72ff4027 100644
--- a/llama.cpp/README.llamafile
+++ b/llama.cpp/README.llamafile
@@ -8,12 +8,13 @@ LICENSE
ORIGIN
- https://github.com/ggerganov/llama.cpp/pull/4406/
- 152da28ae54139e3754189b9e6e1c28e11277502
- 2024-05-23
+ https://github.com/ggerganov/llama.cpp/
+ 8b3befc0e2ed8fb18b903735831496b8b0c80949
+ 2024-08-16
LOCAL MODIFICATIONS
+ - See [jart] and [kawrakow] annotations
- Remove MAP_POPULATE because it makes mmap(tinyllama) block for 100ms
- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs
- Unify main, server, and llava-cli into single llamafile program
diff --git a/llama.cpp/ggml-aarch64.c b/llama.cpp/ggml-aarch64.c
index af53dea1..7adaadc9 100644
--- a/llama.cpp/ggml-aarch64.c
+++ b/llama.cpp/ggml-aarch64.c
@@ -16,6 +16,8 @@
#if defined(__GNUC__)
#pragma GCC diagnostic ignored "-Woverlength-strings"
+#elif defined(_MSC_VER)
+#pragma warning(disable: 4244 4267) // possible loss of data
#endif
#define UNUSED GGML_UNUSED
@@ -384,8 +386,8 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
}
#endif
@@ -496,8 +498,8 @@ void ggml_gemv_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
}
#endif
@@ -614,7 +616,7 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- if (svcntw() == 8) {
+ if (ggml_sve_cnt_b == QK8_0) {
const void * b_ptr = vx;
const void * a_ptr = vy;
float * res_ptr = s;
@@ -680,12 +682,12 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
return;
}
else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
"performance");
}
else if (ggml_cpu_has_neon()) {
- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&
+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&
"__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
"quantization format for optimal performance");
}
@@ -745,8 +747,8 @@ void ggml_gemm_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
}
#endif
@@ -1266,8 +1268,8 @@ void ggml_gemm_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)
- if (svcntw() == 8) {
- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ if (ggml_sve_cnt_b == QK8_0) {
+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");
}
#endif
@@ -1728,7 +1730,7 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
UNUSED(blocklen);
#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))
- if (svcntw() == 8) {
+ if (ggml_sve_cnt_b == QK8_0) {
const void * b_ptr = vx;
const void * a_ptr = vy;
float * res_ptr = s;
@@ -2139,12 +2141,12 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *
return;
}
else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {
- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&
+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&
"__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal "
"performance");
}
else if (ggml_cpu_has_neon()) {
- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&
+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&
"__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 "
"quantization format for optimal performance");
}
diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.c
index c44d91c1..ffba3268 100644
--- a/llama.cpp/ggml-backend.c
+++ b/llama.cpp/ggml-backend.c
@@ -356,15 +356,10 @@ void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t b
}
// an async copy would normally happen after all the queued operations on both backends are completed
- // sync src, set_async dst
- if (ggml_backend_buffer_is_host(src->buffer)) {
- ggml_backend_synchronize(backend_src);
- ggml_backend_tensor_set_async(backend_dst, dst, src->data, 0, ggml_nbytes(src));
- } else {
- ggml_backend_synchronize(backend_src);
- ggml_backend_tensor_copy(src, dst);
- ggml_backend_synchronize(backend_dst);
- }
+ // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy
+ ggml_backend_synchronize(backend_src);
+ ggml_backend_synchronize(backend_dst);
+ ggml_backend_tensor_copy(src, dst);
}
// events
@@ -1028,10 +1023,6 @@ static bool ggml_is_view_op(enum ggml_op op) {
#define GGML_SCHED_MAX_BACKENDS 16
#endif
-#ifndef GGML_SCHED_MAX_SPLITS
-#define GGML_SCHED_MAX_SPLITS 2048
-#endif
-
#ifndef GGML_SCHED_MAX_SPLIT_INPUTS
#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC
#endif
@@ -1135,7 +1126,8 @@ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, co
}
#if 0
-static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only
+#define GGML_SCHED_MAX_SPLITS_DEBUG 4096
+static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only
#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)
#define GET_CAUSE(node) causes[hash_id(node)]
#else
@@ -1559,7 +1551,6 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct gg
sched->splits = realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));
GGML_ASSERT(sched->splits != NULL);
}
- GGML_ASSERT(i_split < GGML_SCHED_MAX_SPLITS);
split = &sched->splits[i_split];
split->backend_id = node_backend_id;
split->i_start = i;
@@ -1787,7 +1778,17 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
} else {
ggml_backend_synchronize(split_backend);
}
- ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy);
+ // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
+ // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
+ if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
+ ggml_backend_synchronize(input_backend);
+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);
+ } else {
+ ggml_backend_synchronize(split_backend);
+ }
+ ggml_backend_tensor_copy(input, input_cpy);
+ }
}
}
@@ -1865,13 +1866,14 @@ ggml_backend_sched_t ggml_backend_sched_new(
sched->hv_tensor_backend_ids = malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));
sched->hv_tensor_copies = malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));
- const size_t nodes_size = graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2;
+ const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph
+ const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;
sched->node_backend_ids = calloc(nodes_size, sizeof(sched->node_backend_ids[0]));
sched->leaf_backend_ids = calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));
sched->prev_node_backend_ids = calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));
sched->prev_leaf_backend_ids = calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));
- sched->context_buffer_size = GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
+ sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
sched->context_buffer = malloc(sched->context_buffer_size);
const int initial_splits_capacity = 16;
diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.h
index 90149072..9e56788c 100644
--- a/llama.cpp/ggml-cuda.h
+++ b/llama.cpp/ggml-cuda.h
@@ -6,6 +6,9 @@
#ifdef GGML_USE_HIPBLAS
#define GGML_CUDA_NAME "ROCm"
#define GGML_CUBLAS_NAME "hipBLAS"
+#elif defined(GGML_USE_MUSA)
+#define GGML_CUDA_NAME "MUSA"
+#define GGML_CUBLAS_NAME "muBLAS"
#else
#define GGML_CUDA_NAME "CUDA"
#define GGML_CUBLAS_NAME "cuBLAS"
diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.h
index 119ca3c5..190af081 100644
--- a/llama.cpp/ggml-impl.h
+++ b/llama.cpp/ggml-impl.h
@@ -80,8 +80,9 @@ static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {
/**
* Converts float32 to brain16.
*
- * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.
- * Subnormals shall be flushed to zero, and NANs will be quiet.
+ * This is binary identical with Google Brain float conversion.
+ * Floats shall round to nearest even, and NANs shall be quiet.
+ * Subnormals aren't flushed to zero, except perhaps when used.
* This code should vectorize nicely if using modern compilers.
*/
static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {
@@ -142,6 +143,7 @@ extern "C" {
#if defined(__ARM_FEATURE_SVE)
#include <arm_sve.h>
+#include <sys/prctl.h>
#endif
// 16-bit float
diff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.h
index 1d536289..4377cf8f 100644
--- a/llama.cpp/ggml-metal.h
+++ b/llama.cpp/ggml-metal.h
@@ -1,7 +1,7 @@
// An interface allowing to compute ggml_cgraph with Metal
//
// This is a fully functional interface that extends ggml with GPU support for Apple devices.
-// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
+// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)
//
// How it works?
//
@@ -52,6 +52,8 @@ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void
GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
+GGML_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data);
+
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
// helper to check if the device supports a specific family
@@ -65,4 +67,3 @@ GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
#ifdef __cplusplus
}
#endif
-
diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.m
index 7aeb8d88..dbe51253 100644
--- a/llama.cpp/ggml-metal.m
+++ b/llama.cpp/ggml-metal.m
@@ -239,7 +239,7 @@ enum ggml_metal_kernel_type {
GGML_METAL_KERNEL_TYPE_COUNT
};
-struct ggml_metal_context {
+struct ggml_backend_metal_context {
int n_cb;
id<MTLDevice> device;
@@ -253,6 +253,10 @@ struct ggml_metal_context {
bool support_simdgroup_mm;
bool should_capture_next_compute;
+
+ // abort ggml_metal_graph_compute if callback returns true
+ ggml_abort_callback abort_callback;
+ void * abort_callback_data;
};
// MSL code
@@ -320,7 +324,7 @@ static void * ggml_metal_host_malloc(size_t n) {
return data;
}
-static struct ggml_metal_context * ggml_metal_init(int n_cb) {
+static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {
GGML_METAL_LOG_INFO("%s: allocating\n", __func__);
#if TARGET_OS_OSX && !GGML_METAL_NDEBUG
@@ -337,7 +341,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {
GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);
// Configure context
- struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));
+ struct ggml_backend_metal_context * ctx = calloc(1, sizeof(struct ggml_backend_metal_context));
ctx->device = device;
ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
ctx->queue = [ctx->device newCommandQueue];
@@ -699,7 +703,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {
return ctx;
}
-static void ggml_metal_free(struct ggml_metal_context * ctx) {
+static void ggml_metal_free(struct ggml_backend_metal_context * ctx) {
GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);
for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {
@@ -765,7 +769,7 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offs
return nil;
}
-static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {
+static bool ggml_metal_supports_op(const struct ggml_backend_metal_context * ctx, const struct ggml_tensor * op) {
for (size_t i = 0, n = 3; i < n; ++i) {
if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {
return false;
@@ -876,7 +880,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const
}
static enum ggml_status ggml_metal_graph_compute(
- struct ggml_metal_context * ctx,
+ struct ggml_backend_metal_context * ctx,
struct ggml_cgraph * gf) {
@autoreleasepool {
@@ -909,8 +913,11 @@ static enum ggml_status ggml_metal_graph_compute(
id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];
command_buffer_builder[cb_idx] = command_buffer;
- // enqueue the command buffers in order to specify their execution order
- [command_buffer enqueue];
+ // always enqueue the first two command buffers
+ // enqueue all of the command buffers if we don't need to abort
+ if (cb_idx < 2 || ctx->abort_callback == NULL) {
+ [command_buffer enqueue];
+ }
}
const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;
@@ -2260,10 +2267,8 @@ static enum ggml_status ggml_metal_graph_compute(
GGML_ASSERT(ne00 % 4 == 0);
GGML_ASSERT(ggml_is_contiguous(src0));
- //float eps;
- //memcpy(&eps, dst->op_params, sizeof(float));
-
- const float eps = 1e-6f; // TODO: temporarily hardcoded
+ float eps;
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
const int32_t n_groups = ((int32_t *) dst->op_params)[0];
@@ -2339,7 +2344,7 @@ static enum ggml_status ggml_metal_graph_compute(
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
- const bool is_neox = mode & 2;
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
id<MTLComputePipelineState> pipeline = nil;
@@ -2860,7 +2865,9 @@ static enum ggml_status ggml_metal_graph_compute(
[encoder endEncoding];
- [command_buffer commit];
+ if (cb_idx < 2 || ctx->abort_callback == NULL) {
+ [command_buffer commit];
+ }
});
// Wait for completion and check status of each command buffer
@@ -2880,6 +2887,23 @@ static enum ggml_status ggml_metal_graph_compute(
return GGML_STATUS_FAILED;
}
+
+ id<MTLCommandBuffer> next_buffer = (i + 1 < n_cb ? command_buffers[i + 1] : nil);
+ if (!next_buffer) {
+ continue;
+ }
+
+ bool next_queued = ([next_buffer status] != MTLCommandBufferStatusNotEnqueued);
+ if (next_queued) {
+ continue;
+ }
+
+ if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) {
+ GGML_METAL_LOG_INFO("%s: command buffer %d aborted", __func__, i);
+ return GGML_STATUS_ABORTED;
+ }
+
+ [next_buffer commit];
}
if (should_capture) {
@@ -3183,7 +3207,7 @@ GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {
}
GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
ggml_metal_free(ctx);
free(backend);
}
@@ -3195,13 +3219,13 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffe
}
GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;
return ggml_metal_graph_compute(metal_ctx, cgraph);
}
GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;
return ggml_metal_supports_op(metal_ctx, op);
}
@@ -3246,9 +3270,9 @@ static ggml_guid_t ggml_backend_metal_guid(void) {
}
ggml_backend_t ggml_backend_metal_init(void) {
- struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
-
+ struct ggml_backend_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);
if (ctx == NULL) {
+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate context\n", __func__);
return NULL;
}
@@ -3270,15 +3294,24 @@ bool ggml_backend_is_metal(ggml_backend_t backend) {
void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
GGML_ASSERT(ggml_backend_is_metal(backend));
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);
}
+void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data) {
+ GGML_ASSERT(ggml_backend_is_metal(backend));
+
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
+
+ ctx->abort_callback = abort_callback;
+ ctx->abort_callback_data = user_data;
+}
+
bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
GGML_ASSERT(ggml_backend_is_metal(backend));
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];
}
@@ -3286,7 +3319,7 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {
void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
GGML_ASSERT(ggml_backend_is_metal(backend));
- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;
+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;
ctx->should_capture_next_compute = true;
}
diff --git a/llama.cpp/ggml-quants.h b/llama.cpp/ggml-quants.h
index 97c19c64..525d5ee3 100644
--- a/llama.cpp/ggml-quants.h
+++ b/llama.cpp/ggml-quants.h
@@ -127,7 +127,10 @@ void iq2xs_free_impl(enum ggml_type type);
void iq3xs_init_impl(int grid_size);
void iq3xs_free_impl(int grid_size);
+#if defined(__ARM_FEATURE_SVE)
+extern int ggml_sve_cnt_b;
+#endif
+
#ifdef __cplusplus
}
#endif
-
diff --git a/llama.cpp/ggml-quants.inc b/llama.cpp/ggml-quants.inc
index 2a5200eb..3ac736bb 100644
--- a/llama.cpp/ggml-quants.inc
+++ b/llama.cpp/ggml-quants.inc
@@ -3857,7 +3857,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r
float sumf = 0;
#if defined(__ARM_FEATURE_SVE)
- if (svcntb() == QK8_0) {
+ if (ggml_sve_cnt_b == QK8_0) {
const svbool_t ptrueh = svptrue_pat_b8(SV_VL16);
const svbool_t ptruel = svnot_b_z(svptrue_b8(), ptrueh);
@@ -5342,7 +5342,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r
float sumf = 0;
#if defined(__ARM_FEATURE_SVE)
- if (svcntb() == QK8_0) {
+ if (ggml_sve_cnt_b == QK8_0) {
svfloat32_t sumv0 = svdup_n_f32(0.0f);
svfloat32_t sumv1 = svdup_n_f32(0.0f);
diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.c
index 5309e530..70171828 100644
--- a/llama.cpp/ggml-vector-amd-avx.c
+++ b/llama.cpp/ggml-vector-amd-avx.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx
diff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.c
index 547a7b1b..7eaab383 100644
--- a/llama.cpp/ggml-vector-amd-avx2.c
+++ b/llama.cpp/ggml-vector-amd-avx2.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx2
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx2
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx2
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx2
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx2
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx2
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx2
diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.c
index 4f07c007..866cf72c 100644
--- a/llama.cpp/ggml-vector-amd-avx512.c
+++ b/llama.cpp/ggml-vector-amd-avx512.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512
diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.c
index f0278fc6..1d470f74 100644
--- a/llama.cpp/ggml-vector-amd-avx512bf16.c
+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512bf16
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512bf16
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512bf16
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512bf16
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512bf16
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512bf16
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512bf16
diff --git a/llama.cpp/ggml-vector-amd-avx512vl.c b/llama.cpp/ggml-vector-amd-avx512vl.c
index 4e13c22d..bd27e54e 100644
--- a/llama.cpp/ggml-vector-amd-avx512vl.c
+++ b/llama.cpp/ggml-vector-amd-avx512vl.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512vl
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512vl
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512vl
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512vl
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512vl
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512vl
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512vl
diff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.c
index 21cbc28a..7e458f30 100644
--- a/llama.cpp/ggml-vector-amd-f16c.c
+++ b/llama.cpp/ggml-vector-amd-f16c.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_f16c
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_f16c
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_f16c
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_f16c
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_f16c
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_f16c
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_f16c
diff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.c
index 0e3793e2..c38ade50 100644
--- a/llama.cpp/ggml-vector-amd-fma.c
+++ b/llama.cpp/ggml-vector-amd-fma.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_fma
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_fma
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_fma
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_fma
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_fma
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_fma
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_fma
diff --git a/llama.cpp/ggml-vector-amd-k8.c b/llama.cpp/ggml-vector-amd-k8.c
index bf730915..970559f5 100644
--- a/llama.cpp/ggml-vector-amd-k8.c
+++ b/llama.cpp/ggml-vector-amd-k8.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_k8
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_k8
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_k8
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_k8
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_k8
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_k8
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_k8
diff --git a/llama.cpp/ggml-vector-amd-ssse3.c b/llama.cpp/ggml-vector-amd-ssse3.c
index c86127be..15806ef3 100644
--- a/llama.cpp/ggml-vector-amd-ssse3.c
+++ b/llama.cpp/ggml-vector-amd-ssse3.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_ssse3
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_ssse3
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_ssse3
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_ssse3
#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_ssse3
#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_ssse3
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_ssse3
diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.c
index ecc1a178..3733eba3 100644
--- a/llama.cpp/ggml-vector-arm80.c
+++ b/llama.cpp/ggml-vector-arm80.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm80
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm80
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm80
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_arm80
#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm80
#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm80
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm80
diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.c
index cd0eba76..1d87d7c1 100644
--- a/llama.cpp/ggml-vector-arm82.c
+++ b/llama.cpp/ggml-vector-arm82.c
@@ -3,6 +3,7 @@
#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm82
#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm82
#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm82
+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_arm82
#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm82
#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm82
#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm82
diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cpp
index f370ee6d..db44d372 100644
--- a/llama.cpp/ggml-vector.cpp
+++ b/llama.cpp/ggml-vector.cpp
@@ -51,6 +51,18 @@ extern "C" void ggml_fp32_to_bf16_row_amd_k8(const float * x, ggml_bf16_t * y, i
extern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);
extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512vl(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx2(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_f16c(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_fma(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_ssse3(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_amd_k8(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_arm82(const float * x, ggml_bf16_t * y, int64_t n);
+extern "C" void ggml_fp32_to_bf16_row_ref_arm80(const float * x, ggml_bf16_t * y, int64_t n);
+
extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f32_amd_avx512vl(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
@@ -668,6 +680,7 @@ static const struct VectorFuncs {
typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;
typeof(ggml_bf16_to_fp32_row) *ptr_ggml_bf16_to_fp32_row;
typeof(ggml_fp32_to_bf16_row) *ptr_ggml_fp32_to_bf16_row;
+ typeof(ggml_fp32_to_bf16_row_ref) *ptr_ggml_fp32_to_bf16_row_ref;
typeof(ggml_vec_dot_f32) *ptr_ggml_vec_dot_f32;
typeof(ggml_vec_dot_f16) *ptr_ggml_vec_dot_f16;
typeof(ggml_vec_dot_bf16) *ptr_ggml_vec_dot_bf16;
@@ -727,6 +740,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512bf16;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512bf16;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512bf16;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512bf16;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512bf16;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512bf16;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512bf16;
@@ -787,6 +801,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512vl;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512vl;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512vl;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512vl;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512vl;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512vl;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512vl;
@@ -847,6 +862,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512;
@@ -907,6 +923,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx2;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx2;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx2;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx2;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx2;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx2;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx2;
@@ -967,6 +984,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_f16c;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_f16c;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_f16c;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_f16c;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_f16c;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_f16c;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_f16c;
@@ -1027,6 +1045,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_fma;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_fma;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_fma;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_fma;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_fma;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_fma;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_fma;
@@ -1087,6 +1106,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx;
@@ -1147,6 +1167,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_ssse3;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_ssse3;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_ssse3;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_ssse3;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_ssse3;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_ssse3;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_ssse3;
@@ -1207,6 +1228,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_k8;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_k8;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_k8;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_k8;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_k8;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_k8;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_k8;
@@ -1267,6 +1289,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm82;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm82;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm82;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_arm82;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm82;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm82;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm82;
@@ -1327,6 +1350,7 @@ static const struct VectorFuncs {
ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm80;
ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm80;
ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm80;
+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_arm80;
ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm80;
ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm80;
ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm80;
@@ -1400,6 +1424,10 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);
}
+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {
+ return funcs.ptr_ggml_fp32_to_bf16_row_ref(x, y, n);
+}
+
void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc) {
return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);
}
diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.h
index 938d5c37..269bc883 100644
--- a/llama.cpp/ggml-vector.h
+++ b/llama.cpp/ggml-vector.h
@@ -10,6 +10,7 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);
void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);
void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);
void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);
+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n);
void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);
void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);
void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);
diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.inc
index 0554ba3c..04be033a 100644
--- a/llama.cpp/ggml-vector.inc
+++ b/llama.cpp/ggml-vector.inc
@@ -9,6 +9,11 @@
#include <pthread.h>
#include <cosmo.h>
+#if __FINITE_MATH_ONLY__
+#error "some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix"
+#error "ref: https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461"
+#endif
+
/* #define GGML_GELU_FP16 */
#define UNUSED GGML_UNUSED
@@ -862,7 +867,7 @@ void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {
void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
int i = 0;
-#if defined(__AVX512BF16__) && 0 // [jart] it kills subnormals
+#if defined(__AVX512BF16__)
for (; i + 32 <= n; i += 32) {
_mm512_storeu_si512(
(__m512i *)(y + i),
@@ -875,6 +880,12 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {
}
}
+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {
+ for (int i = 0; i < n; i++) {
+ y[i] = ggml_compute_fp32_to_bf16(x[i]);
+ }
+}
+
//
// fundamental operations
//
@@ -1608,6 +1619,7 @@ ggml_vgeluf(__m128 x)
void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
int i = 0;
+
if (!FLAG_trap) { // [jart] preserve this line
#if defined(__ARM_NEON) && defined(__aarch64__)
for (; i + 3 < n; i += 4) {
@@ -1627,6 +1639,7 @@ void ggml_vec_gelu_f32(const int n, float * y, const float * x) {
}
#endif
} // [jart] preserve this line
+
if (i == n)
return;
#ifdef GGML_GELU_FP16
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 70a427c3..9db5bf9f 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -639,7 +639,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {
.is_quantized = false,
.to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,
.from_float = (ggml_from_float_t) ggml_fp32_to_bf16_row,
- .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row,
+ .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,
.vec_dot_type = GGML_TYPE_BF16,
.nrows = 1,
@@ -1537,21 +1537,21 @@ struct ggml_phaser {
};
struct ggml_compute_state_shared {
- const struct ggml_cgraph * cgraph;
- const struct ggml_cplan * cplan;
+ const struct ggml_cgraph * cgraph;
+ const struct ggml_cplan * cplan;
- int n_threads;
+ int n_threads;
- // synchronization primitives
- atomic_int n_barrier;
- struct ggml_phaser *n_barrier_passed;
+ // synchronization primitives
+ atomic_int n_barrier;
+ struct ggml_phaser *n_barrier_passed;
- ggml_abort_callback abort_callback; // abort ggml_graph_compute when true
- void * abort_callback_data;
+ ggml_abort_callback abort_callback; // abort ggml_graph_compute when true
+ void * abort_callback_data;
- atomic_int current_chunk; // currently processing chunk during mul_mat, shared between all the threads
+ atomic_int current_chunk; // currently processing chunk during mul_mat, shared between all the threads
- enum ggml_status ec;
+ enum ggml_status ec;
};
struct ggml_compute_state {
@@ -1652,7 +1652,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
"CROSS_ENTROPY_LOSS_BACK",
};
-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 74");
+static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74");
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"none",
@@ -1740,7 +1740,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
"cross_entropy_loss_back(x,y)",
};
-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 74");
+static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74");
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
@@ -2461,7 +2461,8 @@ static struct ggml_tensor * ggml_new_tensor_impl(
struct ggml_tensor * view_src,
size_t view_offs) {
- assert(n_dims >= 1 && n_dims <= GGML_MAX_DIMS);
+ GGML_ASSERT(type >= 0 && type < GGML_TYPE_COUNT);
+ GGML_ASSERT(n_dims >= 1 && n_dims <= GGML_MAX_DIMS);
// find the base tensor and absolute offset
if (view_src != NULL && view_src->view_src != NULL) {
@@ -4133,6 +4134,7 @@ static struct ggml_tensor * ggml_group_norm_impl(
struct ggml_context * ctx,
struct ggml_tensor * a,
int n_groups,
+ float eps,
bool inplace) {
bool is_node = false;
@@ -4143,7 +4145,8 @@ static struct ggml_tensor * ggml_group_norm_impl(
struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);
- result->op_params[0] = n_groups;
+ ggml_set_op_params_i32(result, 0, n_groups);
+ ggml_set_op_params_f32(result, 1, eps);
result->op = GGML_OP_GROUP_NORM;
result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;
@@ -4155,15 +4158,17 @@ static struct ggml_tensor * ggml_group_norm_impl(
struct ggml_tensor * ggml_group_norm(
struct ggml_context * ctx,
struct ggml_tensor * a,
- int n_groups) {
- return ggml_group_norm_impl(ctx, a, n_groups, false);
+ int n_groups,
+ float eps) {
+ return ggml_group_norm_impl(ctx, a, n_groups, eps, false);
}
struct ggml_tensor * ggml_group_norm_inplace(
struct ggml_context * ctx,
struct ggml_tensor * a,
- int n_groups) {
- return ggml_group_norm_impl(ctx, a, n_groups, true);
+ int n_groups,
+ float eps) {
+ return ggml_group_norm_impl(ctx, a, n_groups, eps, true);
}
// ggml_mul_mat
@@ -10854,10 +10859,11 @@ static void ggml_compute_forward_group_norm_f32(
GGML_TENSOR_UNARY_OP_LOCALS
- const float eps = 1e-6f; // TODO: make this a parameter
-
// TODO: optimize
+ float eps;
+ memcpy(&eps, dst->op_params + 1, sizeof(float));
+
int n_channels = src0->ne[2];
int n_groups = dst->op_params[0];
int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;
@@ -11125,6 +11131,11 @@ UseGgmlGemm1:;
ggml_barrier(params);
}
+ if (ith == 0) {
+ // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
+ atomic_store(&params->shared->current_chunk, nth);
+ }
+
#if GGML_USE_LLAMAFILE
if (src1->type != vec_dot_type) {
const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
@@ -11150,11 +11161,6 @@ UseGgmlGemm1:;
UseGgmlGemm2:;
#endif
- if (ith == 0) {
- // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.
- atomic_store(&params->shared->current_chunk, nth);
- }
-
// This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)
const int64_t nr0 = ne0;
@@ -12840,7 +12846,7 @@ static void ggml_compute_forward_rope_f32(
float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
- const bool is_neox = mode & 2;
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
const float * freq_factors = NULL;
if (src2 != NULL) {
@@ -12965,7 +12971,7 @@ static void ggml_compute_forward_rope_f16(
float corr_dims[2];
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);
- const bool is_neox = mode & 2;
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
const float * freq_factors = NULL;
if (src2 != NULL) {
@@ -17737,13 +17743,41 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cpl
llamafile_debug_graph = cgraph;
#endif
+#ifdef GGML_USE_OPENMP
+ if (n_threads > 1) {
+ #pragma omp parallel num_threads(n_threads)
+ {
+ #pragma omp single
+ {
+ // update the number of threads from the actual number of threads that we got from OpenMP
+ n_threads = omp_get_num_threads();
+ state_shared.n_threads = n_threads;
+ }
+
+ struct ggml_compute_state worker = {
+ .thrd = 0,
+ .ith = omp_get_thread_num(),
+ .shared = &state_shared,
+ };
+ ggml_graph_compute_thread(&worker);
+ }
+ } else {
+ struct ggml_compute_state worker = {
+ .thrd = 0,
+ .ith = 0,
+ .shared = &state_shared,
+ };
+ ggml_graph_compute_thread(&worker);
+ }
+#endif
+
struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads);
struct ggml_compute_cleanup cleanup = {n_threads, workers};
pthread_cleanup_push(ggml_compute_canceled, &cleanup);
for (int j = 0; j < n_threads; ++j) {
workers[j] = (struct ggml_compute_state) {
- .thrd = ATOMIC_VAR_INIT(0),
+ .thrd = 0,
.ith = j,
.shared = &state_shared,
.is_main_thread = false, // [jart]
@@ -19496,7 +19530,7 @@ size_t ggml_quantize_chunk(
case GGML_TYPE_BF16:
{
size_t elemsize = sizeof(ggml_bf16_t);
- ggml_fp32_to_bf16_row(src + start, (ggml_bf16_t *)dst + start, n);
+ ggml_fp32_to_bf16_row_ref(src + start, (ggml_bf16_t *)dst + start, n);
result = n * elemsize;
} break;
case GGML_TYPE_F32:
@@ -19938,7 +19972,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i
(int64_t) info->ne[2] *
(int64_t) info->ne[3];
- if (ne % ggml_blck_size(info->type) != 0) {
+ if (ggml_blck_size(info->type) == 0 || ne % ggml_blck_size(info->type) != 0) {
fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%" PRId64 ")\n",
__func__, info->name.data, (int) info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));
gguf_free(ctx);
diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.h
index 1184065c..bcd17317 100644
--- a/llama.cpp/ggml.h
+++ b/llama.cpp/ggml.h
@@ -247,6 +247,8 @@
#define GGML_EXIT_SUCCESS 0
#define GGML_EXIT_ABORTED 1
+#define GGML_ROPE_TYPE_NEOX 2
+
#define GGUF_MAGIC "GGUF"
#define GGUF_VERSION 3
@@ -352,6 +354,7 @@ extern "C" {
GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16
GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
+ GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);
GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);
struct ggml_object;
@@ -500,9 +503,7 @@ extern "C" {
GGML_OP_ARGSORT,
GGML_OP_LEAKY_RELU,
- GGML_OP_FLASH_ATTN,
GGML_OP_FLASH_ATTN_EXT,
- GGML_OP_FLASH_FF,
GGML_OP_FLASH_ATTN_BACK,
GGML_OP_SSM_CONV,
GGML_OP_SSM_SCAN,
@@ -1166,16 +1167,17 @@ extern "C" {
// group normalize along ne0*ne1*n_groups
// used in stable-diffusion
- // TODO: eps is hardcoded to 1e-6 for now
GGML_API struct ggml_tensor * ggml_group_norm(
struct ggml_context * ctx,
struct ggml_tensor * a,
- int n_groups);
+ int n_groups,
+ float eps);
GGML_API struct ggml_tensor * ggml_group_norm_inplace(
struct ggml_context * ctx,
struct ggml_tensor * a,
- int n_groups);
+ int n_groups,
+ float eps);
// a - x
// b - dy
@@ -1478,11 +1480,10 @@ extern "C" {
struct ggml_tensor * b);
// rotary position embedding
- // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
- // if mode & 2 == 1, GPT-NeoX style
+ // if (mode & 1) - skip n_past elements (NOT SUPPORTED)
+ // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
//
// b is an int32 vector with size a->ne[2], it contains the positions
- // c is freq factors (e.g. phi3-128k), (optional)
GGML_API struct ggml_tensor * ggml_rope(
struct ggml_context * ctx,
struct ggml_tensor * a,
@@ -1499,6 +1500,7 @@ extern "C" {
int mode);
// custom RoPE
+ // c is freq factors (e.g. phi3-128k), (optional)
GGML_API struct ggml_tensor * ggml_rope_ext(
struct ggml_context * ctx,
struct ggml_tensor * a,
diff --git a/llama.cpp/llama-impl.h b/llama.cpp/llama-impl.h
index dcc8c1c1..399b134a 100644
--- a/llama.cpp/llama-impl.h
+++ b/llama.cpp/llama-impl.h
@@ -24,3 +24,18 @@ void llama_log_callback_default(ggml_log_level level, const char * text, void *
#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)
#define LLAMA_LOG_WARN(...) llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)
#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
+
+//
+// helpers
+//
+
+static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
+ if (search.empty()) {
+ return; // Avoid infinite loop if 'search' is an empty string
+ }
+ size_t pos = 0;
+ while ((pos = s.find(search, pos)) != std::string::npos) {
+ s.replace(pos, search.length(), replace);
+ pos += replace.length();
+ }
+}
diff --git a/llama.cpp/llama-sampling.cpp b/llama.cpp/llama-sampling.cpp
index fd258947..804663f9 100644
--- a/llama.cpp/llama-sampling.cpp
+++ b/llama.cpp/llama-sampling.cpp
@@ -87,14 +87,14 @@ void llama_sample_top_k_impl(struct llama_sampling * smpl, llama_token_data_arra
constexpr float bucket_low = -10.0f;
constexpr float bucket_high = 10.0f;
constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);
- constexpr float bucker_inter = -bucket_low * bucket_scale;
+ constexpr float bucket_inter = -bucket_low * bucket_scale;
std::vector<int> bucket_idx(candidates->size);
std::vector<int> histo(nbuckets, 0);
for (int i = 0; i < (int)candidates->size; ++i) {
const float val = candidates->data[i].logit;
- int ib = int(bucket_scale * val + bucker_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
+ int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);
ib = std::max(0, std::min(nbuckets-1, ib));
bucket_idx[i] = ib;
++histo[ib];
diff --git a/llama.cpp/llama-vocab.cpp b/llama.cpp/llama-vocab.cpp
index 288fd5bd..6c340e36 100644
--- a/llama.cpp/llama-vocab.cpp
+++ b/llama.cpp/llama-vocab.cpp
@@ -18,20 +18,6 @@
// helpers
//
-static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
- std::string result;
- for (size_t pos = 0; ; pos += search.length()) {
- auto new_pos = s.find(search, pos);
- if (new_pos == std::string::npos) {
- result += s.substr(pos, s.size() - pos);
- break;
- }
- result += s.substr(pos, new_pos - pos) + replace;
- pos = new_pos;
- }
- s = std::move(result);
-}
-
LLAMA_ATTRIBUTE_FORMAT(1, 2)
static std::string format(const char * fmt, ...) {
va_list ap;
@@ -404,6 +390,7 @@ struct llm_tokenizer_bpe {
case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:
case LLAMA_VOCAB_PRE_TYPE_SMOLLM:
case LLAMA_VOCAB_PRE_TYPE_CODESHELL:
+ case LLAMA_VOCAB_PRE_TYPE_EXAONE:
regex_exprs = {
"\\p{N}",
"'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
@@ -426,6 +413,8 @@ struct llm_tokenizer_bpe {
};
break;
case LLAMA_VOCAB_PRE_TYPE_PORO:
+ case LLAMA_VOCAB_PRE_TYPE_BLOOM:
+ case LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH:
regex_exprs = {
" ?[^(\\s|.,!?…。,、।۔،)]+",
};
@@ -818,6 +807,9 @@ struct llm_tokenizer_ugm {
* the best tokenization.
*/
void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {
+ // get current size of output (for reversal later)
+ size_t output_size = output.size();
+
// normalize the input first
std::string normalized;
normalize(text, &normalized);
@@ -897,7 +889,7 @@ struct llm_tokenizer_ugm {
}
// reverse the output since we added tokens starting from the end of the input
- std::reverse(output.begin(), output.end());
+ std::reverse(output.begin() + output_size, output.end());
}
private:
@@ -1446,7 +1438,8 @@ llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, lla
bool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {
return token != -1 && (
token == llama_token_eos_impl(vocab) ||
- token == llama_token_eot_impl(vocab)
+ token == llama_token_eot_impl(vocab) ||
+ token == llama_token_eom_impl(vocab)
);
}
@@ -1478,11 +1471,11 @@ llama_token llama_token_pad_impl(const struct llama_vocab & vocab) {
return vocab.special_pad_id;
}
-int32_t llama_add_bos_token_impl(const struct llama_vocab & vocab) {
+bool llama_add_bos_token_impl(const struct llama_vocab & vocab) {
return vocab.tokenizer_add_bos;
}
-int32_t llama_add_eos_token_impl(const struct llama_vocab & vocab) {
+bool llama_add_eos_token_impl(const struct llama_vocab & vocab) {
return vocab.tokenizer_add_eos;
}
@@ -1502,6 +1495,10 @@ llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {
return vocab.special_eot_id;
}
+llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {
+ return vocab.special_eom_id;
+}
+
int32_t llama_tokenize_impl(
const struct llama_vocab & vocab,
const char * text,
diff --git a/llama.cpp/llama-vocab.h b/llama.cpp/llama-vocab.h
index 4d0be82b..3741a871 100644
--- a/llama.cpp/llama-vocab.h
+++ b/llama.cpp/llama-vocab.h
@@ -47,6 +47,7 @@ struct llama_vocab {
id special_suffix_id = -1;
id special_middle_id = -1;
id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token
+ id special_eom_id = -1;
// tokenizer flags
bool tokenizer_add_space_prefix = false;
@@ -96,13 +97,14 @@ llama_token llama_token_sep_impl(const struct llama_vocab & vocab);
llama_token llama_token_nl_impl (const struct llama_vocab & vocab);
llama_token llama_token_pad_impl(const struct llama_vocab & vocab);
-int32_t llama_add_bos_token_impl(const struct llama_vocab & vocab);
-int32_t llama_add_eos_token_impl(const struct llama_vocab & vocab);
+bool llama_add_bos_token_impl(const struct llama_vocab & vocab);
+bool llama_add_eos_token_impl(const struct llama_vocab & vocab);
llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);
llama_token llama_token_middle_impl(const struct llama_vocab & vocab);
llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);
llama_token llama_token_eot_impl (const struct llama_vocab & vocab);
+llama_token llama_token_eom_impl (const struct llama_vocab & vocab);
int32_t llama_tokenize_impl(
const struct llama_vocab & vocab,
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 249966ba..e93087ec 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -86,20 +86,6 @@ static std::string trim(const std::string & str) {
return str.substr(start, end - start);
}
-static void replace_all(std::string & s, const std::string & search, const std::string & replace) {
- std::string result;
- for (size_t pos = 0; ; pos += search.length()) {
- auto new_pos = s.find(search, pos);
- if (new_pos == std::string::npos) {
- result += s.substr(pos, s.size() - pos);
- break;
- }
- result += s.substr(pos, new_pos - pos) + replace;
- pos = new_pos;
- }
- s = std::move(result);
-}
-
static bool is_float_close(float a, float b, float abs_tol) {
// Check for non-negative tolerance
if (abs_tol < 0.0) {
@@ -187,7 +173,10 @@ enum llm_arch {
LLM_ARCH_CHATGLM,
LLM_ARCH_BITNET,
LLM_ARCH_T5,
+ LLM_ARCH_T5ENCODER,
LLM_ARCH_JAIS,
+ LLM_ARCH_NEMOTRON,
+ LLM_ARCH_EXAONE,
LLM_ARCH_UNKNOWN,
};
@@ -231,7 +220,10 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_CHATGLM, "chatglm" },
{ LLM_ARCH_BITNET, "bitnet" },
{ LLM_ARCH_T5, "t5" },
+ { LLM_ARCH_T5ENCODER, "t5encoder" },
{ LLM_ARCH_JAIS, "jais" },
+ { LLM_ARCH_NEMOTRON, "nemotron" },
+ { LLM_ARCH_EXAONE, "exaone" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
@@ -327,6 +319,7 @@ enum llm_kv {
LLM_KV_TOKENIZER_SUFFIX_ID,
LLM_KV_TOKENIZER_MIDDLE_ID,
LLM_KV_TOKENIZER_EOT_ID,
+ LLM_KV_TOKENIZER_EOM_ID,
LLM_KV_ADAPTER_TYPE,
LLM_KV_ADAPTER_LORA_ALPHA,
@@ -424,6 +417,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
{ LLM_KV_TOKENIZER_MIDDLE_ID, "tokenizer.ggml.middle_token_id" },
{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },
+ { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },
{ LLM_KV_ADAPTER_TYPE, "adapter.type" },
{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },
@@ -1238,6 +1232,24 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
{ LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },
},
},
+ {
+ LLM_ARCH_T5ENCODER,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
+ { LLM_TENSOR_ENC_ATTN_NORM, "enc.blk.%d.attn_norm" },
+ { LLM_TENSOR_ENC_ATTN_Q, "enc.blk.%d.attn_q" },
+ { LLM_TENSOR_ENC_ATTN_K, "enc.blk.%d.attn_k" },
+ { LLM_TENSOR_ENC_ATTN_V, "enc.blk.%d.attn_v" },
+ { LLM_TENSOR_ENC_ATTN_OUT, "enc.blk.%d.attn_o" },
+ { LLM_TENSOR_ENC_ATTN_REL_B, "enc.blk.%d.attn_rel_b" },
+ { LLM_TENSOR_ENC_FFN_NORM, "enc.blk.%d.ffn_norm" },
+ { LLM_TENSOR_ENC_FFN_GATE, "enc.blk.%d.ffn_gate" },
+ { LLM_TENSOR_ENC_FFN_DOWN, "enc.blk.%d.ffn_down" },
+ { LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },
+ },
+ },
{
LLM_ARCH_JAIS,
{
@@ -1253,6 +1265,43 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
},
},
+ {
+ LLM_ARCH_NEMOTRON,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
+ {
+ LLM_ARCH_EXAONE,
+ {
+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },
+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },
+ { LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },
+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
+ },
+ },
{
LLM_ARCH_UNKNOWN,
{
@@ -3268,13 +3317,8 @@ namespace GGUFMeta {
using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;
-// TODO: update when needed or think of some clever automatic way to do this
-static size_t llama_model_max_nodes(const llama_model & /*model*/) {
- //if (model.arch == LLM_ARCH_LLAMA && model.hparams.n_layer > ??) { // llama-3 405B
- // return 32768;
- //}
-
- return 8192;
+static size_t llama_model_max_nodes(const llama_model & model) {
+ return std::max<size_t>(8192, model.tensors_by_name.size()*5);
}
struct llama_model_loader {
@@ -3305,7 +3349,7 @@ struct llama_model_loader {
const int tensor_idx = gguf_find_tensor(gguf_ctx, name);
offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx);
- if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > llamafile_size(file->file)) {
+ if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > llamafile_size(file->file)) { // [jart]
throw std::runtime_error(format("tensor '%s' data is not within the file bounds, model is corrupted or incomplete", name));
}
}
@@ -4597,7 +4641,6 @@ static void llm_load_hparams(
} break;
case LLM_ARCH_PHI3:
{
- ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
switch (hparams.n_layer) {
@@ -4606,6 +4649,22 @@ static void llm_load_hparams(
case 40: model.type = e_model::MODEL_14B; break;
default: model.type = e_model::MODEL_UNKNOWN;
}
+
+ // for backward compatibility ; see: https://github.com/ggerganov/llama.cpp/pull/8931
+ if ((hparams.n_layer == 32 || hparams.n_layer == 40) && hparams.n_ctx_train == 4096) {
+ // default value for Phi-3-mini-4k-instruct and Phi-3-medium-4k-instruct
+ hparams.n_swa = 2047;
+ } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) {
+ // default value for Phi-3-mini-128k-instruct
+ hparams.n_swa = 262144;
+ } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) {
+ // default value for Phi-3-medium-128k-instruct
+ hparams.n_swa = 131072;
+ }
+ bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
+ if (!found_swa && hparams.n_swa == 0) {
+ throw std::runtime_error("invalid value for sliding_window");
+ }
} break;
case LLM_ARCH_PLAMO:
{
@@ -4673,6 +4732,7 @@ static void llm_load_hparams(
hparams.attn_soft_cap = true;
switch (hparams.n_layer) {
+ case 26: model.type = e_model::MODEL_2B; break;
case 42: model.type = e_model::MODEL_9B; break;
case 46: model.type = e_model::MODEL_27B; break;
default: model.type = e_model::MODEL_UNKNOWN;
@@ -4902,6 +4962,12 @@ static void llm_load_hparams(
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);
+ model.type = e_model::MODEL_UNKNOWN;
+ } break;
case LLM_ARCH_JAIS:
{
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
@@ -4914,6 +4980,23 @@ static void llm_load_hparams(
default: model.type = e_model::MODEL_UNKNOWN;
}
} break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_4B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ switch (hparams.n_layer) {
+ case 32: model.type = e_model::MODEL_8B; break;
+ default: model.type = e_model::MODEL_UNKNOWN;
+ }
+ } break;
default: (void)0;
}
@@ -5146,6 +5229,15 @@ static void llm_load_vocab(
} else if (
tokenizer_pre == "codeshell") {
vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CODESHELL;
+ } else if (
+ tokenizer_pre == "bloom") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_BLOOM;
+ } else if (
+ tokenizer_pre == "gpt3-finnish") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH;
+ } else if (
+ tokenizer_pre == "exaone") {
+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;
} else {
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
}
@@ -5289,6 +5381,7 @@ static void llm_load_vocab(
{ LLM_KV_TOKENIZER_SUFFIX_ID, vocab.special_suffix_id },
{ LLM_KV_TOKENIZER_MIDDLE_ID, vocab.special_middle_id },
{ LLM_KV_TOKENIZER_EOT_ID, vocab.special_eot_id },
+ { LLM_KV_TOKENIZER_EOM_ID, vocab.special_eom_id },
};
for (const auto & it : special_token_types) {
@@ -5341,6 +5434,17 @@ static void llm_load_vocab(
}
}
}
+
+ // find EOM token: "<|eom_id|>"
+ //
+ // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOM_ID
+ // for now, we apply this workaround to find the EOM token based on its text
+ if (vocab.special_eom_id == -1) {
+ const auto & t = vocab.token_to_id.find("<|eom_id|>");
+ if (t != vocab.token_to_id.end()) {
+ vocab.special_eom_id = t->second;
+ }
+ }
}
// build special tokens cache
@@ -7127,6 +7231,42 @@ static bool llm_load_tensors(
layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_FFN_UP, "weight", i), {n_embd, n_ff});
}
} break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;
+
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm_enc = ml.create_tensor(ctx_output, tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ // if output is NULL, init from the input tok embed
+ if (model.output == NULL) {
+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);
+ }
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_rel_b_enc = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ENC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.wq_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wk_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});
+
+ layer.ffn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_gate_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_GATE, "weight", i), {n_embd, n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_down_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
case LLM_ARCH_JAIS:
{
model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
@@ -7196,6 +7336,78 @@ static bool llm_load_tensors(
layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});
}
} break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});
+
+ // optional bias tensors
+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});
+
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+
+ // optional MLP bias
+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);
+ }
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
+
+ // output
+ {
+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ ggml_context * ctx_layer = ctx_for_layer(i);
+ ggml_context * ctx_split = ctx_for_layer_split(i);
+
+ auto & layer = model.layers[i];
+
+ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
+
+ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});
+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});
+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});
+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
+
+ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
+ layer.rope_freqs = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FREQS, "weight"), {n_embd/n_head/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});
+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});
+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});
+ }
+ } break;
default:
throw std::runtime_error("unknown architecture");
}
@@ -7882,7 +8094,7 @@ static struct ggml_tensor * llm_build_kqv(
struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);
cb(kq, "kq", il);
- if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2) {
+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_NEMOTRON) {
// for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs
// ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847
ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
@@ -11443,6 +11655,7 @@ struct llm_build_context {
// ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670e
switch (model.type) {
+ case e_model::MODEL_2B:
case e_model::MODEL_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break;
case e_model::MODEL_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break;
default: GGML_ABORT("fatal error");
@@ -12840,7 +13053,7 @@ struct llm_build_context {
return gf;
}
- struct ggml_cgraph * build_t5() {
+ struct ggml_cgraph * build_t5_encoder() {
struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
// mutable variable, needed during the last layer of the computation to skip unused tokens
@@ -12855,303 +13068,323 @@ struct llm_build_context {
inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
- if (lctx.is_encoding) {
- struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false);
+ GGML_ASSERT(lctx.is_encoding);
+ struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false);
- // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
- struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false);
-
- for (int il = 0; il < n_layer; ++il) {
- struct ggml_tensor * inpSA = inpL;
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false);
- // norm
- cur = llm_build_norm(ctx0, inpL, hparams,
- model.layers[il].attn_norm_enc, NULL,
- LLM_NORM_RMS, cb, il);
- cb(cur, "attn_norm", il);
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
- // self-attention
- {
- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq_enc, cur);
- cb(Qcur, "Qcur", il);
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm_enc, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk_enc, cur);
- cb(Kcur, "Kcur", il);
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_enc, cur);
+ cb(Qcur, "Qcur", il);
- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv_enc, cur);
- cb(Vcur, "Vcur", il);
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_enc, cur);
+ cb(Kcur, "Kcur", il);
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_enc, cur);
+ cb(Vcur, "Vcur", il);
- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
- struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
- cb(kq, "kq", il);
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
- struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
- struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b);
- struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
- cb(kq_b, "kq_b", il);
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
- kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias);
- cb(kq, "kq_soft_max_ext", il);
+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;
+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b);
+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
+ cb(kq_b, "kq_b", il);
- struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));
- cb(v, "v", il);
+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);
- cb(kqv, "kqv", il);
+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));
+ cb(v, "v", il);
- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
- cb(kqv_merged, "kqv_merged", il);
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);
+ cb(kqv, "kqv", il);
- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
- cb(cur, "kqv_merged_cont", il);
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
- ggml_build_forward_expand(gf, cur);
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
- cur = ggml_mul_mat(ctx0, model.layers[il].wo_enc, cur);
- cb(cur, "kqv_out", il);
- }
+ ggml_build_forward_expand(gf, cur);
- if (il == n_layer - 1) {
- // skip computing output for unused tokens
- struct ggml_tensor * inp_out_ids = build_inp_out_ids();
- n_tokens = n_outputs;
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
- }
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_enc, cur);
+ cb(cur, "kqv_out", il);
+ }
- struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
- cb(ffn_inp, "ffn_inp", il);
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
- // feed-forward network
- {
- cur = llm_build_norm(ctx0, ffn_inp, hparams,
- model.layers[il].ffn_norm_enc, NULL,
- LLM_NORM_RMS, cb, il);
- cb(cur, "ffn_norm", il);
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
- // T5 uses relu, flan-T5 uses gelu-gated
- cur = llm_build_ffn(ctx0, lctx, cur,
- model.layers[il].ffn_up_enc, NULL, NULL,
- model.layers[il].ffn_gate_enc, NULL, NULL,
- model.layers[il].ffn_down_enc, NULL, NULL,
- NULL,
- model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
- model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
- cb, il);
- cb(cur, "ffn_out", il);
- }
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm_enc, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
- cur = ggml_add(ctx0, cur, ffn_inp);
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up_enc, NULL, NULL,
+ model.layers[il].ffn_gate_enc, NULL, NULL,
+ model.layers[il].ffn_down_enc, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ cb, il);
cb(cur, "ffn_out", il);
+ }
- ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
- if (layer_dir != nullptr) {
- cur = ggml_add(ctx0, cur, layer_dir);
- }
- cb(cur, "l_out", il);
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
- // input for next layer
- inpL = cur;
+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
+ if (layer_dir != nullptr) {
+ cur = ggml_add(ctx0, cur, layer_dir);
}
+ cb(cur, "l_out", il);
- cur = inpL;
- cb(cur, "result_embd", -1);
-
- cur = llm_build_norm(ctx0, cur, hparams,
- model.output_norm_enc, NULL,
- LLM_NORM_RMS, cb, -1);
- cb(cur, "result_norm", -1);
- } else {
- GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first");
+ // input for next layer
+ inpL = cur;
+ }
- struct ggml_tensor * embd_enc = llm_build_inp_embd_enc();
- struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true);
+ cur = inpL;
+ cb(cur, "result_embd", -1);
- struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask();
- struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross();
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm_enc, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
- for (int il = 0; il < n_layer; ++il) {
- struct ggml_tensor * inpSA = inpL;
+ ggml_build_forward_expand(gf, cur);
- // norm
- cur = llm_build_norm(ctx0, inpL, hparams,
- model.layers[il].attn_norm, NULL,
- LLM_NORM_RMS, cb, il);
- cb(cur, "attn_norm", il);
+ return gf;
+ }
- // self-attention
- {
- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);
- cb(Qcur, "Qcur", il);
+ struct ggml_cgraph * build_t5_decoder() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);
- cb(Kcur, "Kcur", il);
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);
- cb(Vcur, "Vcur", il);
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
- llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il);
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
- struct ggml_tensor * k =
- ggml_view_3d(ctx0, kv_self.k_l[il],
- n_embd_head_k, n_kv, n_head_kv,
- ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
- ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),
- 0);
- cb(k, "k", il);
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
- struct ggml_tensor * v =
- ggml_view_3d(ctx0, kv_self.v_l[il],
- n_kv, n_embd_head_v, n_head_kv,
- ggml_element_size(kv_self.v_l[il])*n_ctx,
- ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v,
- 0);
- cb(v, "v", il);
+ GGML_ASSERT(!lctx.is_encoding);
+ GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first");
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ struct ggml_tensor * embd_enc = llm_build_inp_embd_enc();
+ struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true);
- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask();
+ struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross();
- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
- cb(kq, "kq", il);
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
- struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;
- struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b);
- struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
- cb(kq_b, "kq_b", il);
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
- kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias);
- cb(kq, "kq_soft_max_ext", il);
+ // self-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
- cb(kqv, "kqv", il);
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
- cb(kqv_merged, "kqv_merged", il);
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
- cb(cur, "kqv_merged_cont", il);
+ llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il);
- ggml_build_forward_expand(gf, cur);
+ struct ggml_tensor * k =
+ ggml_view_3d(ctx0, kv_self.k_l[il],
+ n_embd_head_k, n_kv, n_head_kv,
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),
+ ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),
+ 0);
+ cb(k, "k", il);
- cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur);
- cb(cur, "kqv_out", il);
- }
+ struct ggml_tensor * v =
+ ggml_view_3d(ctx0, kv_self.v_l[il],
+ n_kv, n_embd_head_v, n_head_kv,
+ ggml_element_size(kv_self.v_l[il])*n_ctx,
+ ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v,
+ 0);
+ cb(v, "v", il);
- cur = ggml_add(ctx0, cur, inpSA);
- cb(cur, "cross_inp", il);
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- struct ggml_tensor * inpCA = cur;
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
- // norm
- cur = llm_build_norm(ctx0, cur, hparams,
- model.layers[il].attn_norm_cross, NULL,
- LLM_NORM_RMS, cb, il);
- cb(cur, "attn_norm_cross", il);
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
- // cross-attention
- {
- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq_cross, cur);
- cb(Qcur, "Qcur", il);
+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;
+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b);
+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);
+ cb(kq_b, "kq_b", il);
- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk_cross, embd_enc);
- cb(Kcur, "Kcur", il);
+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv_cross, embd_enc);
- cb(Vcur, "Vcur", il);
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
+ cb(kqv, "kqv", il);
- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
- struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
- cb(kq, "kq", il);
+ ggml_build_forward_expand(gf, cur);
- kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);
- cb(kq, "kq_soft_max_ext", il);
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur);
+ cb(cur, "kqv_out", il);
+ }
- struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));
- cb(v, "v", il);
+ cur = ggml_add(ctx0, cur, inpSA);
+ cb(cur, "cross_inp", il);
- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);
- cb(kqv, "kqv", il);
+ struct ggml_tensor * inpCA = cur;
- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
- cb(kqv_merged, "kqv_merged", il);
+ // norm
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.layers[il].attn_norm_cross, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm_cross", il);
- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
- cb(cur, "kqv_merged_cont", il);
+ // cross-attention
+ {
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_cross, cur);
+ cb(Qcur, "Qcur", il);
- ggml_build_forward_expand(gf, cur);
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_cross, embd_enc);
+ cb(Kcur, "Kcur", il);
- cur = ggml_mul_mat(ctx0, model.layers[il].wo_cross, cur);
- cb(cur, "kqv_out", il);
- }
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_cross, embd_enc);
+ cb(Vcur, "Vcur", il);
- if (il == n_layer - 1) {
- // skip computing output for unused tokens
- struct ggml_tensor * inp_out_ids = build_inp_out_ids();
- n_tokens = n_outputs;
- cur = ggml_get_rows(ctx0, cur, inp_out_ids);
- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
- inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);
- }
+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);
- struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);
- cb(ffn_inp, "ffn_inp", il);
+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);
+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));
- // feed-forward network
- {
- cur = llm_build_norm(ctx0, ffn_inp, hparams,
- model.layers[il].ffn_norm, NULL,
- LLM_NORM_RMS, cb, il);
- cb(cur, "ffn_norm", il);
+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
+ cb(kq, "kq", il);
- // T5 uses relu, flan-T5 uses gelu-gated
- cur = llm_build_ffn(ctx0, lctx, cur,
- model.layers[il].ffn_up, NULL, NULL,
- model.layers[il].ffn_gate, NULL, NULL,
- model.layers[il].ffn_down, NULL, NULL,
- NULL,
- model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
- model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
- cb, il);
- cb(cur, "ffn_out", il);
- }
+ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);
+ cb(kq, "kq_soft_max_ext", il);
- cur = ggml_add(ctx0, cur, ffn_inp);
- cb(cur, "ffn_out", il);
+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));
+ cb(v, "v", il);
- ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
- if (layer_dir != nullptr) {
- cur = ggml_add(ctx0, cur, layer_dir);
- }
- cb(cur, "l_out", il);
+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);
+ cb(kqv, "kqv", il);
- // input for next layer
- inpL = cur;
+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
+ cb(kqv_merged, "kqv_merged", il);
+
+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);
+ cb(cur, "kqv_merged_cont", il);
+
+ ggml_build_forward_expand(gf, cur);
+
+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_cross, cur);
+ cb(cur, "kqv_out", il);
}
- cur = inpL;
- cb(cur, "result_embd", -1);
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);
+ }
- cur = llm_build_norm(ctx0, cur, hparams,
- model.output_norm, NULL,
- LLM_NORM_RMS, cb, -1);
- cb(cur, "result_norm", -1);
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ {
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ // T5 uses relu, flan-T5 uses gelu-gated
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,
+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,
+ cb, il);
+ cb(cur, "ffn_out", il);
+ }
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);
+ if (layer_dir != nullptr) {
+ cur = ggml_add(ctx0, cur, layer_dir);
+ }
+ cb(cur, "l_out", il);
- // lm_head
- cur = ggml_mul_mat(ctx0, model.output, cur);
- cb(cur, "result_output", -1);
+ // input for next layer
+ inpL = cur;
}
+ cur = inpL;
+ cb(cur, "result_embd", -1);
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
ggml_build_forward_expand(gf, cur);
return gf;
@@ -13362,6 +13595,254 @@ struct llm_build_context {
return gf;
}
+
+ struct ggml_cgraph * build_nemotron() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ //GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm,
+ model.layers[il].attn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm,
+ model.layers[il].ffn_norm_b,
+ LLM_NORM, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,
+ NULL, NULL, NULL,
+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
+ NULL,
+ LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, model.output_norm_b,
+ LLM_NORM, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
+
+ struct ggml_cgraph * build_exaone() {
+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
+
+ // mutable variable, needed during the last layer of the computation to skip unused tokens
+ int32_t n_tokens = this->n_tokens;
+
+ const int64_t n_embd_head = hparams.n_embd_head_v;
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
+ GGML_ASSERT(n_embd_head == hparams.n_rot);
+
+ struct ggml_tensor * cur;
+ struct ggml_tensor * inpL;
+
+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+
+ // inp_pos - contains the positions
+ struct ggml_tensor * inp_pos = build_inp_pos();
+
+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
+
+ for (int il = 0; il < n_layer; ++il) {
+ struct ggml_tensor * inpSA = inpL;
+
+ // norm
+ cur = llm_build_norm(ctx0, inpL, hparams,
+ model.layers[il].attn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "attn_norm", il);
+
+ // self-attention
+ {
+ // rope freq factors for llama3; may return nullptr for llama2 and other models
+ struct ggml_tensor * rope_factors = build_rope_factors(il);
+
+ // compute Q and K and RoPE them
+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
+ cb(Qcur, "Qcur", il);
+ if (model.layers[il].bq) {
+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
+ cb(Qcur, "Qcur", il);
+ }
+
+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
+ cb(Kcur, "Kcur", il);
+ if (model.layers[il].bk) {
+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
+ cb(Kcur, "Kcur", il);
+ }
+
+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
+ cb(Vcur, "Vcur", il);
+ if (model.layers[il].bv) {
+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
+ cb(Vcur, "Vcur", il);
+ }
+
+ Qcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Qcur, "Qcur", il);
+
+ Kcur = ggml_rope_ext(
+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+ cb(Kcur, "Kcur", il);
+
+ cur = llm_build_kv(ctx0, lctx, kv_self, gf,
+ model.layers[il].wo, model.layers[il].bo,
+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
+ }
+
+ if (il == n_layer - 1) {
+ // skip computing output for unused tokens
+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();
+ n_tokens = n_outputs;
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ // feed-forward network
+ cur = llm_build_norm(ctx0, ffn_inp, hparams,
+ model.layers[il].ffn_norm, NULL,
+ LLM_NORM_RMS, cb, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = llm_build_ffn(ctx0, lctx, cur,
+ model.layers[il].ffn_up, NULL, NULL,
+ model.layers[il].ffn_gate, NULL, NULL,
+ model.layers[il].ffn_down, NULL, NULL,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cb(cur, "ffn_out", il);
+
+ cur = lctx.cvec.apply_to(ctx0, cur, il);
+ cb(cur, "l_out", il);
+
+ // input for next layer
+ inpL = cur;
+ }
+
+ cur = inpL;
+
+ cur = llm_build_norm(ctx0, cur, hparams,
+ model.output_norm, NULL,
+ LLM_NORM_RMS, cb, -1);
+ cb(cur, "result_norm", -1);
+
+ // lm_head
+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+
+ ggml_build_forward_expand(gf, cur);
+
+ return gf;
+ }
};
static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {
@@ -13603,12 +14084,28 @@ static struct ggml_cgraph * llama_build_graph(
} break;
case LLM_ARCH_T5:
{
- result = llm.build_t5();
+ if (lctx.is_encoding) {
+ result = llm.build_t5_encoder();
+ } else {
+ result = llm.build_t5_decoder();
+ }
+ } break;
+ case LLM_ARCH_T5ENCODER:
+ {
+ result = llm.build_t5_encoder();
} break;
case LLM_ARCH_JAIS:
{
result = llm.build_jais();
} break;
+ case LLM_ARCH_NEMOTRON:
+ {
+ result = llm.build_nemotron();
+ } break;
+ case LLM_ARCH_EXAONE:
+ {
+ result = llm.build_exaone();
+ } break;
default:
GGML_ABORT("fatal error");
}
@@ -14051,7 +14548,7 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
// TODO: use a per-batch flag for logits presence instead
const bool has_logits = !cparams.embeddings;
- const bool has_embd = lctx.is_encoding || (cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE));
+ const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0;
@@ -14164,7 +14661,7 @@ struct llama_coder {
static int llama_decode_internal(
llama_context & lctx,
llama_batch batch_all,
- struct llama_coder *coder) { // TODO: rename back to batch
+ struct llama_coder *coder) { // [jart]
lctx.is_encoding = false;
const uint32_t n_tokens_all = batch_all.n_tokens;
@@ -14199,6 +14696,13 @@ static int llama_decode_internal(
const auto n_ubatch = cparams.n_ubatch;
+ // [jart] delete
+ // // TODO: simplify or deprecate
+ // std::vector<llama_pos> pos;
+ // std::vector<int32_t> n_seq_id;
+ // std::vector<llama_seq_id *> seq_id_arr;
+ // std::vector<std::vector<llama_seq_id>> seq_id;
+
// this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokens
const bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;
@@ -14339,12 +14843,15 @@ static int llama_decode_internal(
res = nullptr;
embd = nullptr;
} else if (cparams.embeddings) {
- res = nullptr; // do not extract logits for embedding case
- embd = gf->nodes[gf->n_nodes - 1];
- if (strcmp(embd->name, "result_embd_pooled") != 0) {
- embd = gf->nodes[gf->n_nodes - 2];
+ res = nullptr; // do not extract logits for embedding case
+ embd = nullptr;
+ for (int i = gf->n_nodes - 1; i >= 0; --i) {
+ if (strcmp(gf->nodes[i]->name, "result_embd_pooled") == 0) {
+ embd = gf->nodes[i];
+ break;
+ }
}
- GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");
+ GGML_ASSERT(embd != nullptr && "missing embeddings tensor");
} else {
embd = nullptr; // do not extract embeddings when not needed
GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
@@ -14470,7 +14977,7 @@ static int llama_decode_internal(
static int llama_encode_internal(
llama_context & lctx,
llama_batch batch,
- struct llama_coder *coder) {
+ struct llama_coder *coder) { // [jart]
lctx.is_encoding = true;
@@ -14498,6 +15005,13 @@ static int llama_encode_internal(
const int64_t n_embd = hparams.n_embd;
+ // [jart] remove
+ // // TODO: simplify or deprecate
+ // std::vector<llama_pos> pos;
+ // std::vector<int32_t> n_seq_id;
+ // std::vector<llama_seq_id *> seq_id_arr;
+ // std::vector<std::vector<llama_seq_id>> seq_id;
+
// reserve output buffer
if (llama_output_reserve(lctx, n_tokens) < n_tokens) {
LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
@@ -14546,9 +15060,24 @@ static int llama_encode_internal(
ggml_cgraph * gf = llama_build_graph(lctx, batch, false);
// the output embeddings after the final encoder normalization
- struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 1];
+ struct ggml_tensor * embd = nullptr;
- GGML_ASSERT(strcmp(embd->name, "result_norm") == 0);
+ // there are two cases here
+ if (llama_model_has_decoder(&lctx.model)) {
+ // first case is an encoder-decoder T5 model where embeddings are passed to decoder
+ embd = gf->nodes[gf->n_nodes - 1];
+ GGML_ASSERT(strcmp(embd->name, "result_norm") == 0 && "missing result_output tensor");
+ } else {
+ // second case is an encoder-only T5 model
+ if (cparams.embeddings) {
+ // only output embeddings if required
+ embd = gf->nodes[gf->n_nodes - 1];
+ if (strcmp(embd->name, "result_embd_pooled") != 0) {
+ embd = gf->nodes[gf->n_nodes - 2];
+ }
+ GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");
+ }
+ }
ggml_backend_sched_alloc_graph(lctx.sched, gf);
@@ -14561,20 +15090,54 @@ static int llama_encode_internal(
ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd);
GGML_ASSERT(backend_embd != nullptr);
- // extract token embeddings
- GGML_ASSERT(lctx.embd != nullptr);
+ if (llama_model_has_decoder(&lctx.model)) {
+ lctx.embd_enc.resize(n_tokens*n_embd);
+ float * embd_out = lctx.embd_enc.data();
- lctx.embd_enc.resize(n_tokens*n_embd);
- float * embd_out = lctx.embd_enc.data();
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));
- ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));
+ // remember the sequence ids used during the encoding - needed for cross attention later
+ lctx.seq_ids_enc.resize(n_tokens);
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ for (int s = 0; s < batch.n_seq_id[i]; s++) {
+ llama_seq_id seq_id = batch.seq_id[i][s];
+ lctx.seq_ids_enc[i].insert(seq_id);
+ }
+ }
+ } else {
+ GGML_ASSERT(lctx.embd != nullptr);
- // remember the sequence ids used during the encoding - needed for cross attention later
- lctx.seq_ids_enc.resize(n_tokens);
- for (uint32_t i = 0; i < n_tokens; i++) {
- for (int s = 0; s < batch.n_seq_id[i]; s++) {
- llama_seq_id seq_id = batch.seq_id[i][s];
- lctx.seq_ids_enc[i].insert(seq_id);
+ switch (cparams.pooling_type) {
+ case LLAMA_POOLING_TYPE_NONE:
+ {
+ // extract token embeddings
+ GGML_ASSERT(lctx.embd != nullptr);
+ float * embd_out = lctx.embd;
+
+ GGML_ASSERT(n_tokens*n_embd <= (int64_t) lctx.embd_size);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));
+ } break;
+ case LLAMA_POOLING_TYPE_MEAN:
+ case LLAMA_POOLING_TYPE_CLS:
+ case LLAMA_POOLING_TYPE_LAST:
+ {
+ // extract sequence embeddings
+ auto & embd_seq_out = lctx.embd_seq;
+ embd_seq_out.clear();
+
+ for (uint32_t i = 0; i < n_tokens; i++) {
+ const llama_seq_id seq_id = batch.seq_id[i][0];
+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {
+ continue;
+ }
+ embd_seq_out[seq_id].resize(n_embd);
+ ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));
+ }
+ } break;
+ case LLAMA_POOLING_TYPE_UNSPECIFIED:
+ {
+ GGML_ABORT("unknown pooling type");
+ }
}
}
}
@@ -15010,7 +15573,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type n
const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);
auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {
if (n_expert > 1) {
- // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly
+ // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly
// sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work
// for getting the current layer as I initially thought, and we need to resort to parsing the
// tensor name.
@@ -16043,7 +16606,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {
}
size_t llama_max_devices(void) {
- if (llamafile_has_cuda())
+ if (llamafile_has_cuda()) // [jart]
return GGML_CUDA_MAX_DEVICES;
return 1;
#if defined(GGML_USE_RPC)
@@ -16072,7 +16635,7 @@ bool llama_supports_mlock(void) {
}
bool llama_supports_gpu_offload(void) {
- if (llamafile_has_gpu())
+ if (llamafile_has_gpu()) // [jart]
return true;
#if defined(GGML_USE_CUDA) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \
defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_RPC)
@@ -16292,6 +16855,8 @@ struct llama_context * llama_new_context_with_model(
ctx->sampling.rng = std::mt19937(params.seed);
ctx->logits_all = params.logits_all;
+ // build worst-case graph for encoder if a model contains encoder
+ ctx->is_encoding = llama_model_has_encoder(model);
uint32_t kv_size = cparams.n_ctx;
ggml_type type_k = params.type_k;
@@ -16609,6 +17174,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
case LLM_ARCH_MAMBA:
case LLM_ARCH_JINA_BERT_V2:
case LLM_ARCH_T5:
+ case LLM_ARCH_T5ENCODER:
case LLM_ARCH_JAIS:
return LLAMA_ROPE_TYPE_NONE;
@@ -16647,6 +17213,8 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
case LLM_ARCH_OPENELM:
case LLM_ARCH_GPTNEOX:
case LLM_ARCH_CODESHELL:
+ case LLM_ARCH_NEMOTRON:
+ case LLM_ARCH_EXAONE:
return LLAMA_ROPE_TYPE_NEOX;
// all model arches should be listed explicitly here
@@ -16756,8 +17324,16 @@ struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const ch
bool llama_model_has_encoder(const struct llama_model * model) {
switch (model->arch) {
- case LLM_ARCH_T5: return true;
- default: return false;
+ case LLM_ARCH_T5: return true;
+ case LLM_ARCH_T5ENCODER: return true;
+ default: return false;
+ }
+}
+
+bool llama_model_has_decoder(const struct llama_model * model) {
+ switch (model->arch) {
+ case LLM_ARCH_T5ENCODER: return false;
+ default: return true;
}
}
@@ -17803,7 +18379,7 @@ static bool llama_state_load_file_internal(struct llama_context * ctx, const cha
// restore the context state
{
- const size_t n_state_size_cur = llamafile_size(file.file) - file.tell();
+ const size_t n_state_size_cur = llamafile_size(file.file) - file.tell(); // [jart]
llama_data_read_file data_ctx(&file);
const size_t n_read = llama_state_set_data_internal(ctx, data_ctx);
@@ -17940,7 +18516,7 @@ static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, con
// restore the context state
{
- const size_t state_size = llamafile_size(file.file) - file.tell();
+ const size_t state_size = llamafile_size(file.file) - file.tell(); // [jart]
llama_data_read_file data_ctx(&file);
const size_t nread = llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id);
if (!nread) {
@@ -18278,11 +18854,11 @@ llama_token llama_token_pad(const struct llama_model * model) {
return llama_token_pad_impl(model->vocab);
}
-int32_t llama_add_bos_token(const struct llama_model * model) {
+bool llama_add_bos_token(const struct llama_model * model) {
return llama_add_bos_token_impl(model->vocab);
}
-int32_t llama_add_eos_token(const struct llama_model * model) {
+bool llama_add_eos_token(const struct llama_model * model) {
return llama_add_eos_token_impl(model->vocab);
}
@@ -18583,6 +19159,22 @@ static int32_t llama_chat_apply_template_internal(
if (add_ass) {
ss << "Assistant:";
}
+ } else if (tmpl == "exaone3" || (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]"))) {
+ // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb
+ // EXAONE-3.0-7.8B-Instruct
+ for (auto message : chat) {
+ std::string role(message->role);
+ if (role == "system") {
+ ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";
+ } else if (role == "user") {
+ ss << "[|user|]" << trim(message->content) << "\n";
+ } else if (role == "assistant") {
+ ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";
+ }
+ }
+ if (add_ass) {
+ ss << "[|assistant|]";
+ }
} else {
// template not supported
return -1;
diff --git a/llama.cpp/llama.h b/llama.cpp/llama.h
index 9ec723c1..3f919ae9 100644
--- a/llama.cpp/llama.h
+++ b/llama.cpp/llama.h
@@ -95,15 +95,15 @@ extern "C" {
LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,
LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,
LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,
+ LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,
+ LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,
+ LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,
};
- // note: these values should be synchronized with ggml_rope
- // TODO: maybe move this enum to ggml.h (ggml_rope_type)
enum llama_rope_type {
LLAMA_ROPE_TYPE_NONE = -1,
- LLAMA_ROPE_TYPE_NORM = 0,
- LLAMA_ROPE_TYPE_NEOX = 2,
- LLAMA_ROPE_TYPE_GLM = 4,
+ LLAMA_ROPE_TYPE_NORM = 0,
+ LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,
};
enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file
@@ -348,7 +348,7 @@ extern "C" {
int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()
enum llama_ftype ftype; // quantize to this llama_ftype
enum ggml_type output_tensor_type; // output tensor type
- enum ggml_type token_embedding_type; // itoken embeddings tensor type
+ enum ggml_type token_embedding_type; // token embeddings tensor type
bool allow_requantize; // allow quantizing non-f32/f16 tensors
bool quantize_output_tensor; // quantize output.weight
bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored
@@ -507,6 +507,9 @@ extern "C" {
// Returns true if the model contains an encoder that requires llama_encode() call
LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);
+ // Returns true if the model contains a decoder that requires llama_decode() call
+ LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);
+
// For encoder-decoder models, this function returns id of the token that must be provided
// to the decoder to start generating output sequence. For other models, it returns -1.
LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);
@@ -915,11 +918,8 @@ extern "C" {
LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line
LLAMA_API llama_token llama_token_pad(const struct llama_model * model); // padding
- // Returns -1 if unknown, 1 for true or 0 for false.
- LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model);
-
- // Returns -1 if unknown, 1 for true or 0 for false.
- LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model);
+ LLAMA_API bool llama_add_bos_token(const struct llama_model * model);
+ LLAMA_API bool llama_add_eos_token(const struct llama_model * model);
// Codellama infill tokens
LLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix
diff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cpp
index bc0139b5..c5787641 100644
--- a/llama.cpp/quantize/quantize.cpp
+++ b/llama.cpp/quantize/quantize.cpp
@@ -95,7 +95,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp
}
// usage:
-// ./quantize [--allow-requantize] [--leave-output-tensor] [--pure] models/llama/ggml-model.gguf [models/llama/ggml-model-quant.gguf] type [nthreads]
+// ./llama-quantize [--allow-requantize] [--leave-output-tensor] [--pure] models/llama/ggml-model.gguf [models/llama/ggml-model-quant.gguf] type [nthreads]
//
[[noreturn]]
static void usage(const char * executable) {
diff --git a/stable-diffusion.cpp/README.llamafile b/stable-diffusion.cpp/README.llamafile
index 07adf8d0..0f4fbde5 100644
--- a/stable-diffusion.cpp/README.llamafile
+++ b/stable-diffusion.cpp/README.llamafile
@@ -9,8 +9,8 @@ LICENSE
ORIGIN
[email protected]:leejet/stable-diffusion.cpp.git
- 1d2af5ca3fc30dec9657a6cca8f87c20dfd83f1c
- Tue May 14 23:06:52 2024 +0800
+ 697d000f49dd4c81405e96e2dca2677dd834c9c2
+ 2024-08-10
LOCAL MODIFICATIONS
diff --git a/stable-diffusion.cpp/ggml_extend.hpp b/stable-diffusion.cpp/ggml_extend.hpp
index 16010fd2..41b59442 100644
--- a/stable-diffusion.cpp/ggml_extend.hpp
+++ b/stable-diffusion.cpp/ggml_extend.hpp
@@ -542,7 +542,8 @@ __STATIC_INLINE__ void sd_tiling(ggml_tensor* input, ggml_tensor* output, const
__STATIC_INLINE__ struct ggml_tensor* ggml_group_norm_32(struct ggml_context* ctx,
struct ggml_tensor* a) {
- return ggml_group_norm(ctx, a, 32);
+ const float eps = 1e-6f; // default eps parameter
+ return ggml_group_norm(ctx, a, 32, eps);
}
__STATIC_INLINE__ struct ggml_tensor* ggml_nn_linear(struct ggml_context* ctx,
@@ -641,7 +642,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_attention(struct ggml_context* ctx
struct ggml_tensor* k,
struct ggml_tensor* v,
bool mask = false) {
-#if defined(SD_USE_FLASH_ATTENTION) && !defined(SD_USE_CUBLAS) && !defined(SD_USE_METAL)
+#if defined(SD_USE_FLASH_ATTENTION) && !defined(SD_USE_CUBLAS) && !defined(SD_USE_METAL) && !defined(SD_USE_SYCL)
struct ggml_tensor* kqv = ggml_flash_attn(ctx, q, k, v, false); // [N * n_head, n_token, d_head]
#else
float d_head = (float)q->ne[0];
@@ -733,7 +734,8 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_group_norm(struct ggml_context* ct
b = ggml_reshape_4d(ctx, b, 1, 1, b->ne[0], 1);
}
- x = ggml_group_norm(ctx, x, num_groups);
+ const float eps = 1e-6f; // default eps parameter
+ x = ggml_group_norm(ctx, x, num_groups, eps);
if (w != NULL && b != NULL) {
x = ggml_mul(ctx, x, w);
// b = ggml_repeat(ctx, b, x);
@@ -743,7 +745,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_group_norm(struct ggml_context* ct
}
__STATIC_INLINE__ void ggml_backend_tensor_get_and_sync(ggml_backend_t backend, const struct ggml_tensor* tensor, void* data, size_t offset, size_t size) {
-#ifdef SD_USE_CUBLAS
+#if defined (SD_USE_CUBLAS) || defined (SD_USE_SYCL)
if (!ggml_backend_is_cpu(backend)) {
ggml_backend_tensor_get_async(backend, tensor, data, offset, size);
ggml_backend_synchronize(backend);
diff --git a/stable-diffusion.cpp/stable-diffusion.cpp b/stable-diffusion.cpp/stable-diffusion.cpp
index f8f08ff4..29dfd739 100644
--- a/stable-diffusion.cpp/stable-diffusion.cpp
+++ b/stable-diffusion.cpp/stable-diffusion.cpp
@@ -152,13 +152,17 @@ public:
ggml_backend_metal_log_set_callback(ggml_log_callback_default, nullptr);
backend = ggml_backend_metal_init();
#endif
+#ifdef SD_USE_SYCL
+ LOG_DEBUG("Using SYCL backend");
+ backend = ggml_backend_sycl_init(0);
+#endif
if (!backend) {
LOG_DEBUG("Using CPU backend");
backend = ggml_backend_cpu_init();
}
#ifdef SD_USE_FLASH_ATTENTION
-#if defined(SD_USE_CUBLAS) || defined(SD_USE_METAL)
+#if defined(SD_USE_CUBLAS) || defined(SD_USE_METAL) || defined (SD_USE_SYCL)
LOG_WARN("Flash Attention not supported with GPU Backend");
#else
LOG_INFO("Flash Attention enabled");
diff --git a/whisper.cpp/BUILD.mk b/whisper.cpp/BUILD.mk
index 7aa96f1c..218e9ce7 100644
--- a/whisper.cpp/BUILD.mk
+++ b/whisper.cpp/BUILD.mk
@@ -43,4 +43,5 @@ $(WHISPER_CPP_OBJS): whisper.cpp/BUILD.mk
.PHONY: o/$(MODE)/whisper.cpp
o/$(MODE)/whisper.cpp: \
- o/$(MODE)/whisper.cpp/server
+ o/$(MODE)/whisper.cpp/server \
+ o/$(MODE)/whisper.cpp/main