Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Sync with upstream projects
Justine Tunney1ca6e4292c81
diff --git a/Makefile b/Makefileindex e026b9ba..997a35db 100644--- a/Makefile+++ b/Makefile@@ -23,7 +23,7 @@ include stb/BUILD.mk.PHONY: o/$(MODE)/o/$(MODE)/: o/$(MODE)/llamafile \o/$(MODE)/llama.cpp \- o/$(MODE)/stable-diffusion.cpp \+ o/$(MODE)/stable-diffusion.cpp \o/$(MODE)/whisper.cpp \o/$(MODE)/stb \o/$(MODE)/depend.test@@ -44,6 +44,7 @@ install: llamafile/zipalign.1 \o/$(MODE)/llama.cpp/llama-bench/llama-bench \o/$(MODE)/llama.cpp/perplexity/perplexity \o/$(MODE)/llama.cpp/llava/llava-quantize \+ o/$(MODE)/whisper.cpp/main \o/$(MODE)/llamafile/server/mainmkdir -p $(PREFIX)/bin$(INSTALL) o/$(MODE)/llamafile/zipalign $(PREFIX)/bin/zipalign@@ -57,6 +58,8 @@ install: llamafile/zipalign.1 \$(INSTALL) o/$(MODE)/llama.cpp/perplexity/perplexity $(PREFIX)/bin/llamafile-perplexity$(INSTALL) o/$(MODE)/llama.cpp/llava/llava-quantize $(PREFIX)/bin/llava-quantize$(INSTALL) o/$(MODE)/llamafile/server/main $(PREFIX)/bin/llamafiler+ $(INSTALL) o/$(MODE)/stable-diffusion.cpp/main $(PREFIX)/bin/sdfile+ $(INSTALL) o/$(MODE)/whisper.cpp/main $(PREFIX)/bin/whisperfilemkdir -p $(PREFIX)/share/man/man1$(INSTALL) -m 0644 llamafile/zipalign.1 $(PREFIX)/share/man/man1/zipalign.1$(INSTALL) -m 0644 llama.cpp/main/main.1 $(PREFIX)/share/man/man1/llamafile.1diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex 68d7e732..72ff4027 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -8,12 +8,13 @@ LICENSEORIGIN- https://github.com/ggerganov/llama.cpp/pull/4406/- 152da28ae54139e3754189b9e6e1c28e11277502- 2024-05-23+ https://github.com/ggerganov/llama.cpp/+ 8b3befc0e2ed8fb18b903735831496b8b0c80949+ 2024-08-16LOCAL MODIFICATIONS+ - See [jart] and [kawrakow] annotations- Remove MAP_POPULATE because it makes mmap(tinyllama) block for 100ms- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs- Unify main, server, and llava-cli into single llamafile programdiff --git a/llama.cpp/ggml-aarch64.c b/llama.cpp/ggml-aarch64.cindex af53dea1..7adaadc9 100644--- a/llama.cpp/ggml-aarch64.c+++ b/llama.cpp/ggml-aarch64.c@@ -16,6 +16,8 @@#if defined(__GNUC__)#pragma GCC diagnostic ignored "-Woverlength-strings"+#elif defined(_MSC_VER)+#pragma warning(disable: 4244 4267) // possible loss of data#endif#define UNUSED GGML_UNUSED@@ -384,8 +386,8 @@ void ggml_gemv_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE)- if (svcntw() == 8) {- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&+ if (ggml_sve_cnt_b == QK8_0) {+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");}#endif@@ -496,8 +498,8 @@ void ggml_gemv_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE)- if (svcntw() == 8) {- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&+ if (ggml_sve_cnt_b == QK8_0) {+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");}#endif@@ -614,7 +616,7 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE) && ! ((defined(_MSC_VER)) && ! defined(__clang__))- if (svcntw() == 8) {+ if (ggml_sve_cnt_b == QK8_0) {const void * b_ptr = vx;const void * a_ptr = vy;float * res_ptr = s;@@ -680,12 +682,12 @@ void ggml_gemv_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *return;}else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal ""performance");}else if (ggml_cpu_has_neon()) {- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&"__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 ""quantization format for optimal performance");}@@ -745,8 +747,8 @@ void ggml_gemm_q4_0_4x4_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)- if (svcntw() == 8) {- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&+ if (ggml_sve_cnt_b == QK8_0) {+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");}#endif@@ -1266,8 +1268,8 @@ void ggml_gemm_q4_0_4x8_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8)- if (svcntw() == 8) {- GGML_ASSERT(!(ggml_cpu_has_sve() && (svcntw() == 8)) &&+ if (ggml_sve_cnt_b == QK8_0) {+ GGML_ASSERT(!(ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE defined, use the Q4_0_8_8 quantization format for optimal performance");}#endif@@ -1728,7 +1730,7 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *UNUSED(blocklen);#if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_MATMUL_INT8) && ! ((defined(_MSC_VER)) && ! defined(__clang__))- if (svcntw() == 8) {+ if (ggml_sve_cnt_b == QK8_0) {const void * b_ptr = vx;const void * a_ptr = vy;float * res_ptr = s;@@ -2139,12 +2141,12 @@ void ggml_gemm_q4_0_8x8_q8_0(int n, float * restrict s, size_t bs, const void *return;}else if (ggml_cpu_has_neon() && ggml_cpu_has_matmul_int8()) {- GGML_ASSERT((ggml_cpu_has_sve() && (svcntw() == 8)) &&+ GGML_ASSERT((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) &&"__ARM_FEATURE_SVE for vector size of 256-bits not defined, use the Q4_0_4_8 quantization format for optimal ""performance");}else if (ggml_cpu_has_neon()) {- GGML_ASSERT(((ggml_cpu_has_sve() && (svcntw() == 8)) || ggml_cpu_has_matmul_int8()) &&+ GGML_ASSERT(((ggml_cpu_has_sve() && (ggml_sve_cnt_b == QK8_0)) || ggml_cpu_has_matmul_int8()) &&"__ARM_FEATURE_SVE for vector size of 256-bits and __ARM_FEATURE_MATMUL_INT8 not defined, use the Q4_0_4_4 ""quantization format for optimal performance");}diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex c44d91c1..ffba3268 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -356,15 +356,10 @@ void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t b}// an async copy would normally happen after all the queued operations on both backends are completed- // sync src, set_async dst- if (ggml_backend_buffer_is_host(src->buffer)) {- ggml_backend_synchronize(backend_src);- ggml_backend_tensor_set_async(backend_dst, dst, src->data, 0, ggml_nbytes(src));- } else {- ggml_backend_synchronize(backend_src);- ggml_backend_tensor_copy(src, dst);- ggml_backend_synchronize(backend_dst);- }+ // to simulate the same behavior, we need to synchronize both backends first, and do a blocking copy+ ggml_backend_synchronize(backend_src);+ ggml_backend_synchronize(backend_dst);+ ggml_backend_tensor_copy(src, dst);}// events@@ -1028,10 +1023,6 @@ static bool ggml_is_view_op(enum ggml_op op) {#define GGML_SCHED_MAX_BACKENDS 16#endif-#ifndef GGML_SCHED_MAX_SPLITS-#define GGML_SCHED_MAX_SPLITS 2048-#endif-#ifndef GGML_SCHED_MAX_SPLIT_INPUTS#define GGML_SCHED_MAX_SPLIT_INPUTS GGML_MAX_SRC#endif@@ -1135,7 +1126,8 @@ static int ggml_backend_sched_backend_from_buffer(ggml_backend_sched_t sched, co}#if 0-static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only+#define GGML_SCHED_MAX_SPLITS_DEBUG 4096+static char causes[GGML_DEFAULT_GRAPH_SIZE*16 + GGML_SCHED_MAX_SPLITS_DEBUG*GGML_SCHED_MAX_SPLIT_INPUTS][128]; // debug only#define SET_CAUSE(node, ...) sprintf(causes[hash_id(node)], __VA_ARGS__)#define GET_CAUSE(node) causes[hash_id(node)]#else@@ -1559,7 +1551,6 @@ static void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggsched->splits = realloc(sched->splits, sched->splits_capacity * sizeof(struct ggml_backend_sched_split));GGML_ASSERT(sched->splits != NULL);}- GGML_ASSERT(i_split < GGML_SCHED_MAX_SPLITS);split = &sched->splits[i_split];split->backend_id = node_backend_id;split->i_start = i;@@ -1787,7 +1778,17 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s} else {ggml_backend_synchronize(split_backend);}- ggml_backend_tensor_copy_async(input_backend, split_backend, input, input_cpy);+ // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events+ // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface+ if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {+ ggml_backend_synchronize(input_backend);+ if (sched->events[split_backend_id][sched->cur_copy] != NULL) {+ ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);+ } else {+ ggml_backend_synchronize(split_backend);+ }+ ggml_backend_tensor_copy(input, input_cpy);+ }}}@@ -1865,13 +1866,14 @@ ggml_backend_sched_t ggml_backend_sched_new(sched->hv_tensor_backend_ids = malloc(sched->hash_set.size * sizeof(sched->hv_tensor_backend_ids[0]));sched->hv_tensor_copies = malloc(sched->hash_set.size * sched->n_backends * sched->n_copies * sizeof(struct ggml_tensor *));- const size_t nodes_size = graph_size + GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2;+ const size_t ggml_sched_max_splits = graph_size; // at most there is one split for each node in the graph+ const size_t nodes_size = graph_size + ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2;sched->node_backend_ids = calloc(nodes_size, sizeof(sched->node_backend_ids[0]));sched->leaf_backend_ids = calloc(nodes_size, sizeof(sched->leaf_backend_ids[0]));sched->prev_node_backend_ids = calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));sched->prev_leaf_backend_ids = calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));- sched->context_buffer_size = GGML_SCHED_MAX_SPLITS*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);+ sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);sched->context_buffer = malloc(sched->context_buffer_size);const int initial_splits_capacity = 16;diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex 90149072..9e56788c 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -6,6 +6,9 @@#ifdef GGML_USE_HIPBLAS#define GGML_CUDA_NAME "ROCm"#define GGML_CUBLAS_NAME "hipBLAS"+#elif defined(GGML_USE_MUSA)+#define GGML_CUDA_NAME "MUSA"+#define GGML_CUBLAS_NAME "muBLAS"#else#define GGML_CUDA_NAME "CUDA"#define GGML_CUBLAS_NAME "cuBLAS"diff --git a/llama.cpp/ggml-impl.h b/llama.cpp/ggml-impl.hindex 119ca3c5..190af081 100644--- a/llama.cpp/ggml-impl.h+++ b/llama.cpp/ggml-impl.h@@ -80,8 +80,9 @@ static inline float ggml_compute_bf16_to_fp32(ggml_bf16_t h) {/*** Converts float32 to brain16.*- * This function is binary identical to AMD Zen4 VCVTNEPS2BF16.- * Subnormals shall be flushed to zero, and NANs will be quiet.+ * This is binary identical with Google Brain float conversion.+ * Floats shall round to nearest even, and NANs shall be quiet.+ * Subnormals aren't flushed to zero, except perhaps when used.* This code should vectorize nicely if using modern compilers.*/static inline ggml_bf16_t ggml_compute_fp32_to_bf16(float s) {@@ -142,6 +143,7 @@ extern "C" {#if defined(__ARM_FEATURE_SVE)#include <arm_sve.h>+#include <sys/prctl.h>#endif// 16-bit floatdiff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.hindex 1d536289..4377cf8f 100644--- a/llama.cpp/ggml-metal.h+++ b/llama.cpp/ggml-metal.h@@ -1,7 +1,7 @@// An interface allowing to compute ggml_cgraph with Metal//// This is a fully functional interface that extends ggml with GPU support for Apple devices.-// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)+// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.)//// How it works?//@@ -52,6 +52,8 @@ GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(voidGGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);+GGML_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data);+GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);// helper to check if the device supports a specific family@@ -65,4 +67,3 @@ GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);#ifdef __cplusplus}#endif-diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 7aeb8d88..dbe51253 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -239,7 +239,7 @@ enum ggml_metal_kernel_type {GGML_METAL_KERNEL_TYPE_COUNT};-struct ggml_metal_context {+struct ggml_backend_metal_context {int n_cb;id<MTLDevice> device;@@ -253,6 +253,10 @@ struct ggml_metal_context {bool support_simdgroup_mm;bool should_capture_next_compute;++ // abort ggml_metal_graph_compute if callback returns true+ ggml_abort_callback abort_callback;+ void * abort_callback_data;};// MSL code@@ -320,7 +324,7 @@ static void * ggml_metal_host_malloc(size_t n) {return data;}-static struct ggml_metal_context * ggml_metal_init(int n_cb) {+static struct ggml_backend_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_LOG_INFO("%s: allocating\n", __func__);#if TARGET_OS_OSX && !GGML_METAL_NDEBUG@@ -337,7 +341,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_LOG_INFO("%s: picking default device: %s\n", __func__, [[device name] UTF8String]);// Configure context- struct ggml_metal_context * ctx = malloc(sizeof(struct ggml_metal_context));+ struct ggml_backend_metal_context * ctx = calloc(1, sizeof(struct ggml_backend_metal_context));ctx->device = device;ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);ctx->queue = [ctx->device newCommandQueue];@@ -699,7 +703,7 @@ static struct ggml_metal_context * ggml_metal_init(int n_cb) {return ctx;}-static void ggml_metal_free(struct ggml_metal_context * ctx) {+static void ggml_metal_free(struct ggml_backend_metal_context * ctx) {GGML_METAL_LOG_INFO("%s: deallocating\n", __func__);for (int i = 0; i < GGML_METAL_KERNEL_TYPE_COUNT; ++i) {@@ -765,7 +769,7 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_tensor * t, size_t * offsreturn nil;}-static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const struct ggml_tensor * op) {+static bool ggml_metal_supports_op(const struct ggml_backend_metal_context * ctx, const struct ggml_tensor * op) {for (size_t i = 0, n = 3; i < n; ++i) {if (op->src[i] != NULL && op->src[i]->type == GGML_TYPE_BF16) {return false;@@ -876,7 +880,7 @@ static bool ggml_metal_supports_op(const struct ggml_metal_context * ctx, const}static enum ggml_status ggml_metal_graph_compute(- struct ggml_metal_context * ctx,+ struct ggml_backend_metal_context * ctx,struct ggml_cgraph * gf) {@autoreleasepool {@@ -909,8 +913,11 @@ static enum ggml_status ggml_metal_graph_compute(id<MTLCommandBuffer> command_buffer = [ctx->queue commandBufferWithUnretainedReferences];command_buffer_builder[cb_idx] = command_buffer;- // enqueue the command buffers in order to specify their execution order- [command_buffer enqueue];+ // always enqueue the first two command buffers+ // enqueue all of the command buffers if we don't need to abort+ if (cb_idx < 2 || ctx->abort_callback == NULL) {+ [command_buffer enqueue];+ }}const id<MTLCommandBuffer> *command_buffers = command_buffer_builder;@@ -2260,10 +2267,8 @@ static enum ggml_status ggml_metal_graph_compute(GGML_ASSERT(ne00 % 4 == 0);GGML_ASSERT(ggml_is_contiguous(src0));- //float eps;- //memcpy(&eps, dst->op_params, sizeof(float));-- const float eps = 1e-6f; // TODO: temporarily hardcoded+ float eps;+ memcpy(&eps, dst->op_params + 1, sizeof(float));const int32_t n_groups = ((int32_t *) dst->op_params)[0];@@ -2339,7 +2344,7 @@ static enum ggml_status ggml_metal_graph_compute(memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));- const bool is_neox = mode & 2;+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;id<MTLComputePipelineState> pipeline = nil;@@ -2860,7 +2865,9 @@ static enum ggml_status ggml_metal_graph_compute([encoder endEncoding];- [command_buffer commit];+ if (cb_idx < 2 || ctx->abort_callback == NULL) {+ [command_buffer commit];+ }});// Wait for completion and check status of each command buffer@@ -2880,6 +2887,23 @@ static enum ggml_status ggml_metal_graph_compute(return GGML_STATUS_FAILED;}++ id<MTLCommandBuffer> next_buffer = (i + 1 < n_cb ? command_buffers[i + 1] : nil);+ if (!next_buffer) {+ continue;+ }++ bool next_queued = ([next_buffer status] != MTLCommandBufferStatusNotEnqueued);+ if (next_queued) {+ continue;+ }++ if (ctx->abort_callback && ctx->abort_callback(ctx->abort_callback_data)) {+ GGML_METAL_LOG_INFO("%s: command buffer %d aborted", __func__, i);+ return GGML_STATUS_ABORTED;+ }++ [next_buffer commit];}if (should_capture) {@@ -3183,7 +3207,7 @@ GGML_CALL static const char * ggml_backend_metal_name(ggml_backend_t backend) {}GGML_CALL static void ggml_backend_metal_free(ggml_backend_t backend) {- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;ggml_metal_free(ctx);free(backend);}@@ -3195,13 +3219,13 @@ GGML_CALL static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffe}GGML_CALL static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;return ggml_metal_graph_compute(metal_ctx, cgraph);}GGML_CALL static bool ggml_backend_metal_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {- struct ggml_metal_context * metal_ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * metal_ctx = (struct ggml_backend_metal_context *)backend->context;return ggml_metal_supports_op(metal_ctx, op);}@@ -3246,9 +3270,9 @@ static ggml_guid_t ggml_backend_metal_guid(void) {}ggml_backend_t ggml_backend_metal_init(void) {- struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);-+ struct ggml_backend_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);if (ctx == NULL) {+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate context\n", __func__);return NULL;}@@ -3270,15 +3294,24 @@ bool ggml_backend_is_metal(ggml_backend_t backend) {void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {GGML_ASSERT(ggml_backend_is_metal(backend));- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;ctx->n_cb = MIN(n_cb, GGML_METAL_MAX_BUFFERS);}+void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data) {+ GGML_ASSERT(ggml_backend_is_metal(backend));++ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;++ ctx->abort_callback = abort_callback;+ ctx->abort_callback_data = user_data;+}+bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {GGML_ASSERT(ggml_backend_is_metal(backend));- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;return [ctx->device supportsFamily:(MTLGPUFamilyApple1 + family - 1)];}@@ -3286,7 +3319,7 @@ bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family) {void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {GGML_ASSERT(ggml_backend_is_metal(backend));- struct ggml_metal_context * ctx = (struct ggml_metal_context *)backend->context;+ struct ggml_backend_metal_context * ctx = (struct ggml_backend_metal_context *)backend->context;ctx->should_capture_next_compute = true;}diff --git a/llama.cpp/ggml-quants.h b/llama.cpp/ggml-quants.hindex 97c19c64..525d5ee3 100644--- a/llama.cpp/ggml-quants.h+++ b/llama.cpp/ggml-quants.h@@ -127,7 +127,10 @@ void iq2xs_free_impl(enum ggml_type type);void iq3xs_init_impl(int grid_size);void iq3xs_free_impl(int grid_size);+#if defined(__ARM_FEATURE_SVE)+extern int ggml_sve_cnt_b;+#endif+#ifdef __cplusplus}#endif-diff --git a/llama.cpp/ggml-quants.inc b/llama.cpp/ggml-quants.incindex 2a5200eb..3ac736bb 100644--- a/llama.cpp/ggml-quants.inc+++ b/llama.cpp/ggml-quants.inc@@ -3857,7 +3857,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * rfloat sumf = 0;#if defined(__ARM_FEATURE_SVE)- if (svcntb() == QK8_0) {+ if (ggml_sve_cnt_b == QK8_0) {const svbool_t ptrueh = svptrue_pat_b8(SV_VL16);const svbool_t ptruel = svnot_b_z(svptrue_b8(), ptrueh);@@ -5342,7 +5342,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * rfloat sumf = 0;#if defined(__ARM_FEATURE_SVE)- if (svcntb() == QK8_0) {+ if (ggml_sve_cnt_b == QK8_0) {svfloat32_t sumv0 = svdup_n_f32(0.0f);svfloat32_t sumv1 = svdup_n_f32(0.0f);diff --git a/llama.cpp/ggml-vector-amd-avx.c b/llama.cpp/ggml-vector-amd-avx.cindex 5309e530..70171828 100644--- a/llama.cpp/ggml-vector-amd-avx.c+++ b/llama.cpp/ggml-vector-amd-avx.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avxdiff --git a/llama.cpp/ggml-vector-amd-avx2.c b/llama.cpp/ggml-vector-amd-avx2.cindex 547a7b1b..7eaab383 100644--- a/llama.cpp/ggml-vector-amd-avx2.c+++ b/llama.cpp/ggml-vector-amd-avx2.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx2#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx2#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx2+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx2#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx2#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx2#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx2diff --git a/llama.cpp/ggml-vector-amd-avx512.c b/llama.cpp/ggml-vector-amd-avx512.cindex 4f07c007..866cf72c 100644--- a/llama.cpp/ggml-vector-amd-avx512.c+++ b/llama.cpp/ggml-vector-amd-avx512.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512diff --git a/llama.cpp/ggml-vector-amd-avx512bf16.c b/llama.cpp/ggml-vector-amd-avx512bf16.cindex f0278fc6..1d470f74 100644--- a/llama.cpp/ggml-vector-amd-avx512bf16.c+++ b/llama.cpp/ggml-vector-amd-avx512bf16.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512bf16#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512bf16#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512bf16+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512bf16#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512bf16#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512bf16#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512bf16diff --git a/llama.cpp/ggml-vector-amd-avx512vl.c b/llama.cpp/ggml-vector-amd-avx512vl.cindex 4e13c22d..bd27e54e 100644--- a/llama.cpp/ggml-vector-amd-avx512vl.c+++ b/llama.cpp/ggml-vector-amd-avx512vl.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_avx512vl#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_avx512vl#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_avx512vl+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_avx512vl#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_avx512vl#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_avx512vl#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_avx512vldiff --git a/llama.cpp/ggml-vector-amd-f16c.c b/llama.cpp/ggml-vector-amd-f16c.cindex 21cbc28a..7e458f30 100644--- a/llama.cpp/ggml-vector-amd-f16c.c+++ b/llama.cpp/ggml-vector-amd-f16c.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_f16c#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_f16c#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_f16c+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_f16c#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_f16c#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_f16c#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_f16cdiff --git a/llama.cpp/ggml-vector-amd-fma.c b/llama.cpp/ggml-vector-amd-fma.cindex 0e3793e2..c38ade50 100644--- a/llama.cpp/ggml-vector-amd-fma.c+++ b/llama.cpp/ggml-vector-amd-fma.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_fma#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_fma#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_fma+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_fma#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_fma#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_fma#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_fmadiff --git a/llama.cpp/ggml-vector-amd-k8.c b/llama.cpp/ggml-vector-amd-k8.cindex bf730915..970559f5 100644--- a/llama.cpp/ggml-vector-amd-k8.c+++ b/llama.cpp/ggml-vector-amd-k8.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_k8#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_k8#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_k8+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_k8#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_k8#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_k8#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_k8diff --git a/llama.cpp/ggml-vector-amd-ssse3.c b/llama.cpp/ggml-vector-amd-ssse3.cindex c86127be..15806ef3 100644--- a/llama.cpp/ggml-vector-amd-ssse3.c+++ b/llama.cpp/ggml-vector-amd-ssse3.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_amd_ssse3#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_amd_ssse3#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_amd_ssse3+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_amd_ssse3#define ggml_vec_dot_f32 ggml_vec_dot_f32_amd_ssse3#define ggml_vec_dot_f16 ggml_vec_dot_f16_amd_ssse3#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_amd_ssse3diff --git a/llama.cpp/ggml-vector-arm80.c b/llama.cpp/ggml-vector-arm80.cindex ecc1a178..3733eba3 100644--- a/llama.cpp/ggml-vector-arm80.c+++ b/llama.cpp/ggml-vector-arm80.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm80#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm80#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm80+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_arm80#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm80#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm80#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm80diff --git a/llama.cpp/ggml-vector-arm82.c b/llama.cpp/ggml-vector-arm82.cindex cd0eba76..1d87d7c1 100644--- a/llama.cpp/ggml-vector-arm82.c+++ b/llama.cpp/ggml-vector-arm82.c@@ -3,6 +3,7 @@#define ggml_fp32_to_fp16_row ggml_fp32_to_fp16_row_arm82#define ggml_bf16_to_fp32_row ggml_bf16_to_fp32_row_arm82#define ggml_fp32_to_bf16_row ggml_fp32_to_bf16_row_arm82+#define ggml_fp32_to_bf16_row_ref ggml_fp32_to_bf16_row_ref_arm82#define ggml_vec_dot_f32 ggml_vec_dot_f32_arm82#define ggml_vec_dot_f16 ggml_vec_dot_f16_arm82#define ggml_vec_dot_bf16 ggml_vec_dot_bf16_arm82diff --git a/llama.cpp/ggml-vector.cpp b/llama.cpp/ggml-vector.cppindex f370ee6d..db44d372 100644--- a/llama.cpp/ggml-vector.cpp+++ b/llama.cpp/ggml-vector.cpp@@ -51,6 +51,18 @@ extern "C" void ggml_fp32_to_bf16_row_amd_k8(const float * x, ggml_bf16_t * y, iextern "C" void ggml_fp32_to_bf16_row_arm82(const float * x, ggml_bf16_t * y, int64_t n);extern "C" void ggml_fp32_to_bf16_row_arm80(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512bf16(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512vl(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx512(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx2(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_f16c(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_fma(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_avx(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_ssse3(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_amd_k8(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_arm82(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_fp32_to_bf16_row_ref_arm80(const float * x, ggml_bf16_t * y, int64_t n);+extern "C" void ggml_vec_dot_f32_amd_avx512bf16(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);extern "C" void ggml_vec_dot_f32_amd_avx512vl(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);extern "C" void ggml_vec_dot_f32_amd_avx512(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);@@ -668,6 +680,7 @@ static const struct VectorFuncs {typeof(ggml_fp32_to_fp16_row) *ptr_ggml_fp32_to_fp16_row;typeof(ggml_bf16_to_fp32_row) *ptr_ggml_bf16_to_fp32_row;typeof(ggml_fp32_to_bf16_row) *ptr_ggml_fp32_to_bf16_row;+ typeof(ggml_fp32_to_bf16_row_ref) *ptr_ggml_fp32_to_bf16_row_ref;typeof(ggml_vec_dot_f32) *ptr_ggml_vec_dot_f32;typeof(ggml_vec_dot_f16) *ptr_ggml_vec_dot_f16;typeof(ggml_vec_dot_bf16) *ptr_ggml_vec_dot_bf16;@@ -727,6 +740,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512bf16;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512bf16;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512bf16;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512bf16;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512bf16;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512bf16;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512bf16;@@ -787,6 +801,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512vl;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512vl;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512vl;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512vl;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512vl;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512vl;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512vl;@@ -847,6 +862,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx512;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx512;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx512;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx512;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx512;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx512;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx512;@@ -907,6 +923,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx2;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx2;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx2;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx2;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx2;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx2;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx2;@@ -967,6 +984,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_f16c;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_f16c;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_f16c;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_f16c;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_f16c;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_f16c;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_f16c;@@ -1027,6 +1045,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_fma;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_fma;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_fma;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_fma;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_fma;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_fma;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_fma;@@ -1087,6 +1106,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_avx;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_avx;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_avx;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_avx;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_avx;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_avx;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_avx;@@ -1147,6 +1167,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_ssse3;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_ssse3;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_ssse3;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_ssse3;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_ssse3;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_ssse3;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_ssse3;@@ -1207,6 +1228,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_amd_k8;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_amd_k8;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_amd_k8;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_amd_k8;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_amd_k8;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_amd_k8;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_amd_k8;@@ -1267,6 +1289,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm82;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm82;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm82;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_arm82;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm82;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm82;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm82;@@ -1327,6 +1350,7 @@ static const struct VectorFuncs {ptr_ggml_fp32_to_fp16_row = ggml_fp32_to_fp16_row_arm80;ptr_ggml_bf16_to_fp32_row = ggml_bf16_to_fp32_row_arm80;ptr_ggml_fp32_to_bf16_row = ggml_fp32_to_bf16_row_arm80;+ ptr_ggml_fp32_to_bf16_row_ref = ggml_fp32_to_bf16_row_ref_arm80;ptr_ggml_vec_dot_f32 = ggml_vec_dot_f32_arm80;ptr_ggml_vec_dot_f16 = ggml_vec_dot_f16_arm80;ptr_ggml_vec_dot_bf16 = ggml_vec_dot_bf16_arm80;@@ -1400,6 +1424,10 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {return funcs.ptr_ggml_fp32_to_bf16_row(x, y, n);}+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {+ return funcs.ptr_ggml_fp32_to_bf16_row_ref(x, y, n);+}+void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc) {return funcs.ptr_ggml_vec_dot_f32(n, s, bs, x, bx, y, by, nrc);}diff --git a/llama.cpp/ggml-vector.h b/llama.cpp/ggml-vector.hindex 938d5c37..269bc883 100644--- a/llama.cpp/ggml-vector.h+++ b/llama.cpp/ggml-vector.h@@ -10,6 +10,7 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n);void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n);void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n);void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n);+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n);void ggml_vec_dot_f32(int n, float * s, size_t bs, const float * x, size_t bx, const float * y, size_t by, int nrc);void ggml_vec_dot_f16(int n, float * s, size_t bs, ggml_fp16_t * x, size_t bx, ggml_fp16_t * y, size_t by, int nrc);void ggml_vec_dot_bf16(int n, float * s, size_t bs, ggml_bf16_t * x, size_t bx, ggml_bf16_t * y, size_t by, int nrc);diff --git a/llama.cpp/ggml-vector.inc b/llama.cpp/ggml-vector.incindex 0554ba3c..04be033a 100644--- a/llama.cpp/ggml-vector.inc+++ b/llama.cpp/ggml-vector.inc@@ -9,6 +9,11 @@#include <pthread.h>#include <cosmo.h>+#if __FINITE_MATH_ONLY__+#error "some routines in ggml.c require non-finite math arithmetics -- pass -fno-finite-math-only to the compiler to fix"+#error "ref: https://github.com/ggerganov/llama.cpp/pull/7154#issuecomment-2143844461"+#endif+/* #define GGML_GELU_FP16 */#define UNUSED GGML_UNUSED@@ -862,7 +867,7 @@ void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {int i = 0;-#if defined(__AVX512BF16__) && 0 // [jart] it kills subnormals+#if defined(__AVX512BF16__)for (; i + 32 <= n; i += 32) {_mm512_storeu_si512((__m512i *)(y + i),@@ -875,6 +880,12 @@ void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {}}+void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {+ for (int i = 0; i < n; i++) {+ y[i] = ggml_compute_fp32_to_bf16(x[i]);+ }+}+//// fundamental operations//@@ -1608,6 +1619,7 @@ ggml_vgeluf(__m128 x)void ggml_vec_gelu_f32(const int n, float * y, const float * x) {int i = 0;+if (!FLAG_trap) { // [jart] preserve this line#if defined(__ARM_NEON) && defined(__aarch64__)for (; i + 3 < n; i += 4) {@@ -1627,6 +1639,7 @@ void ggml_vec_gelu_f32(const int n, float * y, const float * x) {}#endif} // [jart] preserve this line+if (i == n)return;#ifdef GGML_GELU_FP16diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 70a427c3..9db5bf9f 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -639,7 +639,7 @@ static const ggml_type_traits_t type_traits[GGML_TYPE_COUNT] = {.is_quantized = false,.to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row,.from_float = (ggml_from_float_t) ggml_fp32_to_bf16_row,- .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row,+ .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_bf16,.vec_dot_type = GGML_TYPE_BF16,.nrows = 1,@@ -1537,21 +1537,21 @@ struct ggml_phaser {};struct ggml_compute_state_shared {- const struct ggml_cgraph * cgraph;- const struct ggml_cplan * cplan;+ const struct ggml_cgraph * cgraph;+ const struct ggml_cplan * cplan;- int n_threads;+ int n_threads;- // synchronization primitives- atomic_int n_barrier;- struct ggml_phaser *n_barrier_passed;+ // synchronization primitives+ atomic_int n_barrier;+ struct ggml_phaser *n_barrier_passed;- ggml_abort_callback abort_callback; // abort ggml_graph_compute when true- void * abort_callback_data;+ ggml_abort_callback abort_callback; // abort ggml_graph_compute when true+ void * abort_callback_data;- atomic_int current_chunk; // currently processing chunk during mul_mat, shared between all the threads+ atomic_int current_chunk; // currently processing chunk during mul_mat, shared between all the threads- enum ggml_status ec;+ enum ggml_status ec;};struct ggml_compute_state {@@ -1652,7 +1652,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {"CROSS_ENTROPY_LOSS_BACK",};-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 74");+static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74");static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"none",@@ -1740,7 +1740,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {"cross_entropy_loss_back(x,y)",};-static_assert(GGML_OP_COUNT == 76, "GGML_OP_COUNT != 74");+static_assert(GGML_OP_COUNT == 74, "GGML_OP_COUNT != 74");static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");@@ -2461,7 +2461,8 @@ static struct ggml_tensor * ggml_new_tensor_impl(struct ggml_tensor * view_src,size_t view_offs) {- assert(n_dims >= 1 && n_dims <= GGML_MAX_DIMS);+ GGML_ASSERT(type >= 0 && type < GGML_TYPE_COUNT);+ GGML_ASSERT(n_dims >= 1 && n_dims <= GGML_MAX_DIMS);// find the base tensor and absolute offsetif (view_src != NULL && view_src->view_src != NULL) {@@ -4133,6 +4134,7 @@ static struct ggml_tensor * ggml_group_norm_impl(struct ggml_context * ctx,struct ggml_tensor * a,int n_groups,+ float eps,bool inplace) {bool is_node = false;@@ -4143,7 +4145,8 @@ static struct ggml_tensor * ggml_group_norm_impl(struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);- result->op_params[0] = n_groups;+ ggml_set_op_params_i32(result, 0, n_groups);+ ggml_set_op_params_f32(result, 1, eps);result->op = GGML_OP_GROUP_NORM;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4155,15 +4158,17 @@ static struct ggml_tensor * ggml_group_norm_impl(struct ggml_tensor * ggml_group_norm(struct ggml_context * ctx,struct ggml_tensor * a,- int n_groups) {- return ggml_group_norm_impl(ctx, a, n_groups, false);+ int n_groups,+ float eps) {+ return ggml_group_norm_impl(ctx, a, n_groups, eps, false);}struct ggml_tensor * ggml_group_norm_inplace(struct ggml_context * ctx,struct ggml_tensor * a,- int n_groups) {- return ggml_group_norm_impl(ctx, a, n_groups, true);+ int n_groups,+ float eps) {+ return ggml_group_norm_impl(ctx, a, n_groups, eps, true);}// ggml_mul_mat@@ -10854,10 +10859,11 @@ static void ggml_compute_forward_group_norm_f32(GGML_TENSOR_UNARY_OP_LOCALS- const float eps = 1e-6f; // TODO: make this a parameter-// TODO: optimize+ float eps;+ memcpy(&eps, dst->op_params + 1, sizeof(float));+int n_channels = src0->ne[2];int n_groups = dst->op_params[0];int n_channels_per_group = (n_channels + n_groups - 1) / n_groups;@@ -11125,6 +11131,11 @@ UseGgmlGemm1:;ggml_barrier(params);}+ if (ith == 0) {+ // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.+ atomic_store(¶ms->shared->current_chunk, nth);+ }+#if GGML_USE_LLAMAFILEif (src1->type != vec_dot_type) {const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;@@ -11150,11 +11161,6 @@ UseGgmlGemm1:;UseGgmlGemm2:;#endif- if (ith == 0) {- // Every thread starts at ith, so the first unprocessed chunk is nth. This save a bit of coordination right at the start.- atomic_store(¶ms->shared->current_chunk, nth);- }-// This is the size of the first dimension of the result, so we can iterate that way. (see the ASSERT above, these are the same numbers)const int64_t nr0 = ne0;@@ -12840,7 +12846,7 @@ static void ggml_compute_forward_rope_f32(float corr_dims[2];ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);- const bool is_neox = mode & 2;+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;const float * freq_factors = NULL;if (src2 != NULL) {@@ -12965,7 +12971,7 @@ static void ggml_compute_forward_rope_f16(float corr_dims[2];ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims);- const bool is_neox = mode & 2;+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;const float * freq_factors = NULL;if (src2 != NULL) {@@ -17737,13 +17743,41 @@ enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplllamafile_debug_graph = cgraph;#endif+#ifdef GGML_USE_OPENMP+ if (n_threads > 1) {+ #pragma omp parallel num_threads(n_threads)+ {+ #pragma omp single+ {+ // update the number of threads from the actual number of threads that we got from OpenMP+ n_threads = omp_get_num_threads();+ state_shared.n_threads = n_threads;+ }++ struct ggml_compute_state worker = {+ .thrd = 0,+ .ith = omp_get_thread_num(),+ .shared = &state_shared,+ };+ ggml_graph_compute_thread(&worker);+ }+ } else {+ struct ggml_compute_state worker = {+ .thrd = 0,+ .ith = 0,+ .shared = &state_shared,+ };+ ggml_graph_compute_thread(&worker);+ }+#endif+struct ggml_compute_state * workers = alloca(sizeof(struct ggml_compute_state)*n_threads);struct ggml_compute_cleanup cleanup = {n_threads, workers};pthread_cleanup_push(ggml_compute_canceled, &cleanup);for (int j = 0; j < n_threads; ++j) {workers[j] = (struct ggml_compute_state) {- .thrd = ATOMIC_VAR_INIT(0),+ .thrd = 0,.ith = j,.shared = &state_shared,.is_main_thread = false, // [jart]@@ -19496,7 +19530,7 @@ size_t ggml_quantize_chunk(case GGML_TYPE_BF16:{size_t elemsize = sizeof(ggml_bf16_t);- ggml_fp32_to_bf16_row(src + start, (ggml_bf16_t *)dst + start, n);+ ggml_fp32_to_bf16_row_ref(src + start, (ggml_bf16_t *)dst + start, n);result = n * elemsize;} break;case GGML_TYPE_F32:@@ -19938,7 +19972,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_i(int64_t) info->ne[2] *(int64_t) info->ne[3];- if (ne % ggml_blck_size(info->type) != 0) {+ if (ggml_blck_size(info->type) == 0 || ne % ggml_blck_size(info->type) != 0) {fprintf(stderr, "%s: tensor '%s' of type %d (%s) number of elements (%" PRId64 ") is not a multiple of block size (%" PRId64 ")\n",__func__, info->name.data, (int) info->type, ggml_type_name(info->type), ne, ggml_blck_size(info->type));gguf_free(ctx);diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 1184065c..bcd17317 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -247,6 +247,8 @@#define GGML_EXIT_SUCCESS 0#define GGML_EXIT_ABORTED 1+#define GGML_ROPE_TYPE_NEOX 2+#define GGUF_MAGIC "GGUF"#define GGUF_VERSION 3@@ -352,6 +354,7 @@ extern "C" {GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);+ GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t);GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t);struct ggml_object;@@ -500,9 +503,7 @@ extern "C" {GGML_OP_ARGSORT,GGML_OP_LEAKY_RELU,- GGML_OP_FLASH_ATTN,GGML_OP_FLASH_ATTN_EXT,- GGML_OP_FLASH_FF,GGML_OP_FLASH_ATTN_BACK,GGML_OP_SSM_CONV,GGML_OP_SSM_SCAN,@@ -1166,16 +1167,17 @@ extern "C" {// group normalize along ne0*ne1*n_groups// used in stable-diffusion- // TODO: eps is hardcoded to 1e-6 for nowGGML_API struct ggml_tensor * ggml_group_norm(struct ggml_context * ctx,struct ggml_tensor * a,- int n_groups);+ int n_groups,+ float eps);GGML_API struct ggml_tensor * ggml_group_norm_inplace(struct ggml_context * ctx,struct ggml_tensor * a,- int n_groups);+ int n_groups,+ float eps);// a - x// b - dy@@ -1478,11 +1480,10 @@ extern "C" {struct ggml_tensor * b);// rotary position embedding- // if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)- // if mode & 2 == 1, GPT-NeoX style+ // if (mode & 1) - skip n_past elements (NOT SUPPORTED)+ // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style//// b is an int32 vector with size a->ne[2], it contains the positions- // c is freq factors (e.g. phi3-128k), (optional)GGML_API struct ggml_tensor * ggml_rope(struct ggml_context * ctx,struct ggml_tensor * a,@@ -1499,6 +1500,7 @@ extern "C" {int mode);// custom RoPE+ // c is freq factors (e.g. phi3-128k), (optional)GGML_API struct ggml_tensor * ggml_rope_ext(struct ggml_context * ctx,struct ggml_tensor * a,diff --git a/llama.cpp/llama-impl.h b/llama.cpp/llama-impl.hindex dcc8c1c1..399b134a 100644--- a/llama.cpp/llama-impl.h+++ b/llama.cpp/llama-impl.h@@ -24,3 +24,18 @@ void llama_log_callback_default(ggml_log_level level, const char * text, void *#define LLAMA_LOG_INFO(...) llama_log_internal(GGML_LOG_LEVEL_INFO , __VA_ARGS__)#define LLAMA_LOG_WARN(...) llama_log_internal(GGML_LOG_LEVEL_WARN , __VA_ARGS__)#define LLAMA_LOG_ERROR(...) llama_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)++//+// helpers+//++static void replace_all(std::string & s, const std::string & search, const std::string & replace) {+ if (search.empty()) {+ return; // Avoid infinite loop if 'search' is an empty string+ }+ size_t pos = 0;+ while ((pos = s.find(search, pos)) != std::string::npos) {+ s.replace(pos, search.length(), replace);+ pos += replace.length();+ }+}diff --git a/llama.cpp/llama-sampling.cpp b/llama.cpp/llama-sampling.cppindex fd258947..804663f9 100644--- a/llama.cpp/llama-sampling.cpp+++ b/llama.cpp/llama-sampling.cpp@@ -87,14 +87,14 @@ void llama_sample_top_k_impl(struct llama_sampling * smpl, llama_token_data_arraconstexpr float bucket_low = -10.0f;constexpr float bucket_high = 10.0f;constexpr float bucket_scale = nbuckets/(bucket_high - bucket_low);- constexpr float bucker_inter = -bucket_low * bucket_scale;+ constexpr float bucket_inter = -bucket_low * bucket_scale;std::vector<int> bucket_idx(candidates->size);std::vector<int> histo(nbuckets, 0);for (int i = 0; i < (int)candidates->size; ++i) {const float val = candidates->data[i].logit;- int ib = int(bucket_scale * val + bucker_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);+ int ib = int(bucket_scale * val + bucket_inter); //nbuckets * (val - bucket_low) / (bucket_high - bucket_low);ib = std::max(0, std::min(nbuckets-1, ib));bucket_idx[i] = ib;++histo[ib];diff --git a/llama.cpp/llama-vocab.cpp b/llama.cpp/llama-vocab.cppindex 288fd5bd..6c340e36 100644--- a/llama.cpp/llama-vocab.cpp+++ b/llama.cpp/llama-vocab.cpp@@ -18,20 +18,6 @@// helpers//-static void replace_all(std::string & s, const std::string & search, const std::string & replace) {- std::string result;- for (size_t pos = 0; ; pos += search.length()) {- auto new_pos = s.find(search, pos);- if (new_pos == std::string::npos) {- result += s.substr(pos, s.size() - pos);- break;- }- result += s.substr(pos, new_pos - pos) + replace;- pos = new_pos;- }- s = std::move(result);-}-LLAMA_ATTRIBUTE_FORMAT(1, 2)static std::string format(const char * fmt, ...) {va_list ap;@@ -404,6 +390,7 @@ struct llm_tokenizer_bpe {case LLAMA_VOCAB_PRE_TYPE_COMMAND_R:case LLAMA_VOCAB_PRE_TYPE_SMOLLM:case LLAMA_VOCAB_PRE_TYPE_CODESHELL:+ case LLAMA_VOCAB_PRE_TYPE_EXAONE:regex_exprs = {"\\p{N}","'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",@@ -426,6 +413,8 @@ struct llm_tokenizer_bpe {};break;case LLAMA_VOCAB_PRE_TYPE_PORO:+ case LLAMA_VOCAB_PRE_TYPE_BLOOM:+ case LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH:regex_exprs = {" ?[^(\\s|.,!?…。,、।۔،)]+",};@@ -818,6 +807,9 @@ struct llm_tokenizer_ugm {* the best tokenization.*/void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) {+ // get current size of output (for reversal later)+ size_t output_size = output.size();+// normalize the input firststd::string normalized;normalize(text, &normalized);@@ -897,7 +889,7 @@ struct llm_tokenizer_ugm {}// reverse the output since we added tokens starting from the end of the input- std::reverse(output.begin(), output.end());+ std::reverse(output.begin() + output_size, output.end());}private:@@ -1446,7 +1438,8 @@ llama_token_attr llama_token_get_attr_impl(const struct llama_vocab & vocab, llabool llama_token_is_eog_impl(const struct llama_vocab & vocab, llama_token token) {return token != -1 && (token == llama_token_eos_impl(vocab) ||- token == llama_token_eot_impl(vocab)+ token == llama_token_eot_impl(vocab) ||+ token == llama_token_eom_impl(vocab));}@@ -1478,11 +1471,11 @@ llama_token llama_token_pad_impl(const struct llama_vocab & vocab) {return vocab.special_pad_id;}-int32_t llama_add_bos_token_impl(const struct llama_vocab & vocab) {+bool llama_add_bos_token_impl(const struct llama_vocab & vocab) {return vocab.tokenizer_add_bos;}-int32_t llama_add_eos_token_impl(const struct llama_vocab & vocab) {+bool llama_add_eos_token_impl(const struct llama_vocab & vocab) {return vocab.tokenizer_add_eos;}@@ -1502,6 +1495,10 @@ llama_token llama_token_eot_impl(const struct llama_vocab & vocab) {return vocab.special_eot_id;}+llama_token llama_token_eom_impl(const struct llama_vocab & vocab) {+ return vocab.special_eom_id;+}+int32_t llama_tokenize_impl(const struct llama_vocab & vocab,const char * text,diff --git a/llama.cpp/llama-vocab.h b/llama.cpp/llama-vocab.hindex 4d0be82b..3741a871 100644--- a/llama.cpp/llama-vocab.h+++ b/llama.cpp/llama-vocab.h@@ -47,6 +47,7 @@ struct llama_vocab {id special_suffix_id = -1;id special_middle_id = -1;id special_eot_id = -1; // TODO: move above after "eos_id", and here add "file separator" token+ id special_eom_id = -1;// tokenizer flagsbool tokenizer_add_space_prefix = false;@@ -96,13 +97,14 @@ llama_token llama_token_sep_impl(const struct llama_vocab & vocab);llama_token llama_token_nl_impl (const struct llama_vocab & vocab);llama_token llama_token_pad_impl(const struct llama_vocab & vocab);-int32_t llama_add_bos_token_impl(const struct llama_vocab & vocab);-int32_t llama_add_eos_token_impl(const struct llama_vocab & vocab);+bool llama_add_bos_token_impl(const struct llama_vocab & vocab);+bool llama_add_eos_token_impl(const struct llama_vocab & vocab);llama_token llama_token_prefix_impl(const struct llama_vocab & vocab);llama_token llama_token_middle_impl(const struct llama_vocab & vocab);llama_token llama_token_suffix_impl(const struct llama_vocab & vocab);llama_token llama_token_eot_impl (const struct llama_vocab & vocab);+llama_token llama_token_eom_impl (const struct llama_vocab & vocab);int32_t llama_tokenize_impl(const struct llama_vocab & vocab,diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 249966ba..e93087ec 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -86,20 +86,6 @@ static std::string trim(const std::string & str) {return str.substr(start, end - start);}-static void replace_all(std::string & s, const std::string & search, const std::string & replace) {- std::string result;- for (size_t pos = 0; ; pos += search.length()) {- auto new_pos = s.find(search, pos);- if (new_pos == std::string::npos) {- result += s.substr(pos, s.size() - pos);- break;- }- result += s.substr(pos, new_pos - pos) + replace;- pos = new_pos;- }- s = std::move(result);-}-static bool is_float_close(float a, float b, float abs_tol) {// Check for non-negative toleranceif (abs_tol < 0.0) {@@ -187,7 +173,10 @@ enum llm_arch {LLM_ARCH_CHATGLM,LLM_ARCH_BITNET,LLM_ARCH_T5,+ LLM_ARCH_T5ENCODER,LLM_ARCH_JAIS,+ LLM_ARCH_NEMOTRON,+ LLM_ARCH_EXAONE,LLM_ARCH_UNKNOWN,};@@ -231,7 +220,10 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {{ LLM_ARCH_CHATGLM, "chatglm" },{ LLM_ARCH_BITNET, "bitnet" },{ LLM_ARCH_T5, "t5" },+ { LLM_ARCH_T5ENCODER, "t5encoder" },{ LLM_ARCH_JAIS, "jais" },+ { LLM_ARCH_NEMOTRON, "nemotron" },+ { LLM_ARCH_EXAONE, "exaone" },{ LLM_ARCH_UNKNOWN, "(unknown)" },};@@ -327,6 +319,7 @@ enum llm_kv {LLM_KV_TOKENIZER_SUFFIX_ID,LLM_KV_TOKENIZER_MIDDLE_ID,LLM_KV_TOKENIZER_EOT_ID,+ LLM_KV_TOKENIZER_EOM_ID,LLM_KV_ADAPTER_TYPE,LLM_KV_ADAPTER_LORA_ALPHA,@@ -424,6 +417,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {{ LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },{ LLM_KV_TOKENIZER_MIDDLE_ID, "tokenizer.ggml.middle_token_id" },{ LLM_KV_TOKENIZER_EOT_ID, "tokenizer.ggml.eot_token_id" },+ { LLM_KV_TOKENIZER_EOM_ID, "tokenizer.ggml.eom_token_id" },{ LLM_KV_ADAPTER_TYPE, "adapter.type" },{ LLM_KV_ADAPTER_LORA_ALPHA, "adapter.lora.alpha" },@@ -1238,6 +1232,24 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{ LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },},},+ {+ LLM_ARCH_T5ENCODER,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },+ { LLM_TENSOR_ENC_ATTN_NORM, "enc.blk.%d.attn_norm" },+ { LLM_TENSOR_ENC_ATTN_Q, "enc.blk.%d.attn_q" },+ { LLM_TENSOR_ENC_ATTN_K, "enc.blk.%d.attn_k" },+ { LLM_TENSOR_ENC_ATTN_V, "enc.blk.%d.attn_v" },+ { LLM_TENSOR_ENC_ATTN_OUT, "enc.blk.%d.attn_o" },+ { LLM_TENSOR_ENC_ATTN_REL_B, "enc.blk.%d.attn_rel_b" },+ { LLM_TENSOR_ENC_FFN_NORM, "enc.blk.%d.ffn_norm" },+ { LLM_TENSOR_ENC_FFN_GATE, "enc.blk.%d.ffn_gate" },+ { LLM_TENSOR_ENC_FFN_DOWN, "enc.blk.%d.ffn_down" },+ { LLM_TENSOR_ENC_FFN_UP, "enc.blk.%d.ffn_up" },+ },+ },{LLM_ARCH_JAIS,{@@ -1253,6 +1265,43 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },},},+ {+ LLM_ARCH_NEMOTRON,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },+ {+ LLM_ARCH_EXAONE,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,{@@ -3268,13 +3317,8 @@ namespace GGUFMeta {using llama_buf_map = std::unordered_map<uint32_t, ggml_backend_buffer_t>;-// TODO: update when needed or think of some clever automatic way to do this-static size_t llama_model_max_nodes(const llama_model & /*model*/) {- //if (model.arch == LLM_ARCH_LLAMA && model.hparams.n_layer > ??) { // llama-3 405B- // return 32768;- //}-- return 8192;+static size_t llama_model_max_nodes(const llama_model & model) {+ return std::max<size_t>(8192, model.tensors_by_name.size()*5);}struct llama_model_loader {@@ -3305,7 +3349,7 @@ struct llama_model_loader {const int tensor_idx = gguf_find_tensor(gguf_ctx, name);offs = gguf_get_data_offset(gguf_ctx) + gguf_get_tensor_offset(gguf_ctx, tensor_idx);- if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > llamafile_size(file->file)) {+ if (offs + ggml_nbytes(tensor) < offs || offs + ggml_nbytes(tensor) > llamafile_size(file->file)) { // [jart]throw std::runtime_error(format("tensor '%s' data is not within the file bounds, model is corrupted or incomplete", name));}}@@ -4597,7 +4641,6 @@ static void llm_load_hparams(} break;case LLM_ARCH_PHI3:{- ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer) {@@ -4606,6 +4649,22 @@ static void llm_load_hparams(case 40: model.type = e_model::MODEL_14B; break;default: model.type = e_model::MODEL_UNKNOWN;}++ // for backward compatibility ; see: https://github.com/ggerganov/llama.cpp/pull/8931+ if ((hparams.n_layer == 32 || hparams.n_layer == 40) && hparams.n_ctx_train == 4096) {+ // default value for Phi-3-mini-4k-instruct and Phi-3-medium-4k-instruct+ hparams.n_swa = 2047;+ } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) {+ // default value for Phi-3-mini-128k-instruct+ hparams.n_swa = 262144;+ } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) {+ // default value for Phi-3-medium-128k-instruct+ hparams.n_swa = 131072;+ }+ bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);+ if (!found_swa && hparams.n_swa == 0) {+ throw std::runtime_error("invalid value for sliding_window");+ }} break;case LLM_ARCH_PLAMO:{@@ -4673,6 +4732,7 @@ static void llm_load_hparams(hparams.attn_soft_cap = true;switch (hparams.n_layer) {+ case 26: model.type = e_model::MODEL_2B; break;case 42: model.type = e_model::MODEL_9B; break;case 46: model.type = e_model::MODEL_27B; break;default: model.type = e_model::MODEL_UNKNOWN;@@ -4902,6 +4962,12 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_T5ENCODER:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);+ model.type = e_model::MODEL_UNKNOWN;+ } break;case LLM_ARCH_JAIS:{ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);@@ -4914,6 +4980,23 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_NEMOTRON:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ switch (hparams.n_layer) {+ case 32: model.type = e_model::MODEL_4B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_EXAONE:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);++ switch (hparams.n_layer) {+ case 32: model.type = e_model::MODEL_8B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}@@ -5146,6 +5229,15 @@ static void llm_load_vocab(} else if (tokenizer_pre == "codeshell") {vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_CODESHELL;+ } else if (+ tokenizer_pre == "bloom") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_BLOOM;+ } else if (+ tokenizer_pre == "gpt3-finnish") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH;+ } else if (+ tokenizer_pre == "exaone") {+ vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_EXAONE;} else {throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));}@@ -5289,6 +5381,7 @@ static void llm_load_vocab({ LLM_KV_TOKENIZER_SUFFIX_ID, vocab.special_suffix_id },{ LLM_KV_TOKENIZER_MIDDLE_ID, vocab.special_middle_id },{ LLM_KV_TOKENIZER_EOT_ID, vocab.special_eot_id },+ { LLM_KV_TOKENIZER_EOM_ID, vocab.special_eom_id },};for (const auto & it : special_token_types) {@@ -5341,6 +5434,17 @@ static void llm_load_vocab(}}}++ // find EOM token: "<|eom_id|>"+ //+ // TODO: convert scripts should provide this token through the KV metadata LLAMA_KV_TOKENIZER_EOM_ID+ // for now, we apply this workaround to find the EOM token based on its text+ if (vocab.special_eom_id == -1) {+ const auto & t = vocab.token_to_id.find("<|eom_id|>");+ if (t != vocab.token_to_id.end()) {+ vocab.special_eom_id = t->second;+ }+ }}// build special tokens cache@@ -7127,6 +7231,42 @@ static bool llm_load_tensors(layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_DEC_FFN_UP, "weight", i), {n_embd, n_ff});}} break;+ case LLM_ARCH_T5ENCODER:+ {+ const auto n_rel_attn_bkts = hparams.n_rel_attn_bkts;++ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm_enc = ml.create_tensor(ctx_output, tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);+ // if output is NULL, init from the input tok embed+ if (model.output == NULL) {+ model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED);+ }+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_rel_b_enc = ml.create_tensor(ctx_input, tn(LLM_TENSOR_ENC_ATTN_REL_B, "weight", i), {n_head, n_rel_attn_bkts}, llama_model_loader::TENSOR_NOT_REQUIRED);++ layer.wq_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_Q, "weight", i), {n_embd, n_embd_k_gqa});+ layer.wk_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});+ layer.wv_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});+ layer.wo_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_ATTN_OUT, "weight", i), {n_embd_v_gqa, n_embd});++ layer.ffn_norm_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_gate_enc = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ENC_FFN_GATE, "weight", i), {n_embd, n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);+ layer.ffn_down_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up_enc = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ENC_FFN_UP, "weight", i), {n_embd, n_ff});+ }+ } break;case LLM_ARCH_JAIS:{model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});@@ -7196,6 +7336,78 @@ static bool llm_load_tensors(layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd});}} break;+ case LLM_ARCH_NEMOTRON:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output_norm_b = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});+ layer.attn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd});++ // optional bias tensors+ layer.bq = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);+ layer.bk = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);+ layer.bv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, llama_model_loader::TENSOR_NOT_REQUIRED);+ layer.bo = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.ffn_norm_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd});++ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});++ // optional MLP bias+ layer.ffn_down_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, llama_model_loader::TENSOR_NOT_REQUIRED);+ layer.ffn_up_b = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, llama_model_loader::TENSOR_NOT_REQUIRED);+ }+ } break;+ case LLM_ARCH_EXAONE:+ {+ model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});++ // output+ {+ model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});+ model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab});+ }++ for (int i = 0; i < n_layer; ++i) {+ ggml_context * ctx_layer = ctx_for_layer(i);+ ggml_context * ctx_split = ctx_for_layer_split(i);++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});++ layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head});+ layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa});+ layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa});+ layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});++ layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});+ layer.rope_freqs = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ROPE_FREQS, "weight"), {n_embd/n_head/2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));+ layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff});+ layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd});+ layer.ffn_up = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff});+ }+ } break;default:throw std::runtime_error("unknown architecture");}@@ -7882,7 +8094,7 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q);cb(kq, "kq", il);- if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2) {+ if (model.arch == LLM_ARCH_PHI2 || model.arch == LLM_ARCH_PHI3 || model.arch == LLM_ARCH_GPTNEOX || model.arch == LLM_ARCH_QWEN2 || model.arch == LLM_ARCH_NEMOTRON) {// for this arch, we need to perform the KQ multiplication with F32 precision, otherwise we get NaNs// ref: https://github.com/ggerganov/llama.cpp/pull/4490#issuecomment-1859055847ggml_mul_mat_set_prec(kq, GGML_PREC_F32);@@ -11443,6 +11655,7 @@ struct llm_build_context {// ref: https://github.com/google/gemma_pytorch/commit/03e657582d17cb5a8617ebf333c1c16f3694670eswitch (model.type) {+ case e_model::MODEL_2B:case e_model::MODEL_9B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd_head_k))); break;case e_model::MODEL_27B: Qcur = ggml_scale(ctx0, Qcur, 1.0f / sqrtf(float(n_embd / n_head))); break;default: GGML_ABORT("fatal error");@@ -12840,7 +13053,7 @@ struct llm_build_context {return gf;}- struct ggml_cgraph * build_t5() {+ struct ggml_cgraph * build_t5_encoder() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);// mutable variable, needed during the last layer of the computation to skip unused tokens@@ -12855,303 +13068,323 @@ struct llm_build_context {inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);- if (lctx.is_encoding) {- struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false);+ GGML_ASSERT(lctx.is_encoding);+ struct ggml_tensor * pos_bucket_enc = llm_build_pos_bucket(false);- // KQ_mask (mask for 1 head, it will be broadcasted to all heads)- struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false);-- for (int il = 0; il < n_layer; ++il) {- struct ggml_tensor * inpSA = inpL;+ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask_enc = build_inp_KQ_mask(false);- // norm- cur = llm_build_norm(ctx0, inpL, hparams,- model.layers[il].attn_norm_enc, NULL,- LLM_NORM_RMS, cb, il);- cb(cur, "attn_norm", il);+ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;- // self-attention- {- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq_enc, cur);- cb(Qcur, "Qcur", il);+ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm_enc, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk_enc, cur);- cb(Kcur, "Kcur", il);+ // self-attention+ {+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_enc, cur);+ cb(Qcur, "Qcur", il);- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv_enc, cur);- cb(Vcur, "Vcur", il);+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_enc, cur);+ cb(Kcur, "Kcur", il);- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_enc, cur);+ cb(Vcur, "Vcur", il);- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);- struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);- cb(kq, "kq", il);+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));- struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;- struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b);- struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);- cb(kq_b, "kq_b", il);+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);+ cb(kq, "kq", il);- kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias);- cb(kq, "kq_soft_max_ext", il);+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b_enc ? model.layers[il].attn_rel_b_enc : model.layers[0].attn_rel_b_enc;+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_enc, attn_rel_b);+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);+ cb(kq_b, "kq_b", il);- struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));- cb(v, "v", il);+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_enc, 1.0f, hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);- cb(kqv, "kqv", il);+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_tokens)));+ cb(v, "v", il);- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);- cb(kqv_merged, "kqv_merged", il);+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_tokens, n_embd_head, n_head_kv), kq);+ cb(kqv, "kqv", il);- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);- cb(cur, "kqv_merged_cont", il);+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);+ cb(kqv_merged, "kqv_merged", il);- ggml_build_forward_expand(gf, cur);+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);+ cb(cur, "kqv_merged_cont", il);- cur = ggml_mul_mat(ctx0, model.layers[il].wo_enc, cur);- cb(cur, "kqv_out", il);- }+ ggml_build_forward_expand(gf, cur);- if (il == n_layer - 1) {- // skip computing output for unused tokens- struct ggml_tensor * inp_out_ids = build_inp_out_ids();- n_tokens = n_outputs;- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);- }+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_enc, cur);+ cb(cur, "kqv_out", il);+ }- struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);- cb(ffn_inp, "ffn_inp", il);+ if (il == n_layer - 1) {+ // skip computing output for unused tokens+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();+ n_tokens = n_outputs;+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }- // feed-forward network- {- cur = llm_build_norm(ctx0, ffn_inp, hparams,- model.layers[il].ffn_norm_enc, NULL,- LLM_NORM_RMS, cb, il);- cb(cur, "ffn_norm", il);+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);- // T5 uses relu, flan-T5 uses gelu-gated- cur = llm_build_ffn(ctx0, lctx, cur,- model.layers[il].ffn_up_enc, NULL, NULL,- model.layers[il].ffn_gate_enc, NULL, NULL,- model.layers[il].ffn_down_enc, NULL, NULL,- NULL,- model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,- model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,- cb, il);- cb(cur, "ffn_out", il);- }+ // feed-forward network+ {+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm_enc, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);- cur = ggml_add(ctx0, cur, ffn_inp);+ // T5 uses relu, flan-T5 uses gelu-gated+ cur = llm_build_ffn(ctx0, lctx, cur,+ model.layers[il].ffn_up_enc, NULL, NULL,+ model.layers[il].ffn_gate_enc, NULL, NULL,+ model.layers[il].ffn_down_enc, NULL, NULL,+ NULL,+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,+ cb, il);cb(cur, "ffn_out", il);+ }- ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);- if (layer_dir != nullptr) {- cur = ggml_add(ctx0, cur, layer_dir);- }- cb(cur, "l_out", il);+ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);- // input for next layer- inpL = cur;+ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);+ if (layer_dir != nullptr) {+ cur = ggml_add(ctx0, cur, layer_dir);}+ cb(cur, "l_out", il);- cur = inpL;- cb(cur, "result_embd", -1);-- cur = llm_build_norm(ctx0, cur, hparams,- model.output_norm_enc, NULL,- LLM_NORM_RMS, cb, -1);- cb(cur, "result_norm", -1);- } else {- GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first");+ // input for next layer+ inpL = cur;+ }- struct ggml_tensor * embd_enc = llm_build_inp_embd_enc();- struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true);+ cur = inpL;+ cb(cur, "result_embd", -1);- struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask();- struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross();+ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm_enc, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);- for (int il = 0; il < n_layer; ++il) {- struct ggml_tensor * inpSA = inpL;+ ggml_build_forward_expand(gf, cur);- // norm- cur = llm_build_norm(ctx0, inpL, hparams,- model.layers[il].attn_norm, NULL,- LLM_NORM_RMS, cb, il);- cb(cur, "attn_norm", il);+ return gf;+ }- // self-attention- {- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);- cb(Qcur, "Qcur", il);+ struct ggml_cgraph * build_t5_decoder() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);- cb(Kcur, "Kcur", il);+ // mutable variable, needed during the last layer of the computation to skip unused tokens+ int32_t n_tokens = this->n_tokens;- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);- cb(Vcur, "Vcur", il);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);- llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il);+ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;- struct ggml_tensor * k =- ggml_view_3d(ctx0, kv_self.k_l[il],- n_embd_head_k, n_kv, n_head_kv,- ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),- ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),- 0);- cb(k, "k", il);+ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);- struct ggml_tensor * v =- ggml_view_3d(ctx0, kv_self.v_l[il],- n_kv, n_embd_head_v, n_head_kv,- ggml_element_size(kv_self.v_l[il])*n_ctx,- ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v,- 0);- cb(v, "v", il);+ GGML_ASSERT(!lctx.is_encoding);+ GGML_ASSERT(n_outputs_enc > 0 && "call llama_encode() first");- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ struct ggml_tensor * embd_enc = llm_build_inp_embd_enc();+ struct ggml_tensor * pos_bucket_dec = llm_build_pos_bucket(true);- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ struct ggml_tensor * KQ_mask_dec = build_inp_KQ_mask();+ struct ggml_tensor * KQ_mask_cross = llm_build_inp_KQ_mask_cross();- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);- cb(kq, "kq", il);+ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;- struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;- struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b);- struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);- cb(kq_b, "kq_b", il);+ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);- kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias);- cb(kq, "kq_soft_max_ext", il);+ // self-attention+ {+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);- cb(kqv, "kqv", il);+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);- cb(kqv_merged, "kqv_merged", il);+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);- cb(cur, "kqv_merged_cont", il);+ llm_build_kv_store(ctx0, hparams, cparams, kv_self, gf, Kcur, Vcur, n_tokens, kv_head, cb, il);- ggml_build_forward_expand(gf, cur);+ struct ggml_tensor * k =+ ggml_view_3d(ctx0, kv_self.k_l[il],+ n_embd_head_k, n_kv, n_head_kv,+ ggml_row_size(kv_self.k_l[il]->type, n_embd_k_gqa),+ ggml_row_size(kv_self.k_l[il]->type, n_embd_head_k),+ 0);+ cb(k, "k", il);- cur = ggml_mul_mat(ctx0, model.layers[il].wo, cur);- cb(cur, "kqv_out", il);- }+ struct ggml_tensor * v =+ ggml_view_3d(ctx0, kv_self.v_l[il],+ n_kv, n_embd_head_v, n_head_kv,+ ggml_element_size(kv_self.v_l[il])*n_ctx,+ ggml_element_size(kv_self.v_l[il])*n_ctx*n_embd_head_v,+ 0);+ cb(v, "v", il);- cur = ggml_add(ctx0, cur, inpSA);- cb(cur, "cross_inp", il);+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- struct ggml_tensor * inpCA = cur;+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);- // norm- cur = llm_build_norm(ctx0, cur, hparams,- model.layers[il].attn_norm_cross, NULL,- LLM_NORM_RMS, cb, il);- cb(cur, "attn_norm_cross", il);+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);+ cb(kq, "kq", il);- // cross-attention- {- struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq_cross, cur);- cb(Qcur, "Qcur", il);+ struct ggml_tensor * attn_rel_b = model.layers[il].attn_rel_b ? model.layers[il].attn_rel_b : model.layers[0].attn_rel_b;+ struct ggml_tensor * pos_bias = llm_build_pos_bias(pos_bucket_dec, attn_rel_b);+ struct ggml_tensor * kq_b = ggml_add(ctx0, kq, pos_bias);+ cb(kq_b, "kq_b", il);- struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk_cross, embd_enc);- cb(Kcur, "Kcur", il);+ kq = ggml_soft_max_ext(ctx0, kq_b, KQ_mask_dec, 1.0f, hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);- struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv_cross, embd_enc);- cb(Vcur, "Vcur", il);+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);+ cb(kqv, "kqv", il);- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);+ cb(kqv_merged, "kqv_merged", il);- struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);- struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));+ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);+ cb(cur, "kqv_merged_cont", il);- struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);- cb(kq, "kq", il);+ ggml_build_forward_expand(gf, cur);- kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);- cb(kq, "kq_soft_max_ext", il);+ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo, cur);+ cb(cur, "kqv_out", il);+ }- struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));- cb(v, "v", il);+ cur = ggml_add(ctx0, cur, inpSA);+ cb(cur, "cross_inp", il);- struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);- cb(kqv, "kqv", il);+ struct ggml_tensor * inpCA = cur;- struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);- cb(kqv_merged, "kqv_merged", il);+ // norm+ cur = llm_build_norm(ctx0, cur, hparams,+ model.layers[il].attn_norm_cross, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm_cross", il);- cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);- cb(cur, "kqv_merged_cont", il);+ // cross-attention+ {+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq_cross, cur);+ cb(Qcur, "Qcur", il);- ggml_build_forward_expand(gf, cur);+ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk_cross, embd_enc);+ cb(Kcur, "Kcur", il);- cur = ggml_mul_mat(ctx0, model.layers[il].wo_cross, cur);- cb(cur, "kqv_out", il);- }+ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv_cross, embd_enc);+ cb(Vcur, "Vcur", il);- if (il == n_layer - 1) {- // skip computing output for unused tokens- struct ggml_tensor * inp_out_ids = build_inp_out_ids();- n_tokens = n_outputs;- cur = ggml_get_rows(ctx0, cur, inp_out_ids);- inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);- inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);- }+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_outputs_enc);- struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);- cb(ffn_inp, "ffn_inp", il);+ struct ggml_tensor * q = ggml_permute(ctx0, Qcur, 0, 2, 1, 3);+ struct ggml_tensor * k = ggml_cont(ctx0, ggml_permute(ctx0, Kcur, 0, 2, 1, 3));- // feed-forward network- {- cur = llm_build_norm(ctx0, ffn_inp, hparams,- model.layers[il].ffn_norm, NULL,- LLM_NORM_RMS, cb, il);- cb(cur, "ffn_norm", il);+ struct ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);+ cb(kq, "kq", il);- // T5 uses relu, flan-T5 uses gelu-gated- cur = llm_build_ffn(ctx0, lctx, cur,- model.layers[il].ffn_up, NULL, NULL,- model.layers[il].ffn_gate, NULL, NULL,- model.layers[il].ffn_down, NULL, NULL,- NULL,- model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,- model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,- cb, il);- cb(cur, "ffn_out", il);- }+ kq = ggml_soft_max_ext(ctx0, kq, KQ_mask_cross, 1.0f, hparams.f_max_alibi_bias);+ cb(kq, "kq_soft_max_ext", il);- cur = ggml_add(ctx0, cur, ffn_inp);- cb(cur, "ffn_out", il);+ struct ggml_tensor * v = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, Vcur, n_embd_gqa, n_outputs_enc)));+ cb(v, "v", il);- ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);- if (layer_dir != nullptr) {- cur = ggml_add(ctx0, cur, layer_dir);- }- cb(cur, "l_out", il);+ struct ggml_tensor * kqv = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, v, n_outputs_enc, n_embd_head, n_head_kv), kq);+ cb(kqv, "kqv", il);- // input for next layer- inpL = cur;+ struct ggml_tensor * kqv_merged = ggml_permute(ctx0, kqv, 0, 2, 1, 3);+ cb(kqv_merged, "kqv_merged", il);++ cur = ggml_cont_2d(ctx0, kqv_merged, n_embd_gqa, n_tokens);+ cb(cur, "kqv_merged_cont", il);++ ggml_build_forward_expand(gf, cur);++ cur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wo_cross, cur);+ cb(cur, "kqv_out", il);}- cur = inpL;- cb(cur, "result_embd", -1);+ if (il == n_layer - 1) {+ // skip computing output for unused tokens+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();+ n_tokens = n_outputs;+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ inpCA = ggml_get_rows(ctx0, inpCA, inp_out_ids);+ }- cur = llm_build_norm(ctx0, cur, hparams,- model.output_norm, NULL,- LLM_NORM_RMS, cb, -1);- cb(cur, "result_norm", -1);+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpCA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ {+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ // T5 uses relu, flan-T5 uses gelu-gated+ cur = llm_build_ffn(ctx0, lctx, cur,+ model.layers[il].ffn_up, NULL, NULL,+ model.layers[il].ffn_gate, NULL, NULL,+ model.layers[il].ffn_down, NULL, NULL,+ NULL,+ model.layers[il].ffn_gate_enc ? LLM_FFN_GELU : LLM_FFN_RELU,+ model.layers[il].ffn_gate_enc ? LLM_FFN_PAR : LLM_FFN_SEQ,+ cb, il);+ cb(cur, "ffn_out", il);+ }++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);++ ggml_tensor * layer_dir = lctx.cvec.tensor_for(il);+ if (layer_dir != nullptr) {+ cur = ggml_add(ctx0, cur, layer_dir);+ }+ cb(cur, "l_out", il);- // lm_head- cur = ggml_mul_mat(ctx0, model.output, cur);- cb(cur, "result_output", -1);+ // input for next layer+ inpL = cur;}+ cur = inpL;+ cb(cur, "result_embd", -1);++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);+ cb(cur, "result_output", -1);+ggml_build_forward_expand(gf, cur);return gf;@@ -13362,6 +13595,254 @@ struct llm_build_context {return gf;}++ struct ggml_cgraph * build_nemotron() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ //GGML_ASSERT(n_embd_head == hparams.n_rot);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = build_inp_pos();++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm,+ model.layers[il].attn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }++ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, nullptr,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, nullptr,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, lctx, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ }++ if (il == n_layer - 1) {+ // skip computing output for unused tokens+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm,+ model.layers[il].ffn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, lctx, cur,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,+ NULL, NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,+ NULL,+ LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);++ cur = lctx.cvec.apply_to(ctx0, cur, il);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, model.output_norm_b,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_exaone() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);++ // mutable variable, needed during the last layer of the computation to skip unused tokens+ int32_t n_tokens = this->n_tokens;++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_head == hparams.n_rot);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = build_inp_pos();++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = build_inp_KQ_mask();++ for (int il = 0; il < n_layer; ++il) {+ struct ggml_tensor * inpSA = inpL;++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ // self-attention+ {+ // rope freq factors for llama3; may return nullptr for llama2 and other models+ struct ggml_tensor * rope_factors = build_rope_factors(il);++ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);+ if (model.layers[il].bq) {+ Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);+ cb(Qcur, "Qcur", il);+ }++ struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);+ if (model.layers[il].bk) {+ Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);+ cb(Kcur, "Kcur", il);+ }++ struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);+ if (model.layers[il].bv) {+ Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);+ cb(Vcur, "Vcur", il);+ }++ Qcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_ext(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,+ n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ cur = llm_build_kv(ctx0, lctx, kv_self, gf,+ model.layers[il].wo, model.layers[il].bo,+ Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ }++ if (il == n_layer - 1) {+ // skip computing output for unused tokens+ struct ggml_tensor * inp_out_ids = build_inp_out_ids();+ n_tokens = n_outputs;+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);+ }++ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);+ cb(ffn_inp, "ffn_inp", il);++ // feed-forward network+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, lctx, cur,+ model.layers[il].ffn_up, NULL, NULL,+ model.layers[il].ffn_gate, NULL, NULL,+ model.layers[il].ffn_down, NULL, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "ffn_out", il);++ cur = lctx.cvec.apply_to(ctx0, cur, il);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }};static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {@@ -13603,12 +14084,28 @@ static struct ggml_cgraph * llama_build_graph(} break;case LLM_ARCH_T5:{- result = llm.build_t5();+ if (lctx.is_encoding) {+ result = llm.build_t5_encoder();+ } else {+ result = llm.build_t5_decoder();+ }+ } break;+ case LLM_ARCH_T5ENCODER:+ {+ result = llm.build_t5_encoder();} break;case LLM_ARCH_JAIS:{result = llm.build_jais();} break;+ case LLM_ARCH_NEMOTRON:+ {+ result = llm.build_nemotron();+ } break;+ case LLM_ARCH_EXAONE:+ {+ result = llm.build_exaone();+ } break;default:GGML_ABORT("fatal error");}@@ -14051,7 +14548,7 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {// TODO: use a per-batch flag for logits presence insteadconst bool has_logits = !cparams.embeddings;- const bool has_embd = lctx.is_encoding || (cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE));+ const bool has_embd = cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;const size_t embd_size = has_embd ? n_embd*n_outputs_max : 0;@@ -14164,7 +14661,7 @@ struct llama_coder {static int llama_decode_internal(llama_context & lctx,llama_batch batch_all,- struct llama_coder *coder) { // TODO: rename back to batch+ struct llama_coder *coder) { // [jart]lctx.is_encoding = false;const uint32_t n_tokens_all = batch_all.n_tokens;@@ -14199,6 +14696,13 @@ static int llama_decode_internal(const auto n_ubatch = cparams.n_ubatch;+ // [jart] delete+ // // TODO: simplify or deprecate+ // std::vector<llama_pos> pos;+ // std::vector<int32_t> n_seq_id;+ // std::vector<llama_seq_id *> seq_id_arr;+ // std::vector<std::vector<llama_seq_id>> seq_id;+// this indicates we are doing pooled embedding, so we ignore batch.logits and output all tokensconst bool embd_pooled = cparams.embeddings && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE;@@ -14339,12 +14843,15 @@ static int llama_decode_internal(res = nullptr;embd = nullptr;} else if (cparams.embeddings) {- res = nullptr; // do not extract logits for embedding case- embd = gf->nodes[gf->n_nodes - 1];- if (strcmp(embd->name, "result_embd_pooled") != 0) {- embd = gf->nodes[gf->n_nodes - 2];+ res = nullptr; // do not extract logits for embedding case+ embd = nullptr;+ for (int i = gf->n_nodes - 1; i >= 0; --i) {+ if (strcmp(gf->nodes[i]->name, "result_embd_pooled") == 0) {+ embd = gf->nodes[i];+ break;+ }}- GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");+ GGML_ASSERT(embd != nullptr && "missing embeddings tensor");} else {embd = nullptr; // do not extract embeddings when not neededGGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");@@ -14470,7 +14977,7 @@ static int llama_decode_internal(static int llama_encode_internal(llama_context & lctx,llama_batch batch,- struct llama_coder *coder) {+ struct llama_coder *coder) { // [jart]lctx.is_encoding = true;@@ -14498,6 +15005,13 @@ static int llama_encode_internal(const int64_t n_embd = hparams.n_embd;+ // [jart] remove+ // // TODO: simplify or deprecate+ // std::vector<llama_pos> pos;+ // std::vector<int32_t> n_seq_id;+ // std::vector<llama_seq_id *> seq_id_arr;+ // std::vector<std::vector<llama_seq_id>> seq_id;+// reserve output bufferif (llama_output_reserve(lctx, n_tokens) < n_tokens) {LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);@@ -14546,9 +15060,24 @@ static int llama_encode_internal(ggml_cgraph * gf = llama_build_graph(lctx, batch, false);// the output embeddings after the final encoder normalization- struct ggml_tensor * embd = gf->nodes[gf->n_nodes - 1];+ struct ggml_tensor * embd = nullptr;- GGML_ASSERT(strcmp(embd->name, "result_norm") == 0);+ // there are two cases here+ if (llama_model_has_decoder(&lctx.model)) {+ // first case is an encoder-decoder T5 model where embeddings are passed to decoder+ embd = gf->nodes[gf->n_nodes - 1];+ GGML_ASSERT(strcmp(embd->name, "result_norm") == 0 && "missing result_output tensor");+ } else {+ // second case is an encoder-only T5 model+ if (cparams.embeddings) {+ // only output embeddings if required+ embd = gf->nodes[gf->n_nodes - 1];+ if (strcmp(embd->name, "result_embd_pooled") != 0) {+ embd = gf->nodes[gf->n_nodes - 2];+ }+ GGML_ASSERT(strcmp(embd->name, "result_embd_pooled") == 0 && "missing embeddings tensor");+ }+ }ggml_backend_sched_alloc_graph(lctx.sched, gf);@@ -14561,20 +15090,54 @@ static int llama_encode_internal(ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(lctx.sched, embd);GGML_ASSERT(backend_embd != nullptr);- // extract token embeddings- GGML_ASSERT(lctx.embd != nullptr);+ if (llama_model_has_decoder(&lctx.model)) {+ lctx.embd_enc.resize(n_tokens*n_embd);+ float * embd_out = lctx.embd_enc.data();- lctx.embd_enc.resize(n_tokens*n_embd);- float * embd_out = lctx.embd_enc.data();+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));- ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));+ // remember the sequence ids used during the encoding - needed for cross attention later+ lctx.seq_ids_enc.resize(n_tokens);+ for (uint32_t i = 0; i < n_tokens; i++) {+ for (int s = 0; s < batch.n_seq_id[i]; s++) {+ llama_seq_id seq_id = batch.seq_id[i][s];+ lctx.seq_ids_enc[i].insert(seq_id);+ }+ }+ } else {+ GGML_ASSERT(lctx.embd != nullptr);- // remember the sequence ids used during the encoding - needed for cross attention later- lctx.seq_ids_enc.resize(n_tokens);- for (uint32_t i = 0; i < n_tokens; i++) {- for (int s = 0; s < batch.n_seq_id[i]; s++) {- llama_seq_id seq_id = batch.seq_id[i][s];- lctx.seq_ids_enc[i].insert(seq_id);+ switch (cparams.pooling_type) {+ case LLAMA_POOLING_TYPE_NONE:+ {+ // extract token embeddings+ GGML_ASSERT(lctx.embd != nullptr);+ float * embd_out = lctx.embd;++ GGML_ASSERT(n_tokens*n_embd <= (int64_t) lctx.embd_size);+ ggml_backend_tensor_get_async(backend_embd, embd, embd_out, 0, n_tokens*n_embd*sizeof(float));+ } break;+ case LLAMA_POOLING_TYPE_MEAN:+ case LLAMA_POOLING_TYPE_CLS:+ case LLAMA_POOLING_TYPE_LAST:+ {+ // extract sequence embeddings+ auto & embd_seq_out = lctx.embd_seq;+ embd_seq_out.clear();++ for (uint32_t i = 0; i < n_tokens; i++) {+ const llama_seq_id seq_id = batch.seq_id[i][0];+ if (embd_seq_out.find(seq_id) != embd_seq_out.end()) {+ continue;+ }+ embd_seq_out[seq_id].resize(n_embd);+ ggml_backend_tensor_get_async(backend_embd, embd, embd_seq_out[seq_id].data(), (n_embd*seq_id)*sizeof(float), n_embd*sizeof(float));+ }+ } break;+ case LLAMA_POOLING_TYPE_UNSPECIFIED:+ {+ GGML_ABORT("unknown pooling type");+ }}}}@@ -15010,7 +15573,7 @@ static ggml_type llama_tensor_get_type(quantize_state_internal & qs, ggml_type nconst int n_expert = std::max(1, (int)qs.model.hparams.n_expert);auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {if (n_expert > 1) {- // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but iccasionally randomly+ // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly// sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work// for getting the current layer as I initially thought, and we need to resort to parsing the// tensor name.@@ -16043,7 +16606,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {}size_t llama_max_devices(void) {- if (llamafile_has_cuda())+ if (llamafile_has_cuda()) // [jart]return GGML_CUDA_MAX_DEVICES;return 1;#if defined(GGML_USE_RPC)@@ -16072,7 +16635,7 @@ bool llama_supports_mlock(void) {}bool llama_supports_gpu_offload(void) {- if (llamafile_has_gpu())+ if (llamafile_has_gpu()) // [jart]return true;#if defined(GGML_USE_CUDA) || defined(GGML_USE_METAL) || defined(GGML_USE_VULKAN) || \defined(GGML_USE_SYCL) || defined(GGML_USE_KOMPUTE) || defined(GGML_USE_RPC)@@ -16292,6 +16855,8 @@ struct llama_context * llama_new_context_with_model(ctx->sampling.rng = std::mt19937(params.seed);ctx->logits_all = params.logits_all;+ // build worst-case graph for encoder if a model contains encoder+ ctx->is_encoding = llama_model_has_encoder(model);uint32_t kv_size = cparams.n_ctx;ggml_type type_k = params.type_k;@@ -16609,6 +17174,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_MAMBA:case LLM_ARCH_JINA_BERT_V2:case LLM_ARCH_T5:+ case LLM_ARCH_T5ENCODER:case LLM_ARCH_JAIS:return LLAMA_ROPE_TYPE_NONE;@@ -16647,6 +17213,8 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {case LLM_ARCH_OPENELM:case LLM_ARCH_GPTNEOX:case LLM_ARCH_CODESHELL:+ case LLM_ARCH_NEMOTRON:+ case LLM_ARCH_EXAONE:return LLAMA_ROPE_TYPE_NEOX;// all model arches should be listed explicitly here@@ -16756,8 +17324,16 @@ struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const chbool llama_model_has_encoder(const struct llama_model * model) {switch (model->arch) {- case LLM_ARCH_T5: return true;- default: return false;+ case LLM_ARCH_T5: return true;+ case LLM_ARCH_T5ENCODER: return true;+ default: return false;+ }+}++bool llama_model_has_decoder(const struct llama_model * model) {+ switch (model->arch) {+ case LLM_ARCH_T5ENCODER: return false;+ default: return true;}}@@ -17803,7 +18379,7 @@ static bool llama_state_load_file_internal(struct llama_context * ctx, const cha// restore the context state{- const size_t n_state_size_cur = llamafile_size(file.file) - file.tell();+ const size_t n_state_size_cur = llamafile_size(file.file) - file.tell(); // [jart]llama_data_read_file data_ctx(&file);const size_t n_read = llama_state_set_data_internal(ctx, data_ctx);@@ -17940,7 +18516,7 @@ static size_t llama_state_seq_load_file_internal(struct llama_context * ctx, con// restore the context state{- const size_t state_size = llamafile_size(file.file) - file.tell();+ const size_t state_size = llamafile_size(file.file) - file.tell(); // [jart]llama_data_read_file data_ctx(&file);const size_t nread = llama_state_seq_set_data_internal(ctx, data_ctx, dest_seq_id);if (!nread) {@@ -18278,11 +18854,11 @@ llama_token llama_token_pad(const struct llama_model * model) {return llama_token_pad_impl(model->vocab);}-int32_t llama_add_bos_token(const struct llama_model * model) {+bool llama_add_bos_token(const struct llama_model * model) {return llama_add_bos_token_impl(model->vocab);}-int32_t llama_add_eos_token(const struct llama_model * model) {+bool llama_add_eos_token(const struct llama_model * model) {return llama_add_eos_token_impl(model->vocab);}@@ -18583,6 +19159,22 @@ static int32_t llama_chat_apply_template_internal(if (add_ass) {ss << "Assistant:";}+ } else if (tmpl == "exaone3" || (tmpl_contains("[|system|]") && tmpl_contains("[|assistant|]") && tmpl_contains("[|endofturn|]"))) {+ // ref: https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct/discussions/8#66bae61b1893d14ee8ed85bb+ // EXAONE-3.0-7.8B-Instruct+ for (auto message : chat) {+ std::string role(message->role);+ if (role == "system") {+ ss << "[|system|]" << trim(message->content) << "[|endofturn|]\n";+ } else if (role == "user") {+ ss << "[|user|]" << trim(message->content) << "\n";+ } else if (role == "assistant") {+ ss << "[|assistant|]" << trim(message->content) << "[|endofturn|]\n";+ }+ }+ if (add_ass) {+ ss << "[|assistant|]";+ }} else {// template not supportedreturn -1;diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 9ec723c1..3f919ae9 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -95,15 +95,15 @@ extern "C" {LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,+ LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,+ LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,+ LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,};- // note: these values should be synchronized with ggml_rope- // TODO: maybe move this enum to ggml.h (ggml_rope_type)enum llama_rope_type {LLAMA_ROPE_TYPE_NONE = -1,- LLAMA_ROPE_TYPE_NORM = 0,- LLAMA_ROPE_TYPE_NEOX = 2,- LLAMA_ROPE_TYPE_GLM = 4,+ LLAMA_ROPE_TYPE_NORM = 0,+ LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,};enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file@@ -348,7 +348,7 @@ extern "C" {int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()enum llama_ftype ftype; // quantize to this llama_ftypeenum ggml_type output_tensor_type; // output tensor type- enum ggml_type token_embedding_type; // itoken embeddings tensor type+ enum ggml_type token_embedding_type; // token embeddings tensor typebool allow_requantize; // allow quantizing non-f32/f16 tensorsbool quantize_output_tensor; // quantize output.weightbool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored@@ -507,6 +507,9 @@ extern "C" {// Returns true if the model contains an encoder that requires llama_encode() callLLAMA_API bool llama_model_has_encoder(const struct llama_model * model);+ // Returns true if the model contains a decoder that requires llama_decode() call+ LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);+// For encoder-decoder models, this function returns id of the token that must be provided// to the decoder to start generating output sequence. For other models, it returns -1.LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);@@ -915,11 +918,8 @@ extern "C" {LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-lineLLAMA_API llama_token llama_token_pad(const struct llama_model * model); // padding- // Returns -1 if unknown, 1 for true or 0 for false.- LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model);-- // Returns -1 if unknown, 1 for true or 0 for false.- LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model);+ LLAMA_API bool llama_add_bos_token(const struct llama_model * model);+ LLAMA_API bool llama_add_eos_token(const struct llama_model * model);// Codellama infill tokensLLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefixdiff --git a/llama.cpp/quantize/quantize.cpp b/llama.cpp/quantize/quantize.cppindex bc0139b5..c5787641 100644--- a/llama.cpp/quantize/quantize.cpp+++ b/llama.cpp/quantize/quantize.cpp@@ -95,7 +95,7 @@ static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftyp}// usage:-// ./quantize [--allow-requantize] [--leave-output-tensor] [--pure] models/llama/ggml-model.gguf [models/llama/ggml-model-quant.gguf] type [nthreads]+// ./llama-quantize [--allow-requantize] [--leave-output-tensor] [--pure] models/llama/ggml-model.gguf [models/llama/ggml-model-quant.gguf] type [nthreads]//[[noreturn]]static void usage(const char * executable) {diff --git a/stable-diffusion.cpp/README.llamafile b/stable-diffusion.cpp/README.llamafileindex 07adf8d0..0f4fbde5 100644--- a/stable-diffusion.cpp/README.llamafile+++ b/stable-diffusion.cpp/README.llamafile@@ -9,8 +9,8 @@ LICENSEORIGIN[email protected]:leejet/stable-diffusion.cpp.git- 1d2af5ca3fc30dec9657a6cca8f87c20dfd83f1c- Tue May 14 23:06:52 2024 +0800+ 697d000f49dd4c81405e96e2dca2677dd834c9c2+ 2024-08-10LOCAL MODIFICATIONSdiff --git a/stable-diffusion.cpp/ggml_extend.hpp b/stable-diffusion.cpp/ggml_extend.hppindex 16010fd2..41b59442 100644--- a/stable-diffusion.cpp/ggml_extend.hpp+++ b/stable-diffusion.cpp/ggml_extend.hpp@@ -542,7 +542,8 @@ __STATIC_INLINE__ void sd_tiling(ggml_tensor* input, ggml_tensor* output, const__STATIC_INLINE__ struct ggml_tensor* ggml_group_norm_32(struct ggml_context* ctx,struct ggml_tensor* a) {- return ggml_group_norm(ctx, a, 32);+ const float eps = 1e-6f; // default eps parameter+ return ggml_group_norm(ctx, a, 32, eps);}__STATIC_INLINE__ struct ggml_tensor* ggml_nn_linear(struct ggml_context* ctx,@@ -641,7 +642,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_attention(struct ggml_context* ctxstruct ggml_tensor* k,struct ggml_tensor* v,bool mask = false) {-#if defined(SD_USE_FLASH_ATTENTION) && !defined(SD_USE_CUBLAS) && !defined(SD_USE_METAL)+#if defined(SD_USE_FLASH_ATTENTION) && !defined(SD_USE_CUBLAS) && !defined(SD_USE_METAL) && !defined(SD_USE_SYCL)struct ggml_tensor* kqv = ggml_flash_attn(ctx, q, k, v, false); // [N * n_head, n_token, d_head]#elsefloat d_head = (float)q->ne[0];@@ -733,7 +734,8 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_group_norm(struct ggml_context* ctb = ggml_reshape_4d(ctx, b, 1, 1, b->ne[0], 1);}- x = ggml_group_norm(ctx, x, num_groups);+ const float eps = 1e-6f; // default eps parameter+ x = ggml_group_norm(ctx, x, num_groups, eps);if (w != NULL && b != NULL) {x = ggml_mul(ctx, x, w);// b = ggml_repeat(ctx, b, x);@@ -743,7 +745,7 @@ __STATIC_INLINE__ struct ggml_tensor* ggml_nn_group_norm(struct ggml_context* ct}__STATIC_INLINE__ void ggml_backend_tensor_get_and_sync(ggml_backend_t backend, const struct ggml_tensor* tensor, void* data, size_t offset, size_t size) {-#ifdef SD_USE_CUBLAS+#if defined (SD_USE_CUBLAS) || defined (SD_USE_SYCL)if (!ggml_backend_is_cpu(backend)) {ggml_backend_tensor_get_async(backend, tensor, data, offset, size);ggml_backend_synchronize(backend);diff --git a/stable-diffusion.cpp/stable-diffusion.cpp b/stable-diffusion.cpp/stable-diffusion.cppindex f8f08ff4..29dfd739 100644--- a/stable-diffusion.cpp/stable-diffusion.cpp+++ b/stable-diffusion.cpp/stable-diffusion.cpp@@ -152,13 +152,17 @@ public:ggml_backend_metal_log_set_callback(ggml_log_callback_default, nullptr);backend = ggml_backend_metal_init();#endif+#ifdef SD_USE_SYCL+ LOG_DEBUG("Using SYCL backend");+ backend = ggml_backend_sycl_init(0);+#endifif (!backend) {LOG_DEBUG("Using CPU backend");backend = ggml_backend_cpu_init();}#ifdef SD_USE_FLASH_ATTENTION-#if defined(SD_USE_CUBLAS) || defined(SD_USE_METAL)+#if defined(SD_USE_CUBLAS) || defined(SD_USE_METAL) || defined (SD_USE_SYCL)LOG_WARN("Flash Attention not supported with GPU Backend");#elseLOG_INFO("Flash Attention enabled");diff --git a/whisper.cpp/BUILD.mk b/whisper.cpp/BUILD.mkindex 7aa96f1c..218e9ce7 100644--- a/whisper.cpp/BUILD.mk+++ b/whisper.cpp/BUILD.mk@@ -43,4 +43,5 @@ $(WHISPER_CPP_OBJS): whisper.cpp/BUILD.mk.PHONY: o/$(MODE)/whisper.cppo/$(MODE)/whisper.cpp: \- o/$(MODE)/whisper.cpp/server+ o/$(MODE)/whisper.cpp/server \+ o/$(MODE)/whisper.cpp/main