Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Sync with llama.cpp upstream
Justine Tunney2e276a1aebf7
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex e10af896..2073272d 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -9,16 +9,16 @@ LICENSEORIGINhttps://github.com/ggerganov/llama.cpp/pull/4406/- e1241d9b461816b679eaf6951631287687a18f66- 2023-12-13+ b3a7c20b5c035250257d2b62851c379b159c899a+ 2024-01-04LOCAL MODIFICATIONS- Refactor ggml.c, llama.cpp, and llava to use llamafile_open() APIs- - Make cuBLAS dependency optional by introducing tinyBLAS library+ - Unify main, server, and llava-cli into single llamafile program+ - Make cuBLAS / hipBLAS optional by introducing tinyBLAS library- Add support to main() programs for Cosmo /zip/.args files- Introduce pledge() SECCOMP sandboxing to improve security- - Use condition variables for server rather than busy loop- Fix OpenAI server sampling w.r.t. temperature and seed- Remove log callback pointer API from Metal GPU module- Write log to /dev/null when main.log fails to open@@ -28,5 +28,4 @@ LOCAL MODIFICATIONS- Use runtime dispatching for matmul quants- Remove operating system #ifdef statements- Introduce --silent-prompt flag to main- - Meld llava-cli and llama.cpp main- Remove stdout logging from LLaVAdiff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex ad4d64f4..2c92125e 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -1351,6 +1351,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const lfprintf(stream, "build_number: %d\n", LLAMA_BUILD_NUMBER);fprintf(stream, "cpu_has_arm_fma: %s\n", ggml_cpu_has_arm_fma() ? "true" : "false");fprintf(stream, "cpu_has_avx: %s\n", ggml_cpu_has_avx() ? "true" : "false");+ fprintf(stream, "cpu_has_avx_vnni: %s\n", ggml_cpu_has_avx_vnni() ? "true" : "false");fprintf(stream, "cpu_has_avx2: %s\n", ggml_cpu_has_avx2() ? "true" : "false");fprintf(stream, "cpu_has_avx512: %s\n", ggml_cpu_has_avx512() ? "true" : "false");fprintf(stream, "cpu_has_avx512_vbmi: %s\n", ggml_cpu_has_avx512_vbmi() ? "true" : "false");diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex fd27d8cc..78dc1d9e 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -54,7 +54,7 @@ struct gpt_params {int32_t n_ctx = 512; // context sizeint32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS)int32_t n_keep = 0; // number of tokens to keep from initial prompt- int32_t n_draft = 16; // number of tokens to draft during speculative decoding+ int32_t n_draft = 8; // number of tokens to draft during speculative decodingint32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)int32_t n_parallel = 1; // number of parallel sequences to decodeint32_t n_sequences = 1; // number of sequences to decodediff --git a/llama.cpp/ggml-alloc.c b/llama.cpp/ggml-alloc.cindex 1c3d95c2..de4c982e 100644--- a/llama.cpp/ggml-alloc.c+++ b/llama.cpp/ggml-alloc.c@@ -74,7 +74,7 @@ static void remove_allocated_tensor(ggml_tallocr_t alloc, struct ggml_tensor * t// check if a tensor is allocated by this bufferstatic bool ggml_tallocr_is_own(ggml_tallocr_t alloc, const struct ggml_tensor * tensor) {- return tensor->buffer == alloc->buffer;+ return tensor->buffer == alloc->buffer && (!tensor->view_src || tensor->view_src->buffer == alloc->buffer);}static bool ggml_is_view(struct ggml_tensor * t) {@@ -451,11 +451,10 @@ static void init_view(ggml_gallocr_t galloc, struct ggml_tensor * view, bool updif (update_backend) {view->backend = view->view_src->backend;}- view->buffer = view->view_src->buffer;+ // views are initialized in the alloc buffer rather than the view_src buffer+ view->buffer = alloc->buffer;view->data = (char *)view->view_src->data + view->view_offs;- // FIXME: the view should be initialized by the owning buffer, but currently this breaks the CUDA backend- // due to the ggml_tensor_extra_gpu ring buffer overwriting the KV cache extrasassert(ggml_tallocr_is_measure(alloc) || !view->buffer || view->buffer->buft == alloc->buffer->buft);if (!alloc->measure) {@@ -738,6 +737,10 @@ void ggml_allocr_set_parse_seq(ggml_allocr_t alloc, const int * list, int n) {}void ggml_allocr_free(ggml_allocr_t alloc) {+ if (alloc == NULL) {+ return;+ }+ggml_gallocr_free(alloc->galloc);ggml_tallocr_free(alloc->talloc);free(alloc);@@ -777,7 +780,7 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conte}if (nbytes == 0) {- fprintf(stderr, "%s: no tensors to allocate\n", __func__);+ // all the tensors in the context are already allocatedreturn NULL;}@@ -791,6 +794,11 @@ ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_conte} else {ggml_backend_view_init(buffer, t);}+ } else {+ if (t->view_src != NULL) {+ // view of a pre-allocated tensor+ ggml_backend_view_init(buffer, t);+ }}}diff --git a/llama.cpp/ggml-alloc.h b/llama.cpp/ggml-alloc.hindex ad87cebc..64a41246 100644--- a/llama.cpp/ggml-alloc.h+++ b/llama.cpp/ggml-alloc.h@@ -43,7 +43,7 @@ GGML_API size_t ggml_allocr_alloc_graph(ggml_allocr_t alloc, struct ggml_cgraph// ggml-backend v2 API//-// Seperate tensor and graph allocator objects+// Separate tensor and graph allocator objects// This is necessary for multi-backend allocation because the graph allocator needs to use multiple tensor allocators// The original API is kept as a wrapper around the new APIdiff --git a/llama.cpp/ggml-backend-impl.h b/llama.cpp/ggml-backend-impl.hindex f588af60..05859935 100644--- a/llama.cpp/ggml-backend-impl.h+++ b/llama.cpp/ggml-backend-impl.h@@ -20,6 +20,9 @@ extern "C" {size_t (*get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignmentsize_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor); // data size needed to allocate the tensor, including paddingbool (*supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend+ // check if tensor data is in host memory+ // should be equivalent to supports_backend(buft, ggml_backend_cpu_init())+ bool (*is_host) (ggml_backend_buffer_type_t buft);};struct ggml_backend_buffer_type {@@ -31,15 +34,16 @@ extern "C" {typedef void * ggml_backend_buffer_context_t;struct ggml_backend_buffer_i {- void (*free_buffer)(ggml_backend_buffer_t buffer);+ void (*free_buffer) (ggml_backend_buffer_t buffer);//void (*reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras- void * (*get_base) (ggml_backend_buffer_t buffer);- void (*init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);- void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);- void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ void * (*get_base) (ggml_backend_buffer_t buffer);+ void (*init_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ void (*set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (*get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);// (optional) copy tensor between different buffer-type, allow for single-copy tranfers- void (*cpy_tensor_from)(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*cpy_tensor_to) (ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+ void (*cpy_tensor_from)(ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+ void (*cpy_tensor_to) (ggml_backend_buffer_t buffer, struct ggml_tensor * src, struct ggml_tensor * dst);+ void (*clear) (ggml_backend_buffer_t buffer, uint8_t value);};struct ggml_backend_buffer {@@ -78,7 +82,7 @@ extern "C" {void (*cpy_tensor_from_async)(ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);void (*cpy_tensor_to_async) (ggml_backend_t backend, struct ggml_tensor * src, struct ggml_tensor * dst);- void (*synchronize) (ggml_backend_t backend);+ void (*synchronize)(ggml_backend_t backend);// compute graph with a planggml_backend_graph_plan_t (*graph_plan_create) (ggml_backend_t backend, struct ggml_cgraph * cgraph);diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex a436589d..55ee1cc4 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -1,8 +1,8 @@// -*- mode:c;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-// vi: set et ft=c ts=4 sts=4 sw=4 fenc=utf-8 :vi+#include "ggml-backend-impl.h"#include "ggml-alloc.h"-#include "llama.cpp/ggml-cuda.h"#include "ggml-impl.h"#include <assert.h>@@ -38,6 +38,13 @@ bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_bareturn buft->iface.supports_backend(buft, backend);}+bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {+ if (buft->iface.is_host) {+ return buft->iface.is_host(buft);+ }+ return false;+}+// backend bufferggml_backend_buffer_t ggml_backend_buffer_init(@@ -97,6 +104,14 @@ size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct greturn ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type(buffer), tensor);}+void ggml_backend_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ buffer->iface.clear(buffer, value);+}++bool ggml_backend_buffer_is_host(ggml_backend_buffer_t buffer) {+ return ggml_backend_buft_is_host(ggml_backend_buffer_type(buffer));+}+ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer) {return buffer->buft;}@@ -258,17 +273,6 @@ static size_t ggml_backend_registry_count = 0;static ggml_backend_t ggml_backend_reg_cpu_init(const char * params, void * user_data);-int ggml_backend_cuda_reg_devices() {- int device_count = ggml_cuda_get_device_count();- //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization- for (int i = 0; i < device_count; i++) {- char name[128];- snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);- ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);- }- return device_count;-}-static void ggml_backend_registry_init(void) {static bool initialized = false;@@ -281,22 +285,22 @@ static void ggml_backend_registry_init(void) {ggml_backend_register("CPU", ggml_backend_reg_cpu_init, ggml_backend_cpu_buffer_type(), NULL);// add forward decls here to avoid including the backend headers-#ifdef GGML_USE_CUBLAS- extern void ggml_backend_cuda_reg_devices(void);+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+ extern int ggml_backend_cuda_reg_devices(void);ggml_backend_cuda_reg_devices();-#endif+ }-#ifdef GGML_USE_METAL+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {extern ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data);extern ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);ggml_backend_register("Metal", ggml_backend_reg_metal_init, ggml_backend_metal_buffer_type(), NULL);-#endif+ }}void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data) {GGML_ASSERT(ggml_backend_registry_count < GGML_MAX_BACKENDS_REG);- int id = ggml_backend_registry_count;+ size_t id = ggml_backend_registry_count;ggml_backend_registry[id] = (struct ggml_backend_reg) {/* .name = */ {0},@@ -329,6 +333,8 @@ size_t ggml_backend_reg_find_by_name(const char * name) {return i;}}++ // not foundreturn SIZE_MAX;}@@ -339,15 +345,15 @@ ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str)const char * params = strchr(backend_str, ':');char backend_name[128];if (params == NULL) {- strcpy(backend_name, backend_str);+ snprintf(backend_name, sizeof(backend_name), "%s", backend_str);params = "";} else {- strncpy(backend_name, backend_str, params - backend_str);- backend_name[params - backend_str] = '\0';+ snprintf(backend_name, sizeof(backend_name), "%.*s", (int)(params - backend_str), backend_str);params++;}size_t backend_i = ggml_backend_reg_find_by_name(backend_name);+if (backend_i == SIZE_MAX) {fprintf(stderr, "%s: backend %s not found\n", __func__, backend_name);return NULL;@@ -392,22 +398,15 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {free(buffer->context);- GGML_UNUSED(buffer);}static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");-memcpy((char *)tensor->data + offset, data, size);GGML_UNUSED(buffer);}static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");-memcpy(data, (const char *)tensor->data + offset, size);GGML_UNUSED(buffer);@@ -425,6 +424,10 @@ static void ggml_backend_cpu_buffer_cpy_tensor_to(ggml_backend_buffer_t buffer,GGML_UNUSED(buffer);}+static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ memset(buffer->context, value, buffer->size);+}+static struct ggml_backend_buffer_i cpu_backend_buffer_i = {/* .free_buffer = */ ggml_backend_cpu_buffer_free_buffer,/* .get_base = */ ggml_backend_cpu_buffer_get_base,@@ -433,6 +436,7 @@ static struct ggml_backend_buffer_i cpu_backend_buffer_i = {/* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,/* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,/* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+ /* .clear = */ ggml_backend_cpu_buffer_clear,};// for buffers from ptr, free is not called@@ -444,6 +448,7 @@ static struct ggml_backend_buffer_i cpu_backend_buffer_i_from_ptr = {/* .get_tensor = */ ggml_backend_cpu_buffer_get_tensor,/* .cpy_tensor_from = */ ggml_backend_cpu_buffer_cpy_tensor_from,/* .cpy_tensor_to = */ ggml_backend_cpu_buffer_cpy_tensor_to,+ /* .clear = */ ggml_backend_cpu_buffer_clear,};static const size_t TENSOR_ALIGNMENT = 64; // should be enough for AVX 512@@ -469,20 +474,70 @@ static bool ggml_backend_cpu_buffer_type_supports_backend(ggml_backend_buffer_tyGGML_UNUSED(buft);}+static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {+ return true;++ GGML_UNUSED(buft);+}+ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {- static struct ggml_backend_buffer_type ggml_backend_buffer_type_cpu = {+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {/* .iface = */ {/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes/* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,},/* .context = */ NULL,};- return &ggml_backend_buffer_type_cpu;+ return &ggml_backend_cpu_buffer_type;+}++#ifdef GGML_USE_CPU_HBM++// buffer type HBM++#include <hbwmalloc.h>++static void ggml_backend_cpu_hbm_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ hbw_free(buffer->context);}+static ggml_backend_buffer_t ggml_backend_cpu_hbm_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ //void * ptr = hbw_malloc(size);+ void * ptr;+ int result = hbw_posix_memalign(&ptr, ggml_backend_cpu_buffer_type_get_alignment(buft), size);+ if (result != 0) {+ fprintf(stderr, "failed to allocate HBM buffer of size %zu\n", size);+ return NULL;+ }++ // FIXME: this is a hack to avoid having to implement a new buffer type+ ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);+ buffer->buft = buft;+ buffer->iface.free_buffer = ggml_backend_cpu_hbm_buffer_free_buffer;++ return buffer;+}++ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type() {+ static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type_hbm = {+ /* .iface = */ {+ /* .alloc_buffer = */ ggml_backend_cpu_hbm_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,+ /* .get_alloc_size = */ NULL, // defaults to ggml_nbytes+ /* .supports_backend = */ ggml_backend_cpu_buffer_type_supports_backend,+ /* .is_host = */ ggml_backend_cpu_buffer_type_is_host,+ },+ /* .context = */ NULL,+ };++ return &ggml_backend_cpu_buffer_type_hbm;+}+#endif+struct ggml_backend_cpu_context {int n_threads;void * work_data;@@ -519,7 +574,7 @@ static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(ggml_backendstruct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu));cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads);- cpu_plan->cgraph = *cgraph;+ cpu_plan->cgraph = *cgraph; // FIXME: deep copyif (cpu_plan->cplan.work_size > 0) {cpu_plan->cplan.work_data = malloc(cpu_plan->cplan.work_size);@@ -562,10 +617,14 @@ static void ggml_backend_cpu_graph_compute(ggml_backend_t backend, struct ggml_c}static bool ggml_backend_cpu_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {- return true;+ switch (op->op) {+ case GGML_OP_MUL_MAT:+ return op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == ggml_internal_get_type_traits(op->src[0]->type).vec_dot_type;+ default:+ return true;+ }GGML_UNUSED(backend);- GGML_UNUSED(op);}static struct ggml_backend_i cpu_backend_i = {@@ -1194,7 +1253,7 @@ void ggml_backend_sched_set_node_backend(ggml_backend_sched_t sched, struct ggml// utilsvoid ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor) {GGML_ASSERT(tensor->buffer == NULL);- GGML_ASSERT(tensor->data == NULL);+ //GGML_ASSERT(tensor->data == NULL); // views of pre-allocted tensors may have the data set, but still need to be initializedGGML_ASSERT(tensor->view_src != NULL);GGML_ASSERT(tensor->view_src->buffer != NULL);GGML_ASSERT(tensor->view_src->data != NULL);diff --git a/llama.cpp/ggml-backend.h b/llama.cpp/ggml-backend.hindex 58d5ccae..a9d2fddd 100644--- a/llama.cpp/ggml-backend.h+++ b/llama.cpp/ggml-backend.h@@ -21,6 +21,7 @@ extern "C" {GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);+ GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);// bufferGGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);@@ -29,6 +30,8 @@ extern "C" {GGML_API void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);+ GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);+ GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_type(ggml_backend_buffer_t buffer);//@@ -76,6 +79,10 @@ extern "C" {GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);+#ifdef GGML_USE_CPU_HBM+ GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);+#endif+//// Backend registry//diff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex 987fbf03..ffd1ac36 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -21,25 +21,31 @@#include <hip/hip_runtime.h>#include <hipblas/hipblas.h>#include <hip/hip_fp16.h>+#ifdef __HIP_PLATFORM_AMD__+// for rocblas_initialize()+#include "rocblas/rocblas.h"+#endif // __HIP_PLATFORM_AMD__#define CUBLAS_COMPUTE_16F TINYBLAS_COMPUTE_16F#define CUBLAS_COMPUTE_32F TINYBLAS_COMPUTE_32F#define CUBLAS_GEMM_DEFAULT TINYBLAS_GEMM_DEFAULT_TENSOR_OP#define CUBLAS_GEMM_DEFAULT_TENSOR_OP TINYBLAS_GEMM_DEFAULT_TENSOR_OP#define CUBLAS_OP_N TINYBLAS_OP_N#define CUBLAS_OP_T TINYBLAS_OP_T-#define CUBLAS_STATUS_SUCCESS TINYBLAS_STATUS_SUCCESS-#define CUBLAS_STATUS_NOT_SUPPORTED TINYBLAS_STATUS_NOT_SUPPORTED#define CUBLAS_TF32_TENSOR_OP_MATH 0#define CUDA_R_16F HIPBLAS_R_16F#define CUDA_R_32F HIPBLAS_R_32F#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)#define cublasGemmAlgo_t tinyblasGemmAlgo_t#define cublasOperation_t tinyblasOperation_t-#define cublasComputeType_t tinyblasComputeType_t //deprecated, new hipblasComputeType_t not in 5.6+#define cublasComputeType_t tinyblasComputeType_t#define cublasCreate hipblasCreate+#define cublasGemmEx tinyblasGemmEx+#define cublasGemmBatchedEx tinyblasGemmBatchedEx+#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx#define cublasHandle_t tinyblasHandle_t#define cublasSetMathMode(handle, mode) CUBLAS_STATUS_SUCCESS#define cublasSetStream hipblasSetStream+#define cublasSgemm tinyblasSgemm#define cublasStatus_t tinyblasStatus_t#define cudaDataType_t hipblasDatatype_t //deprecated, new hipblasDatatype not in 5.6#define cudaDeviceCanAccessPeer hipDeviceCanAccessPeer@@ -58,15 +64,19 @@#define cudaGetDevice hipGetDevice#define cudaGetDeviceCount hipGetDeviceCount#define cudaGetDeviceProperties hipGetDeviceProperties-#define cudaDeviceGetAttribute hipDeviceGetAttribute-#define cudaDevAttrMultiProcessorCount hipDeviceAttributeMultiprocessorCount#define cudaGetErrorString hipGetErrorString#define cudaGetLastError hipGetLastError+#ifdef GGML_HIP_UMA+#define cudaMalloc hipMallocManaged+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size)+#else#define cudaMalloc hipMalloc#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)+#endif#define cudaMemcpy hipMemcpy-#define cudaMemcpy2DAsync hipMemcpy2DAsync#define cudaMemcpyAsync hipMemcpyAsync+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync+#define cudaMemcpy2DAsync hipMemcpy2DAsync#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost#define cudaMemcpyHostToDevice hipMemcpyHostToDevice@@ -82,15 +92,22 @@#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)#define cudaStream_t hipStream_t#define cudaSuccess hipSuccess+#define __trap abort+#define CUBLAS_STATUS_SUCCESS TINYBLAS_STATUS_SUCCESS+#define CUBLAS_STATUS_NOT_INITIALIZED TINYBLAS_STATUS_NOT_INITIALIZED+#define CUBLAS_STATUS_ALLOC_FAILED TINYBLAS_STATUS_ALLOC_FAILED+#define CUBLAS_STATUS_INVALID_VALUE TINYBLAS_STATUS_INVALID_VALUE+#define CUBLAS_STATUS_ARCH_MISMATCH TINYBLAS_STATUS_ARCH_MISMATCH+#define CUBLAS_STATUS_MAPPING_ERROR TINYBLAS_STATUS_MAPPING_ERROR+#define CUBLAS_STATUS_EXECUTION_FAILED TINYBLAS_STATUS_EXECUTION_FAILED+#define CUBLAS_STATUS_INTERNAL_ERROR TINYBLAS_STATUS_INTERNAL_ERROR+#define CUBLAS_STATUS_NOT_SUPPORTED TINYBLAS_STATUS_NOT_SUPPORTED#include "tinyblas.cu"-#define cublasSgemm tinyblasSgemm-#define cublasGemmEx tinyblasGemmEx-#define cublasGemmBatchedEx tinyblasGemmBatchedEx-#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx-#define cublasGetStatusString(x) "REDACTED!cublasGetStatusString"+#define cublasGetStatusString tinyblasGetStatusString#elif defined(GGML_USE_TINYBLAS)#include "tinyblas.cu"+#include <cuda.h>#define CUBLAS_COMPUTE_16F TINYBLAS_COMPUTE_16F#define CUBLAS_COMPUTE_32F TINYBLAS_COMPUTE_32F#define CUBLAS_OP_N TINYBLAS_OP_N@@ -98,17 +115,24 @@#define CUBLAS_GEMM_DEFAULT TINYBLAS_GEMM_DEFAULT_TENSOR_OP#define CUBLAS_GEMM_DEFAULT_TENSOR_OP TINYBLAS_GEMM_DEFAULT_TENSOR_OP#define CUBLAS_STATUS_SUCCESS TINYBLAS_STATUS_SUCCESS+#define CUBLAS_STATUS_NOT_INITIALIZED TINYBLAS_STATUS_NOT_INITIALIZED+#define CUBLAS_STATUS_ALLOC_FAILED TINYBLAS_STATUS_ALLOC_FAILED+#define CUBLAS_STATUS_INVALID_VALUE TINYBLAS_STATUS_INVALID_VALUE+#define CUBLAS_STATUS_ARCH_MISMATCH TINYBLAS_STATUS_ARCH_MISMATCH+#define CUBLAS_STATUS_MAPPING_ERROR TINYBLAS_STATUS_MAPPING_ERROR+#define CUBLAS_STATUS_EXECUTION_FAILED TINYBLAS_STATUS_EXECUTION_FAILED+#define CUBLAS_STATUS_INTERNAL_ERROR TINYBLAS_STATUS_INTERNAL_ERROR#define CUBLAS_STATUS_NOT_SUPPORTED TINYBLAS_STATUS_NOT_SUPPORTED#define cublasGemmAlgo_t tinyblasGemmAlgo_t#define cublasOperation_t tinyblasOperation_t-#define cublasComputeType_t tinyblasComputeType_t //deprecated, new hipblasComputeType_t not in 5.6+#define cublasComputeType_t tinyblasComputeType_t#define cublasHandle_t tinyblasHandle_t#define cublasStatus_t tinyblasStatus_t#define cublasSgemm tinyblasSgemm#define cublasGemmEx tinyblasGemmEx#define cublasGemmBatchedEx tinyblasGemmBatchedEx#define cublasGemmStridedBatchedEx tinyblasGemmStridedBatchedEx-#define cublasGetStatusString(x) "REDACTED!cublasGetStatusString"+#define cublasGetStatusString tinyblasGetStatusString#elif defined(GGML_USE_HIPBLAS)#include <hip/hip_runtime.h>@@ -120,11 +144,11 @@#endif // __HIP_PLATFORM_AMD__#define CUBLAS_COMPUTE_16F HIPBLAS_R_16F#define CUBLAS_COMPUTE_32F HIPBLAS_R_32F+#define CUBLAS_COMPUTE_32F_FAST_16F HIPBLAS_R_32F#define CUBLAS_GEMM_DEFAULT HIPBLAS_GEMM_DEFAULT#define CUBLAS_GEMM_DEFAULT_TENSOR_OP HIPBLAS_GEMM_DEFAULT#define CUBLAS_OP_N HIPBLAS_OP_N#define CUBLAS_OP_T HIPBLAS_OP_T-#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS#define CUBLAS_TF32_TENSOR_OP_MATH 0#define CUDA_R_16F HIPBLAS_R_16F#define CUDA_R_32F HIPBLAS_R_32F@@ -158,11 +182,17 @@#define cudaGetDeviceProperties hipGetDeviceProperties#define cudaGetErrorString hipGetErrorString#define cudaGetLastError hipGetLastError+#ifdef GGML_HIP_UMA+#define cudaMalloc hipMallocManaged+#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size)+#else#define cudaMalloc hipMalloc#define cudaMallocHost(ptr, size) hipHostMalloc(ptr, size, hipHostMallocDefault)+#endif#define cudaMemcpy hipMemcpy-#define cudaMemcpy2DAsync hipMemcpy2DAsync#define cudaMemcpyAsync hipMemcpyAsync+#define cudaMemcpyPeerAsync hipMemcpyPeerAsync+#define cudaMemcpy2DAsync hipMemcpy2DAsync#define cudaMemcpyDeviceToDevice hipMemcpyDeviceToDevice#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost#define cudaMemcpyHostToDevice hipMemcpyHostToDevice@@ -178,9 +208,20 @@#define cudaStreamWaitEvent(stream, event, flags) hipStreamWaitEvent(stream, event, flags)#define cudaStream_t hipStream_t#define cudaSuccess hipSuccess+#define __trap abort+#define CUBLAS_STATUS_SUCCESS HIPBLAS_STATUS_SUCCESS+#define CUBLAS_STATUS_NOT_INITIALIZED HIPBLAS_STATUS_NOT_INITIALIZED+#define CUBLAS_STATUS_ALLOC_FAILED HIPBLAS_STATUS_ALLOC_FAILED+#define CUBLAS_STATUS_INVALID_VALUE HIPBLAS_STATUS_INVALID_VALUE+#define CUBLAS_STATUS_ARCH_MISMATCH HIPBLAS_STATUS_ARCH_MISMATCH+#define CUBLAS_STATUS_MAPPING_ERROR HIPBLAS_STATUS_MAPPING_ERROR+#define CUBLAS_STATUS_EXECUTION_FAILED HIPBLAS_STATUS_EXECUTION_FAILED+#define CUBLAS_STATUS_INTERNAL_ERROR HIPBLAS_STATUS_INTERNAL_ERROR+#define CUBLAS_STATUS_NOT_SUPPORTED HIPBLAS_STATUS_NOT_SUPPORTED#else#include <cuda_runtime.h>+#include <cuda.h>#include <cublas_v2.h>#include <cuda_fp16.h>#endif // defined(GGML_USE_HIPBLAS)@@ -504,7 +545,9 @@ static size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft,#define MIN_CC_DP4A 610 // minimum compute capability for __dp4a, an intrinsic for byte-wise dot products#define CC_VOLTA 700#define CC_OFFSET_AMD 1000000+#define CC_RDNA1 (CC_OFFSET_AMD + 1010)#define CC_RDNA2 (CC_OFFSET_AMD + 1030)+#define CC_RDNA3 (CC_OFFSET_AMD + 1100)#define GGML_CUDA_MAX_NODES 8192@@ -518,7 +561,6 @@ static size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft,// TODO: improve this to be correct for more hardware// for example, currently fails for GeForce GTX 1660 which is TURING arch (> VOLTA) but does not have tensor cores-// probably other such cases, and not sure what happens on AMD hardware#if !defined(GGML_CUDA_FORCE_MMQ)#define CUDA_USE_TENSOR_CORES#endif@@ -549,7 +591,7 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) {const int8x4_t vb = reinterpret_cast<const int8x4_t&>(b);#if __has_builtin(__builtin_elementwise_sub_sat)const int8x4_t c = __builtin_elementwise_sub_sat(va, vb);- return reinterpret_cast<const int&>(c);+ return reinterpret_cast<const int &>(c);#elseint8x4_t c;int16_t tmp;@@ -560,7 +602,7 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) {if(tmp < std::numeric_limits<int8_t>::min()) tmp = std::numeric_limits<int8_t>::min();c[i] = tmp;}- return reinterpret_cast<int&>(c);+ return reinterpret_cast<int &>(c);#endif // __has_builtin(__builtin_elementwise_sub_sat)}@@ -598,45 +640,59 @@ static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) {static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");-#define CUDA_CHECK(err) \- do { \- cudaError_t err_ = (err); \- if (err_ != cudaSuccess) { \- int id; \- cudaGetDevice(&id); \- fprintf(stderr, "\nCUDA error %d at %s:%d: %s\n", err_, __FILE__, __LINE__, \- cudaGetErrorString(err_)); \- fprintf(stderr, "current device: %d\n", id); \- GGML_ASSERT(!"CUDA error"); \- } \- } while (0)+//[[noreturn]]+static void ggml_cuda_error(const char * stmt, const char * func, const char * file, const int line, const char * msg) {+ int id = -1; // in case cudaGetDevice fails+ cudaGetDevice(&id);-#if CUDART_VERSION >= 12000 && !defined(GGML_USE_TINYBLAS)-#define CUBLAS_CHECK(err) \- do { \- cublasStatus_t err_ = (err); \- if (err_ != CUBLAS_STATUS_SUCCESS) { \- int id; \- cudaGetDevice(&id); \- fprintf(stderr, "\ncuBLAS error %d at %s:%d: %s\n", \- err_, __FILE__, __LINE__, cublasGetStatusString(err_)); \- fprintf(stderr, "current device: %d\n", id); \- GGML_ASSERT(!"cuBLAS error"); \- } \+ fprintf(stderr, "CUDA error: %s\n", msg);+ fprintf(stderr, " current device: %d, in function %s at %s:%d\n", id, func, file, line);+ fprintf(stderr, " %s\n", stmt);+ // abort with GGML_ASSERT to get a stack trace+ GGML_ASSERT(!"CUDA error");+}++#define CUDA_CHECK_GEN(err, success, error_fn) \+ do { \+ auto err_ = (err); \+ if (err_ != (success)) { \+ ggml_cuda_error(#err, __func__, __FILE__, __LINE__, error_fn(err_)); \+ } \} while (0)++#define CUDA_CHECK(err) CUDA_CHECK_GEN(err, cudaSuccess, cudaGetErrorString)++#if CUDART_VERSION >= 12000+ static const char * cublas_get_error_str(const cublasStatus_t err) {+ return cublasGetStatusString(err);+ }#else-#define CUBLAS_CHECK(err) \- do { \- cublasStatus_t err_ = (err); \- if (err_ != CUBLAS_STATUS_SUCCESS) { \- int id; \- cudaGetDevice(&id); \- fprintf(stderr, "\ncuBLAS error %d at %s:%d\n", err_, __FILE__, __LINE__); \- fprintf(stderr, "current device: %d\n", id); \- GGML_ASSERT(!"cuBLAS error"); \- } \- } while (0)-#endif // CUDART_VERSION >= 11+ static const char * cublas_get_error_str(const cublasStatus_t err) {+ switch (err) {+ case CUBLAS_STATUS_SUCCESS: return "CUBLAS_STATUS_SUCCESS";+ case CUBLAS_STATUS_NOT_INITIALIZED: return "CUBLAS_STATUS_NOT_INITIALIZED";+ case CUBLAS_STATUS_ALLOC_FAILED: return "CUBLAS_STATUS_ALLOC_FAILED";+ case CUBLAS_STATUS_INVALID_VALUE: return "CUBLAS_STATUS_INVALID_VALUE";+ case CUBLAS_STATUS_ARCH_MISMATCH: return "CUBLAS_STATUS_ARCH_MISMATCH";+ case CUBLAS_STATUS_MAPPING_ERROR: return "CUBLAS_STATUS_MAPPING_ERROR";+ case CUBLAS_STATUS_EXECUTION_FAILED: return "CUBLAS_STATUS_EXECUTION_FAILED";+ case CUBLAS_STATUS_INTERNAL_ERROR: return "CUBLAS_STATUS_INTERNAL_ERROR";+ case CUBLAS_STATUS_NOT_SUPPORTED: return "CUBLAS_STATUS_NOT_SUPPORTED";+ default: return "unknown error";+ }+ }+#endif // CUDART_VERSION >= 12000++#define CUBLAS_CHECK(err) CUDA_CHECK_GEN(err, CUBLAS_STATUS_SUCCESS, cublas_get_error_str)++#if !defined(GGML_USE_HIPBLAS)+static const char * cu_get_error_str(CUresult err) {+ const char * err_str;+ cuGetErrorString(err, &err_str);+ return err_str;+}+#define CU_CHECK(err) CUDA_CHECK_GEN(err, CUDA_SUCCESS, cu_get_error_str)+#endif#if CUDART_VERSION >= 11100#define GGML_CUDA_ASSUME(x) __builtin_assume(x)@@ -692,10 +748,10 @@ typedef void (*ggml_cuda_func_t)(const ggml_tensor * src0, const ggml_tensor * stypedef void (*ggml_cuda_op_mul_mat_t)(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, const cudaStream_t & stream);+ const int64_t src1_padded_row_size, cudaStream_t stream);typedef void (*ggml_cuda_op_flatten_t)(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream);+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream);// QK = number of values after dequantization// QR = QK / number of values before dequantization@@ -901,22 +957,29 @@ struct ggml_tensor_extra_gpu {// this is faster on Windows// probably because the Windows CUDA libraries forget to make this check before invoking the drivers-inline cudaError_t ggml_cuda_set_device(const int device) {+static void ggml_cuda_set_device(const int device) {int current_device;CUDA_CHECK(cudaGetDevice(¤t_device));if (device == current_device) {- return cudaSuccess;+ return;}- return cudaSetDevice(device);+ CUDA_CHECK(cudaSetDevice(device));}static int g_device_count = -1;static int g_main_device = 0;-static int g_compute_capabilities[GGML_CUDA_MAX_DEVICES];static float g_tensor_split[GGML_CUDA_MAX_DEVICES] = {0};+struct cuda_device_capabilities {+ int cc; // compute capability+ bool vmm; // virtual memory support+ size_t vmm_granularity; // granularity of virtual memory+};++static cuda_device_capabilities g_device_caps[GGML_CUDA_MAX_DEVICES] = { {0, false, 0} };+static void * g_scratch_buffer = nullptr;static size_t g_scratch_size = 0; // disabled by defaultstatic size_t g_scratch_offset = 0;@@ -925,6 +988,14 @@ static size_t g_scratch_offset = 0;static cublasHandle_t g_cublas_handles[GGML_CUDA_MAX_DEVICES] = {nullptr};#endif+//[[noreturn]]+static __device__ void bad_arch() {+ printf("ERROR: ggml-cuda was compiled without support for the current GPU architecture.\n");+ __trap();++ (void) bad_arch; // suppress unused function warning+}+static __device__ __forceinline__ float warp_reduce_sum(float x) {#pragma unrollfor (int mask = 16; mask > 0; mask >>= 1) {@@ -952,6 +1023,7 @@ static __device__ __forceinline__ float warp_reduce_max(float x) {static __device__ __forceinline__ float op_repeat(const float a, const float b) {return b;+ GGML_UNUSED(a);}static __device__ __forceinline__ float op_add(const float a, const float b) {@@ -1073,7 +1145,7 @@ static __global__ void silu_f32(const float * x, float * dst, const int k) {dst[i] = x[i] / (1.0f + expf(-x[i]));}-static __global__ void gelu_quick_f32(const float *x, float *dst, int k) {+static __global__ void gelu_quick_f32(const float * x, float * dst, int k) {const float GELU_QUICK_COEF = -1.702f;const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= k) {@@ -1082,7 +1154,7 @@ static __global__ void gelu_quick_f32(const float *x, float *dst, int k) {dst[i] = x[i] * (1.0f / (1.0f + expf(GELU_QUICK_COEF * x[i])));}-static __global__ void tanh_f32(const float *x, float *dst, int k) {+static __global__ void tanh_f32(const float * x, float * dst, int k) {const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= k) {return;@@ -1099,7 +1171,7 @@ static __global__ void relu_f32(const float * x, float * dst, const int k) {dst[i] = fmaxf(x[i], 0);}-static __global__ void leaky_relu_f32(const float *x, float *dst, const int k, const float negative_slope) {+static __global__ void leaky_relu_f32(const float * x, float * dst, const int k, const float negative_slope) {const int i = blockDim.x*blockIdx.x + threadIdx.x;if (i >= k) {return;@@ -1152,7 +1224,7 @@ static __global__ void norm_f32(const float * x, float * dst, const int ncols, c}}-static __global__ void concat_f32(const float *x,const float *y, float *dst, const int ne0, const int ne02) {+static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {int nidx = threadIdx.x + blockIdx.x * blockDim.x;if (nidx >= ne0) {return;@@ -1177,7 +1249,7 @@ static __global__ void concat_f32(const float *x,const float *y, float *dst, c}}-static __global__ void upscale_f32(const float *x, float *dst, const int ne00, const int nb02, const int scale_factor) {+static __global__ void upscale_f32(const float * x, float * dst, const int ne00, const int nb02, const int scale_factor) {int ne0 = ne00 * scale_factor;int nidx = threadIdx.x + blockIdx.x * blockDim.x;if (nidx >= ne0) {@@ -1197,7 +1269,7 @@ static __global__ void upscale_f32(const float *x, float *dst, const int ne00,dst[offset_dst] = x[offset_src];}-static __global__ void pad_f32(const float *x, float *dst, const int ne0, const int ne00, const int ne01, const int ne02) {+static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02) {int nidx = threadIdx.x + blockIdx.x * blockDim.x;if (nidx >= ne0) {return;@@ -2385,8 +2457,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_0_q8_1_imp// second part effectively subtracts 8 from each quant valuereturn d4 * (sumi * ds8f.x - (8*vdr/QI4_0) * ds8f.y);#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2423,8 +2494,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q4_1_q8_1_imp// scale second part of sum by QI8_1/(vdr * QR4_1) to compensate for multiple threads adding itreturn sumi * d4d8 + m4s8 / (QI8_1 / (vdr * QR4_1));#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2459,8 +2529,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_0_q8_1_imp// second part effectively subtracts 16 from each quant valuereturn d5 * (sumi * ds8f.x - (16*vdr/QI5_0) * ds8f.y);#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2505,8 +2574,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q5_1_q8_1_impreturn sumi*d5d8 + m5s8 / (QI5_1 / vdr);#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2527,8 +2595,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_0_q8_1_impreturn d8_0*d8_1 * sumi;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2558,8 +2625,7 @@ template <int vdr> static __device__ __forceinline__ float vec_dot_q8_1_q8_1_imp// scale second part of sum by QI8_1/ vdr to compensate for multiple threads adding itreturn sumi*d8d8 + m8s8 / (QI8_1 / vdr);#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2594,8 +2660,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmvq(return dm2f.x*sumf_d - dm2f.y*sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2632,8 +2697,7 @@ static __device__ __forceinline__ float vec_dot_q2_K_q8_1_impl_mmq(return d8 * (dm2f.x*sumi_d - dm2f.y*sumi_m);#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2673,8 +2737,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmvq(return d3 * sumf;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2699,8 +2762,7 @@ static __device__ __forceinline__ float vec_dot_q3_K_q8_1_impl_mmq(return d3*d8 * sumi;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2733,8 +2795,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_vmmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2767,8 +2828,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1_impl_mmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2808,8 +2868,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_vmmq(return dm5f.x*sumf_d - dm5f.y*sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2842,8 +2901,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1_impl_mmq(return dm4f.x*sumf_d - dm4f.y*sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2873,8 +2931,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmvq(return d*sumf;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -2905,8 +2962,7 @@ static __device__ __forceinline__ float vec_dot_q6_K_q8_1_impl_mmq(return d6 * sumf_d;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A}@@ -3772,8 +3828,7 @@ static __device__ __forceinline__ float vec_dot_q4_K_q8_1(return dall * sumf_d - dmin * sumf_m;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A#endif@@ -3956,8 +4011,7 @@ static __device__ __forceinline__ float vec_dot_q5_K_q8_1(return d * sumf_d;#else- assert(false);- return 0.0f; // only to satisfy the compiler+ bad_arch();#endif // __CUDA_ARCH__ >= MIN_CC_DP4A#endif@@ -4367,7 +4421,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_0_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4436,7 +4490,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_1_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4503,7 +4557,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_0_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4570,7 +4624,7 @@ mul_mat_q5_1((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_1_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4637,7 +4691,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q8_0_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4704,7 +4758,7 @@ mul_mat_q2_K((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q2_K_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4773,7 +4827,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q3_K_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4842,7 +4896,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q4_K_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4909,7 +4963,7 @@ mul_mat_q5_K((vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q5_K_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -4978,7 +5032,7 @@ template <bool need_check> static __global__ void(vx, vy, dst, ncols_x, nrows_x, ncols_y, nrows_y, nrows_dst);#else(void) vec_dot_q6_K_q8_1_mul_mat;- assert(false);+ bad_arch();#endif // __CUDA_ARCH__ >= CC_VOLTA}@@ -5117,7 +5171,6 @@ static __global__ void mul_mat_p021_f16_f32(const int row_y = col_x;-// y is not transposed but permutedconst int iy = channel*nrows_y + row_y;@@ -5683,17 +5736,17 @@ static __global__ void im2col_f32_f16(const int ky = (i - kd) / OW;const int ix = i % OW;- const int iiw = ix * s0 + kx * d0 - p0;- const int iih = blockIdx.y * s1 + ky * d1 - p1;+ const int64_t iiw = ix * s0 + kx * d0 - p0;+ const int64_t iih = blockIdx.y * s1 + ky * d1 - p1;- const int offset_dst =+ const int64_t offset_dst =(blockIdx.y * OW + ix) * CHW +(blockIdx.z * (KW * KH) + ky * KW + kx);if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {dst[offset_dst] = __float2half(0.0f);} else {- const int offset_src = blockIdx.z * offset_delta;+ const int64_t offset_src = blockIdx.z * offset_delta;dst[offset_dst] = __float2half(x[offset_src + iih * IW + iiw]);}}@@ -5792,7 +5845,7 @@ struct bin_bcast_cuda {cne[3] = 1;};- auto collapse_nb = [](size_t cnb[], int64_t cne[]) {+ auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {cnb[1] *= cne[1];cnb[2] *= cne[2];cnb[3] *= cne[3];@@ -6285,7 +6338,7 @@ static void ggml_mul_mat_q4_0_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6330,7 +6383,7 @@ static void ggml_mul_mat_q4_1_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6375,7 +6428,7 @@ static void ggml_mul_mat_q5_0_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6420,7 +6473,7 @@ static void ggml_mul_mat_q5_1_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6465,7 +6518,7 @@ static void ggml_mul_mat_q8_0_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6510,7 +6563,7 @@ static void ggml_mul_mat_q2_K_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6557,7 +6610,7 @@ static void ggml_mul_mat_q3_K_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6603,7 +6656,7 @@ static void ggml_mul_mat_q4_K_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6648,7 +6701,7 @@ static void ggml_mul_mat_q5_K_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6693,7 +6746,7 @@ static void ggml_mul_mat_q6_K_q8_1_cuda(int id;CUDA_CHECK(cudaGetDevice(&id));- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;int mmq_x, mmq_y, nwarps;if (compute_capability >= CC_RDNA2) {@@ -6953,30 +7006,30 @@ struct scoped_spin_lock {scoped_spin_lock& operator=(const scoped_spin_lock&) = delete;};-struct cuda_buffer {+static std::atomic_flag g_cuda_pool_lock = ATOMIC_FLAG_INIT;++// #define DEBUG_CUDA_MALLOC+struct ggml_cuda_buffer {void * ptr = nullptr;size_t size = 0;};-static cuda_buffer g_cuda_buffer_pool[GGML_CUDA_MAX_DEVICES][MAX_CUDA_BUFFERS];-static std::atomic_flag g_cuda_pool_lock = ATOMIC_FLAG_INIT;+static ggml_cuda_buffer g_cuda_buffer_pool[GGML_CUDA_MAX_DEVICES][MAX_CUDA_BUFFERS];+static size_t g_cuda_pool_size[GGML_CUDA_MAX_DEVICES] = {0};-static void * ggml_cuda_pool_malloc(size_t size, size_t * actual_size) {+static void * ggml_cuda_pool_malloc_leg(int device, size_t size, size_t * actual_size) {scoped_spin_lock lock(g_cuda_pool_lock);- int id;- CUDA_CHECK(cudaGetDevice(&id));#ifdef DEBUG_CUDA_MALLOCint nnz = 0;- size_t max_size = 0, tot_size = 0;+ size_t max_size = 0;#endifsize_t best_diff = 1ull << 36;int ibest = -1;for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) {- cuda_buffer& b = g_cuda_buffer_pool[id][i];+ ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i];if (b.ptr != nullptr) {#ifdef DEBUG_CUDA_MALLOC++nnz;- tot_size += b.size;if (b.size > max_size) max_size = b.size;#endifif (b.size >= size) {@@ -6996,32 +7049,32 @@ static void * ggml_cuda_pool_malloc(size_t size, size_t * actual_size) {}}if (ibest >= 0) {- cuda_buffer& b = g_cuda_buffer_pool[id][ibest];+ ggml_cuda_buffer& b = g_cuda_buffer_pool[device][ibest];void * ptr = b.ptr;*actual_size = b.size;b.ptr = nullptr;b.size = 0;return ptr;}-#ifdef DEBUG_CUDA_MALLOC- fprintf(stderr, "%s: %d buffers, max_size = %u MB, tot_size = %u MB, requested %u MB\n", __func__, nnz,- (uint32_t)(max_size/1024/1024), (uint32_t)(tot_size/1024/1024), (uint32_t)(size/1024/1024));-#endifvoid * ptr;size_t look_ahead_size = (size_t) (1.05 * size);look_ahead_size = 256 * ((look_ahead_size + 255)/256);+ ggml_cuda_set_device(device);CUDA_CHECK(cudaMalloc((void **) &ptr, look_ahead_size));*actual_size = look_ahead_size;+ g_cuda_pool_size[device] += look_ahead_size;+#ifdef DEBUG_CUDA_MALLOC+ fprintf(stderr, "%s[%d]: %d buffers, max_size = %u MB, pool_size = %u MB, requested %u MB\n", __func__, id, nnz,+ (uint32_t)(max_size/1024/1024), (uint32_t)(g_cuda_pool_size[id]/1024/1024), (uint32_t)(size/1024/1024));+#endifreturn ptr;}-static void ggml_cuda_pool_free(void * ptr, size_t size) {+static void ggml_cuda_pool_free_leg(int device, void * ptr, size_t size) {scoped_spin_lock lock(g_cuda_pool_lock);- int id;- CUDA_CHECK(cudaGetDevice(&id));for (int i = 0; i < MAX_CUDA_BUFFERS; ++i) {- cuda_buffer& b = g_cuda_buffer_pool[id][i];+ ggml_cuda_buffer& b = g_cuda_buffer_pool[device][i];if (b.ptr == nullptr) {b.ptr = ptr;b.size = size;@@ -7029,9 +7082,149 @@ static void ggml_cuda_pool_free(void * ptr, size_t size) {}}fprintf(stderr, "WARNING: cuda buffer pool full, increase MAX_CUDA_BUFFERS\n");+ ggml_cuda_set_device(device);CUDA_CHECK(cudaFree(ptr));+ g_cuda_pool_size[device] -= size;}+#if !defined(GGML_USE_HIPBLAS)+// pool with virtual memory+static CUdeviceptr g_cuda_pool_addr[GGML_CUDA_MAX_DEVICES] = {0};+static size_t g_cuda_pool_used[GGML_CUDA_MAX_DEVICES] = {0};+static const size_t CUDA_POOL_VMM_MAX_SIZE = 1ull << 35; // 32 GB++static void * ggml_cuda_pool_malloc_vmm(int device, size_t size, size_t * actual_size) {+ scoped_spin_lock lock(g_cuda_pool_lock);++ // round up the allocation size to the alignment to ensure that all allocations are aligned for all data types+ const size_t alignment = 128;+ size = alignment * ((size + alignment - 1) / alignment);++ size_t avail = g_cuda_pool_size[device] - g_cuda_pool_used[device];++ if (size > avail) {+ // round up to the next multiple of the granularity+ size_t reserve_size = size - avail;+ const size_t granularity = g_device_caps[device].vmm_granularity;+ reserve_size = granularity * ((reserve_size + granularity - 1) / granularity);++ GGML_ASSERT(g_cuda_pool_size[device] + reserve_size <= CUDA_POOL_VMM_MAX_SIZE);++ // allocate more physical memory+ CUmemAllocationProp prop = {};+ prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;+ prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ prop.location.id = device;+ CUmemGenericAllocationHandle handle;+ CU_CHECK(cuMemCreate(&handle, reserve_size, &prop, 0));++ // reserve virtual address space (if not already reserved)+ if (g_cuda_pool_addr[device] == 0) {+ CU_CHECK(cuMemAddressReserve(&g_cuda_pool_addr[device], CUDA_POOL_VMM_MAX_SIZE, 0, 0, 0));+ }++ // map at the end of the pool+ CU_CHECK(cuMemMap(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, 0, handle, 0));++ // the memory allocation handle is no longer needed after mapping+ CU_CHECK(cuMemRelease(handle));++ // set access+ CUmemAccessDesc access = {};+ access.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ access.location.id = device;+ access.flags = CU_MEM_ACCESS_FLAGS_PROT_READWRITE;+ CU_CHECK(cuMemSetAccess(g_cuda_pool_addr[device] + g_cuda_pool_size[device], reserve_size, &access, 1));++ // add to the pool+ g_cuda_pool_size[device] += reserve_size;++ //printf("cuda pool[%d]: size increased to %llu MB (reserved %llu MB)\n",+ // id, (unsigned long long) (g_cuda_pool_size[id]/1024/1024),+ // (unsigned long long) (reserve_size/1024/1024));+ }++ GGML_ASSERT(g_cuda_pool_addr[device] != 0);++ void * ptr = (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device]);+ *actual_size = size;+ g_cuda_pool_used[device] += size;++#ifdef DEBUG_CUDA_MALLOC+ printf("cuda pool[%d]: allocated %llu bytes at %llx [%s]\n", id, (unsigned long long) size, ptr);+#endif++ return ptr;+}++static void ggml_cuda_pool_free_vmm(int device, void * ptr, size_t size) {+ scoped_spin_lock lock(g_cuda_pool_lock);++#ifdef DEBUG_CUDA_MALLOC+ printf("cuda pool[%d]: freed %llu bytes at %llx\n", id, (unsigned long long) size, ptr);+#endif++ g_cuda_pool_used[device] -= size;++ // all deallocations must be in reverse order of the allocations+ GGML_ASSERT(ptr == (void *) (g_cuda_pool_addr[device] + g_cuda_pool_used[device]));+}++static void * ggml_cuda_pool_malloc(int device, size_t size, size_t * actual_size) {+ if (g_device_caps[device].vmm) {+ return ggml_cuda_pool_malloc_vmm(device, size, actual_size);+ } else {+ return ggml_cuda_pool_malloc_leg(device, size, actual_size);+ }+}++static void ggml_cuda_pool_free(int device, void * ptr, size_t size) {+ if (g_device_caps[device].vmm) {+ ggml_cuda_pool_free_vmm(device, ptr, size);+ } else {+ ggml_cuda_pool_free_leg(device, ptr, size);+ }+}+#else+#define ggml_cuda_pool_malloc ggml_cuda_pool_malloc_leg+#define ggml_cuda_pool_free ggml_cuda_pool_free_leg+#endif // !defined(GGML_USE_HIPBLAS)++template<typename T>+struct cuda_pool_alloc {+ int device = -1;+ T * ptr = nullptr;+ size_t actual_size = 0;++ // size is in number of elements+ T * alloc(size_t size) {+ GGML_ASSERT(ptr == nullptr);+ CUDA_CHECK(cudaGetDevice(&device));+ ptr = (T *) ggml_cuda_pool_malloc(device, size * sizeof(T), &this->actual_size);+ return ptr;+ }++ cuda_pool_alloc(size_t size) {+ alloc(size);+ }++ ~cuda_pool_alloc() {+ if (ptr != nullptr) {+ ggml_cuda_pool_free(device, ptr, actual_size);+ }+ }++ T * get() {+ return ptr;+ }++ cuda_pool_alloc() = default;+ cuda_pool_alloc(const cuda_pool_alloc &) = delete;+ cuda_pool_alloc(cuda_pool_alloc &&) = delete;+ cuda_pool_alloc& operator=(const cuda_pool_alloc &) = delete;+ cuda_pool_alloc& operator=(cuda_pool_alloc &&) = delete;+};+static bool g_cublas_loaded = false;bool ggml_cublas_loaded(void) {@@ -7072,16 +7265,33 @@ void ggml_init_cublas() {#endiffprintf(stderr, "%s: found %d " GGML_CUDA_NAME " devices:\n", __func__, g_device_count);for (int id = 0; id < g_device_count; ++id) {+ int device_vmm = 0;++#if !defined(GGML_USE_HIPBLAS)+ CUdevice device;+ CU_CHECK(cuDeviceGet(&device, id));+ CU_CHECK(cuDeviceGetAttribute(&device_vmm, CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED, device));++ if (device_vmm) {+ CUmemAllocationProp alloc_prop = {};+ alloc_prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;+ alloc_prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;+ alloc_prop.location.id = id;+ CU_CHECK(cuMemGetAllocationGranularity(&g_device_caps[id].vmm_granularity, &alloc_prop, CU_MEM_ALLOC_GRANULARITY_RECOMMENDED));+ }+#endif // !defined(GGML_USE_HIPBLAS)+ g_device_caps[id].vmm = !!device_vmm;+cudaDeviceProp prop;CUDA_CHECK(cudaGetDeviceProperties(&prop, id));- fprintf(stderr, " Device %d: %s, compute capability %d.%d\n", id, prop.name, prop.major, prop.minor);+ fprintf(stderr, " Device %d: %s, compute capability %d.%d, VMM: %s\n", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no");g_tensor_split[id] = total_vram;total_vram += prop.totalGlobalMem;#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)- g_compute_capabilities[id] = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;+ g_device_caps[id].cc = 100*prop.major + 10*prop.minor + CC_OFFSET_AMD;#else- g_compute_capabilities[id] = 100*prop.major + 10*prop.minor;+ g_device_caps[id].cc = 100*prop.major + 10*prop.minor;#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)}for (int id = 0; id < g_device_count; ++id) {@@ -7089,7 +7299,7 @@ void ggml_init_cublas() {}for (int id = 0; id < g_device_count; ++id) {- CUDA_CHECK(ggml_cuda_set_device(id));+ ggml_cuda_set_device(id);// create cuda streamsfor (int is = 0; is < MAX_STREAMS; ++is) {@@ -7143,8 +7353,7 @@ void * ggml_cuda_host_malloc(size_t size) {void * ptr = nullptr;cudaError_t err = cudaMallocHost((void **) &ptr, size);if (err != cudaSuccess) {- // The allocation error can be bypassed. A null ptr will assigned out of this function.- // This can fixed the OOM error in WSL.+ // clear the errorcudaGetLastError();fprintf(stderr, "WARNING: failed to allocate %.2f MB of pinned memory: %s\n",size/1024.0/1024.0, cudaGetErrorString(err));@@ -7207,7 +7416,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(static void ggml_cuda_op_get_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_d, const float * src1_d, float * dst_d, const cudaStream_t & stream) {+ const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t stream) {GGML_ASSERT(src1->type == GGML_TYPE_I32);GGML_ASSERT(dst->type == GGML_TYPE_F32);@@ -7242,15 +7451,16 @@ static void ggml_cuda_op_get_rows(break;default:// TODO: k-quants+ fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));GGML_ASSERT(false);break;}}template<class op>-inline void ggml_cuda_op_bin_bcast(+static void ggml_cuda_op_bin_bcast(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -7269,7 +7479,7 @@ inline void ggml_cuda_op_bin_bcast(static void ggml_cuda_op_repeat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_d, const float * src1_d, float * dst_d, const cudaStream_t & main_stream) {+ const float * src0_d, const float * src1_d, float * dst_d, cudaStream_t main_stream) {ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, src0, dst, nullptr, src0_d, dst_d, main_stream);@@ -7277,16 +7487,16 @@ static void ggml_cuda_op_repeat((void) src1_d;}-inline void ggml_cuda_op_add(+static void ggml_cuda_op_add(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);}-inline void ggml_cuda_op_acc(+static void ggml_cuda_op_acc(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -7303,23 +7513,23 @@ inline void ggml_cuda_op_acc((void) dst;}-inline void ggml_cuda_op_mul(+static void ggml_cuda_op_mul(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);}-inline void ggml_cuda_op_div(+static void ggml_cuda_op_div(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(src0, src1, dst, src0_dd, src1_dd, dst_dd, main_stream);}-inline void ggml_cuda_op_gelu(+static void ggml_cuda_op_gelu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7331,9 +7541,9 @@ inline void ggml_cuda_op_gelu((void) src1_dd;}-inline void ggml_cuda_op_silu(+static void ggml_cuda_op_silu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7345,9 +7555,9 @@ inline void ggml_cuda_op_silu((void) src1_dd;}-inline void ggml_cuda_op_gelu_quick(+static void ggml_cuda_op_gelu_quick(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7359,9 +7569,9 @@ inline void ggml_cuda_op_gelu_quick((void) src1_dd;}-inline void ggml_cuda_op_tanh(+static void ggml_cuda_op_tanh(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7373,9 +7583,9 @@ inline void ggml_cuda_op_tanh((void) src1_dd;}-inline void ggml_cuda_op_relu(+static void ggml_cuda_op_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7387,9 +7597,9 @@ inline void ggml_cuda_op_relu((void) src1_dd;}-inline void ggml_cuda_op_leaky_relu(+static void ggml_cuda_op_leaky_relu(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7404,9 +7614,9 @@ inline void ggml_cuda_op_leaky_relu((void) src1_dd;}-inline void ggml_cuda_op_sqr(+static void ggml_cuda_op_sqr(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7418,9 +7628,9 @@ inline void ggml_cuda_op_sqr((void) src1_dd;}-inline void ggml_cuda_op_norm(+static void ggml_cuda_op_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7438,10 +7648,9 @@ inline void ggml_cuda_op_norm((void) src1_dd;}--inline void ggml_cuda_op_group_norm(+static void ggml_cuda_op_group_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7455,9 +7664,9 @@ inline void ggml_cuda_op_group_norm((void) src1_dd;}-inline void ggml_cuda_op_concat(+static void ggml_cuda_op_concat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -7471,9 +7680,9 @@ inline void ggml_cuda_op_concat((void) dst;}-inline void ggml_cuda_op_upscale(+static void ggml_cuda_op_upscale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT(dst->type == GGML_TYPE_F32);@@ -7488,9 +7697,9 @@ inline void ggml_cuda_op_upscale((void) src1_dd;}-inline void ggml_cuda_op_pad(+static void ggml_cuda_op_pad(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT(dst->type == GGML_TYPE_F32);@@ -7505,9 +7714,9 @@ inline void ggml_cuda_op_pad((void) src1_dd;}-inline void ggml_cuda_op_rms_norm(+static void ggml_cuda_op_rms_norm(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -7525,10 +7734,10 @@ inline void ggml_cuda_op_rms_norm((void) src1_dd;}-inline void ggml_cuda_op_mul_mat_q(+static void ggml_cuda_op_mul_mat_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, const cudaStream_t & stream) {+ const int64_t src1_padded_row_size, cudaStream_t stream) {const int64_t ne00 = src0->ne[0];@@ -7590,13 +7799,13 @@ inline void ggml_cuda_op_mul_mat_q(static int64_t get_row_rounding(ggml_type type) {int64_t min_compute_capability = INT_MAX;int64_t max_compute_capability = INT_MIN;- for (int64_t id = 0; id < g_device_count; ++id) {+ for (int id = 0; id < g_device_count; ++id) {if (g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) {- if (min_compute_capability > g_compute_capabilities[id]) {- min_compute_capability = g_compute_capabilities[id];+ if (min_compute_capability > g_device_caps[id].cc) {+ min_compute_capability = g_device_caps[id].cc;}- if (max_compute_capability < g_compute_capabilities[id]) {- max_compute_capability = g_compute_capabilities[id];+ if (max_compute_capability < g_device_caps[id].cc) {+ max_compute_capability = g_device_caps[id].cc;}}}@@ -7648,10 +7857,10 @@ static int64_t get_row_rounding(ggml_type type) {#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)}-inline void ggml_cuda_op_mul_mat_vec_q(+static void ggml_cuda_op_mul_mat_vec_q(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, const cudaStream_t & stream) {+ const int64_t src1_padded_row_size, cudaStream_t stream) {GGML_ASSERT(ggml_nrows(src1) == 1);@@ -7701,18 +7910,20 @@ inline void ggml_cuda_op_mul_mat_vec_q((void) src1_padded_row_size;}-inline void ggml_cuda_op_dequantize_mul_mat_vec(+static void ggml_cuda_op_dequantize_mul_mat_vec(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, const cudaStream_t & stream) {+ const int64_t src1_padded_row_size, cudaStream_t stream) {const int64_t ne00 = src0->ne[0];const int64_t row_diff = row_high - row_low;+ GGML_ASSERT(src1->type == GGML_TYPE_F32);+// on some GPUs it is faster to convert src1 to half and to use half precision intrinsics#ifdef GGML_CUDA_F16- size_t ash;- dfloat * src1_dfloat = nullptr; // dfloat == half+ cuda_pool_alloc<half> src1_dfloat_a;+ half * src1_dfloat = nullptr; // dfloat == halfbool src1_convert_f16 =src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 ||@@ -7720,7 +7931,7 @@ inline void ggml_cuda_op_dequantize_mul_mat_vec(src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16;if (src1_convert_f16) {- src1_dfloat = (half *) ggml_cuda_pool_malloc(ne00*sizeof(half), &ash);+ src1_dfloat = src1_dfloat_a.alloc(ne00);ggml_cpy_f32_f16_cuda((const char *) src1_ddf_i, (char *) src1_dfloat, ne00,ne00, 1, sizeof(float), 0, 0,ne00, 1, sizeof(half), 0, 0, stream);@@ -7768,12 +7979,6 @@ inline void ggml_cuda_op_dequantize_mul_mat_vec(break;}-#ifdef GGML_CUDA_F16- if (src1_convert_f16) {- ggml_cuda_pool_free(src1_dfloat, ash);- }-#endif // GGML_CUDA_F16-(void) src1;(void) dst;(void) src1_ddq_i;@@ -7781,10 +7986,10 @@ inline void ggml_cuda_op_dequantize_mul_mat_vec((void) src1_padded_row_size;}-inline void ggml_cuda_op_mul_mat_cublas(+static void ggml_cuda_op_mul_mat_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,- const int64_t src1_padded_row_size, const cudaStream_t & stream) {+ const int64_t src1_padded_row_size, cudaStream_t stream) {CUBLAS_ENTRY();GGML_ASSERT(src0_dd_i != nullptr);@@ -7805,33 +8010,31 @@ inline void ggml_cuda_op_mul_mat_cublas(// ldc == nrows of the matrix that cuBLAS writes intoint ldc = dst->backend == GGML_BACKEND_GPU && id == g_main_device ? ne0 : row_diff;- const int compute_capability = g_compute_capabilities[id];+ const int compute_capability = g_device_caps[id].cc;if (compute_capability >= CC_VOLTA && (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT) {+ //printf("this branch\n");// convert src0 and src1 to fp16, multiply as fp16, convert dst to fp32- half * src0_as_f16 = nullptr;- size_t src0_as = 0;+ cuda_pool_alloc<half> src0_as_f16;if (src0->type != GGML_TYPE_F16) {const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src0->type);GGML_ASSERT(to_fp16_cuda != nullptr);size_t ne = row_diff*ne00;- src0_as_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &src0_as);- to_fp16_cuda(src0_dd_i, src0_as_f16, ne, stream);+ src0_as_f16.alloc(ne);+ to_fp16_cuda(src0_dd_i, src0_as_f16.get(), ne, stream);}- const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16;+ const half * src0_ptr = src0->type == GGML_TYPE_F16 ? (const half *) src0_dd_i : src0_as_f16.get();- half * src1_as_f16 = nullptr;- size_t src1_as = 0;+ cuda_pool_alloc<half> src1_as_f16;if (src1->type != GGML_TYPE_F16) {const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);GGML_ASSERT(to_fp16_cuda != nullptr);size_t ne = src1_ncols*ne10;- src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &src1_as);- to_fp16_cuda(src1_ddf_i, src1_as_f16, ne, stream);+ src1_as_f16.alloc(ne);+ to_fp16_cuda(src1_ddf_i, src1_as_f16.get(), ne, stream);}- const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16;- size_t dst_as = 0;- half * dst_f16 = (half *) ggml_cuda_pool_malloc(row_diff*src1_ncols * sizeof(half), &dst_as);+ const half * src1_ptr = src1->type == GGML_TYPE_F16 ? (const half *) src1_ddf_i : src1_as_f16.get();+ cuda_pool_alloc<half> dst_f16(row_diff*src1_ncols);const half alpha_f16 = 1.0f;const half beta_f16 = 0.0f;@@ -7840,36 +8043,33 @@ inline void ggml_cuda_op_mul_mat_cublas(CUBLAS_CHECK(cublasGemmEx(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,row_diff, src1_ncols, ne10,- &alpha_f16, src0_ptr, CUDA_R_16F, ne00,- src1_ptr, CUDA_R_16F, ne10,- &beta_f16, dst_f16, CUDA_R_16F, ldc,+ &alpha_f16, src0_ptr, CUDA_R_16F, ne00,+ src1_ptr, CUDA_R_16F, ne10,+ &beta_f16, dst_f16.get(), CUDA_R_16F, ldc,CUBLAS_COMPUTE_16F,CUBLAS_GEMM_DEFAULT_TENSOR_OP));const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);- to_fp32_cuda(dst_f16, dst_dd_i, row_diff*src1_ncols, stream);-- ggml_cuda_pool_free(dst_f16, dst_as);-- if (src0_as != 0) {- ggml_cuda_pool_free(src0_as_f16, src0_as);- }-- if (src1_as != 0) {- ggml_cuda_pool_free(src1_as_f16, src1_as);- }- }- else {- float * src0_ddq_as_f32 = nullptr;- size_t src0_as = 0;+ to_fp32_cuda(dst_f16.get(), dst_dd_i, row_diff*src1_ncols, stream);+ } else {+ cuda_pool_alloc<float> src0_ddq_as_f32;+ cuda_pool_alloc<float> src1_ddq_as_f32;if (src0->type != GGML_TYPE_F32) {const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src0->type);GGML_ASSERT(to_fp32_cuda != nullptr);- src0_ddq_as_f32 = (float *) ggml_cuda_pool_malloc(row_diff*ne00 * sizeof(float), &src0_as); // NOLINT- to_fp32_cuda(src0_dd_i, src0_ddq_as_f32, row_diff*ne00, stream);+ src0_ddq_as_f32.alloc(row_diff*ne00);+ to_fp32_cuda(src0_dd_i, src0_ddq_as_f32.get(), row_diff*ne00, stream);+ }+ if (src1->type != GGML_TYPE_F32) {+ const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(src1->type);+ GGML_ASSERT(to_fp32_cuda != nullptr);+ src1_ddq_as_f32.alloc(src1_ncols*ne10);+ to_fp32_cuda(src1_ddf_i, src1_ddq_as_f32.get(), src1_ncols*ne10, stream);}- const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32;++ const float * src0_ddf_i = src0->type == GGML_TYPE_F32 ? (const float *) src0_dd_i : src0_ddq_as_f32.get();+ const float * src1_ddf1_i = src1->type == GGML_TYPE_F32 ? (const float *) src1_ddf_i : src1_ddq_as_f32.get();const float alpha = 1.0f;const float beta = 0.0f;@@ -7878,13 +8078,9 @@ inline void ggml_cuda_op_mul_mat_cublas(CUBLAS_CHECK(cublasSgemm(CUBLAS_HANDLE(id), CUBLAS_OP_T, CUBLAS_OP_N,row_diff, src1_ncols, ne10,- &alpha, src0_ddf_i, ne00,- src1_ddf_i, ne10,- &beta, dst_dd_i, ldc));-- if (src0_as != 0) {- ggml_cuda_pool_free(src0_ddq_as_f32, src0_as);- }+ &alpha, src0_ddf_i, ne00,+ src1_ddf1_i, ne10,+ &beta, dst_dd_i, ldc));}(void) dst;@@ -7892,9 +8088,9 @@ inline void ggml_cuda_op_mul_mat_cublas((void) src1_padded_row_size;}-inline void ggml_cuda_op_rope(+static void ggml_cuda_op_rope(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);@@ -7972,9 +8168,9 @@ inline void ggml_cuda_op_rope((void) src1_dd;}-inline void ggml_cuda_op_alibi(+static void ggml_cuda_op_alibi(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -8003,9 +8199,9 @@ inline void ggml_cuda_op_alibi((void) src1_dd;}-inline void ggml_cuda_op_im2col(+static void ggml_cuda_op_im2col(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F16);GGML_ASSERT(src1->type == GGML_TYPE_F32);@@ -8038,10 +8234,9 @@ inline void ggml_cuda_op_im2col((void) src0_dd;}--inline void ggml_cuda_op_sum_rows(+static void ggml_cuda_op_sum_rows(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -8056,9 +8251,9 @@ inline void ggml_cuda_op_sum_rows((void) src1_dd;}-inline void ggml_cuda_op_argsort(+static void ggml_cuda_op_argsort(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_I32);@@ -8075,9 +8270,9 @@ inline void ggml_cuda_op_argsort((void) src1_dd;}-inline void ggml_cuda_op_diag_mask_inf(+static void ggml_cuda_op_diag_mask_inf(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -8095,9 +8290,9 @@ inline void ggml_cuda_op_diag_mask_inf((void) src1_dd;}-inline void ggml_cuda_op_soft_max(+static void ggml_cuda_op_soft_max(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -8116,22 +8311,15 @@ inline void ggml_cuda_op_soft_max((void) dst;}-inline void ggml_cuda_op_scale(+static void ggml_cuda_op_scale(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);- GGML_ASSERT(src1->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);float scale;- // HACK: support for ggml backend interface- if (src1->backend == GGML_BACKEND_CPU) {- scale = ((float *) src1->data)[0];- } else {- // TODO: pass pointer to kernel instead of copying to host- CUDA_CHECK(cudaMemcpy(&scale, src1->data, sizeof(float), cudaMemcpyDeviceToHost));- }+ memcpy(&scale, dst->op_params, sizeof(float));scale_f32_cuda(src0_dd, dst_dd, scale, ggml_nelements(src0), main_stream);CUDA_CHECK(cudaGetLastError());@@ -8141,9 +8329,9 @@ inline void ggml_cuda_op_scale((void) src1_dd;}-inline void ggml_cuda_op_clamp(+static void ggml_cuda_op_clamp(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,- const float * src0_dd, const float * src1_dd, float * dst_dd, const cudaStream_t & main_stream) {+ const float * src0_dd, const float * src1_dd, float * dst_dd, cudaStream_t main_stream) {GGML_ASSERT(src0->type == GGML_TYPE_F32);GGML_ASSERT( dst->type == GGML_TYPE_F32);@@ -8178,40 +8366,37 @@ static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * sconst bool src1_on_device = use_src1 && src1->backend == GGML_BACKEND_GPU;const bool dst_on_device = dst->backend == GGML_BACKEND_GPU;- const bool src1_stays_on_host = use_src1 && dst->op == GGML_OP_SCALE;-// dd = data devicefloat * src0_ddf = nullptr;float * src1_ddf = nullptr;float * dst_ddf = nullptr;- // as = actual size- size_t src0_asf = 0;- size_t src1_asf = 0;- size_t dst_asf = 0;+ cuda_pool_alloc<float> src0_f;+ cuda_pool_alloc<float> src1_f;+ cuda_pool_alloc<float> dst_f;ggml_cuda_set_device(g_main_device);- const cudaStream_t main_stream = g_cudaStreams[g_main_device][0];+ cudaStream_t main_stream = g_cudaStreams[g_main_device][0];if (src0_on_device) {src0_ddf = (float *) src0_extra->data_device[g_main_device];} else {- src0_ddf = (float *) ggml_cuda_pool_malloc(ggml_nbytes(src0), &src0_asf);+ src0_ddf = src0_f.alloc(ggml_nelements(src0));CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_ddf, src0, 0, 0, 0, nrows0, main_stream));}- if (use_src1 && !src1_stays_on_host) {+ if (use_src1) {if (src1_on_device) {src1_ddf = (float *) src1_extra->data_device[g_main_device];} else {- src1_ddf = (float *) ggml_cuda_pool_malloc(ggml_nbytes(src1), &src1_asf);+ src1_ddf = src1_f.alloc(ggml_nelements(src1));CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src1_ddf, src1, 0, 0, 0, nrows1, main_stream));}}if (dst_on_device) {dst_ddf = (float *) dst_extra->data_device[g_main_device];} else {- dst_ddf = (float *) ggml_cuda_pool_malloc(ggml_nbytes(dst), &dst_asf);+ dst_ddf = dst_f.alloc(ggml_nelements(dst));}// do the computation@@ -8223,16 +8408,6 @@ static void ggml_cuda_op_flatten(const ggml_tensor * src0, const ggml_tensor * sCUDA_CHECK(cudaMemcpyAsync(dst->data, dst_ddf, ggml_nbytes(dst), cudaMemcpyDeviceToHost, main_stream));}- if (src0_asf > 0) {- ggml_cuda_pool_free(src0_ddf, src0_asf);- }- if (src1_asf > 0) {- ggml_cuda_pool_free(src1_ddf, src1_asf);- }- if (dst_asf > 0) {- ggml_cuda_pool_free(dst_ddf, dst_asf);- }-if (dst->backend == GGML_BACKEND_CPU) {CUDA_CHECK(cudaDeviceSynchronize());}@@ -8249,7 +8424,12 @@ static void ggml_cuda_set_peer_access(const int n_tokens) {#ifdef NDEBUGfor (int id = 0; id < g_device_count; ++id) {- CUDA_CHECK(ggml_cuda_set_device(id));+ ggml_cuda_set_device(id);+ CUDA_CHECK(cudaDeviceSynchronize());+ }++ for (int id = 0; id < g_device_count; ++id) {+ ggml_cuda_set_device(id);for (int id_other = 0; id_other < g_device_count; ++id_other) {if (id == id_other) {@@ -8283,7 +8463,6 @@ static void ggml_cuda_op_mul_mat(const int64_t ne01 = src0->ne[1];const int64_t ne02 = src0->ne[2];const int64_t ne03 = src0->ne[3];- const int64_t nrows0 = ggml_nrows(src0);const int64_t ne10 = src1->ne[0];const int64_t ne11 = src1->ne[1];@@ -8299,10 +8478,9 @@ static void ggml_cuda_op_mul_mat(const int nb2 = dst->nb[2];const int nb3 = dst->nb[3];- ggml_cuda_set_peer_access(ne11);-GGML_ASSERT(dst->backend != GGML_BACKEND_GPU_SPLIT);GGML_ASSERT(src1->backend != GGML_BACKEND_GPU_SPLIT);+ GGML_ASSERT(src1->type == GGML_TYPE_F32 || (src1->ne[2] == 1 && src1->ne[3] == 1));GGML_ASSERT(ne12 >= ne02 && ne12 % ne02 == 0);@@ -8328,27 +8506,29 @@ static void ggml_cuda_op_mul_mat(GGML_ASSERT(!(split && ne03 > 1));GGML_ASSERT(!(split && ne02 < ne12));- // dd = data device- char * src0_dd[GGML_CUDA_MAX_DEVICES] = {nullptr};- float * src1_ddf[GGML_CUDA_MAX_DEVICES] = {nullptr}; // float- char * src1_ddq[GGML_CUDA_MAX_DEVICES] = {nullptr}; // q8_1- float * dst_dd[GGML_CUDA_MAX_DEVICES] = {nullptr};+ struct dev_data {+ cuda_pool_alloc<char> src0_dd_alloc;+ cuda_pool_alloc<float> src1_ddf_alloc;+ cuda_pool_alloc<char> src1_ddq_alloc;+ cuda_pool_alloc<float> dst_dd_alloc;++ char * src0_dd = nullptr;+ float * src1_ddf = nullptr; // float+ char * src1_ddq = nullptr; // q8_1+ float * dst_dd = nullptr;- // as = actual size- size_t src0_as[GGML_CUDA_MAX_DEVICES] = {0};- size_t src1_asf[GGML_CUDA_MAX_DEVICES] = {0};- size_t src1_asq[GGML_CUDA_MAX_DEVICES] = {0};- size_t dst_as[GGML_CUDA_MAX_DEVICES] = {0};+ int64_t row_low;+ int64_t row_high;+ };- int64_t row_low[GGML_CUDA_MAX_DEVICES];- int64_t row_high[GGML_CUDA_MAX_DEVICES];+ dev_data dev[GGML_CUDA_MAX_DEVICES];int used_devices = 0;- for (int64_t id = 0; id < g_device_count; ++id) {+ for (int id = 0; id < g_device_count; ++id) {// by default, use all rows- row_low[id] = 0;- row_high[id] = ne01;+ dev[id].row_low = 0;+ dev[id].row_high = ne01;// for multi GPU, get the row boundaries from tensor split// and round to mul_mat_q tile sizes@@ -8356,19 +8536,23 @@ static void ggml_cuda_op_mul_mat(const int64_t rounding = get_row_rounding(src0->type);if (id != 0) {- row_low[id] = ne01*g_tensor_split[id];- row_low[id] -= row_low[id] % rounding;+ dev[id].row_low = ne01*g_tensor_split[id];+ if (dev[id].row_low < ne01) {+ dev[id].row_low -= dev[id].row_low % rounding;+ }}if (id != g_device_count - 1) {- row_high[id] = ne01*g_tensor_split[id + 1];- row_high[id] -= row_high[id] % rounding;+ dev[id].row_high = ne01*g_tensor_split[id + 1];+ if (dev[id].row_high < ne01) {+ dev[id].row_high -= dev[id].row_high % rounding;+ }}}}- for (int64_t id = 0; id < g_device_count; ++id) {- if ((!split && id != g_main_device) || row_low[id] == row_high[id]) {+ for (int id = 0; id < g_device_count; ++id) {+ if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) {continue;}@@ -8378,42 +8562,41 @@ static void ggml_cuda_op_mul_mat(const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device;ggml_cuda_set_device(id);- const cudaStream_t stream = g_cudaStreams[id][0];+ cudaStream_t stream = g_cudaStreams[id][0];if (src0_on_device && src0_is_contiguous) {- src0_dd[id] = (char *) src0_extra->data_device[id];+ dev[id].src0_dd = (char *) src0_extra->data_device[id];} else {- // const size_t size_src0_ddq = split ? (row_high[id]-row_low[id])*ne00 * src0_ts/src0_bs : ggml_nbytes(src0);- src0_dd[id] = (char *) ggml_cuda_pool_malloc(ggml_nbytes(src0), &src0_as[id]);+ dev[id].src0_dd = dev[id].src0_dd_alloc.alloc(ggml_nbytes(src0));}if (src1_on_device && src1_is_contiguous) {- src1_ddf[id] = (float *) src1_extra->data_device[id];+ dev[id].src1_ddf = (float *) src1_extra->data_device[id];} else {- src1_ddf[id] = (float *) ggml_cuda_pool_malloc(ggml_nbytes(src1), &src1_asf[id]);+ dev[id].src1_ddf = dev[id].src1_ddf_alloc.alloc(ggml_nelements(src1));}if (convert_src1_to_q8_1) {- src1_ddq[id] = (char *) ggml_cuda_pool_malloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs, &src1_asq[id]);+ dev[id].src1_ddq = dev[id].src1_ddq_alloc.alloc(nrows1*src1_padded_col_size*q8_1_ts/q8_1_bs);if (src1_on_device && src1_is_contiguous) {- quantize_row_q8_1_cuda(src1_ddf[id], src1_ddq[id], ne10, nrows1, src1_padded_col_size, stream);+ quantize_row_q8_1_cuda(dev[id].src1_ddf, dev[id].src1_ddq, ne10, nrows1, src1_padded_col_size, stream);CUDA_CHECK(cudaGetLastError());}}if (dst_on_device) {- dst_dd[id] = (float *) dst_extra->data_device[id];+ dev[id].dst_dd = (float *) dst_extra->data_device[id];} else {- const size_t size_dst_ddf = split ? (row_high[id]-row_low[id])*ne1*sizeof(float) : ggml_nbytes(dst);- dst_dd[id] = (float *) ggml_cuda_pool_malloc(size_dst_ddf, &dst_as[id]);+ const size_t size_dst_ddf = split ? (dev[id].row_high - dev[id].row_low)*ne1 : ggml_nelements(dst);+ dev[id].dst_dd = dev[id].dst_dd_alloc.alloc(size_dst_ddf);}}// if multiple devices are used they need to wait for the main device// here an event is recorded that signals that the main device has finished calculating the input dataif (split && used_devices > 1) {- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);CUDA_CHECK(cudaEventRecord(src0_extra->events[g_main_device][0], g_cudaStreams[g_main_device][0]));}@@ -8422,17 +8605,17 @@ static void ggml_cuda_op_mul_mat(const int64_t is = split ? (src1_col_0/src1_col_stride) % MAX_STREAMS : 0;const int64_t src1_ncols = src1_col_0 + src1_col_stride > ne11 ? ne11 - src1_col_0 : src1_col_stride;- for (int64_t id = 0; id < g_device_count; ++id) {- if ((!split && id != g_main_device) || row_low[id] == row_high[id]) {+ for (int id = 0; id < g_device_count; ++id) {+ if ((!split && id != g_main_device) || dev[id].row_low == dev[id].row_high) {continue;}const bool src1_on_device = src1->backend == GGML_BACKEND_GPU && id == g_main_device;const bool dst_on_device = dst->backend == GGML_BACKEND_GPU && id == g_main_device;- const int64_t row_diff = row_high[id] - row_low[id];+ const int64_t row_diff = dev[id].row_high - dev[id].row_low;ggml_cuda_set_device(id);- const cudaStream_t stream = g_cudaStreams[id][is];+ cudaStream_t stream = g_cudaStreams[id][is];// wait for main GPU data if necessaryif (split && (id != g_main_device || is != 0)) {@@ -8446,34 +8629,34 @@ static void ggml_cuda_op_mul_mat(const size_t src1_ddq_i_offset = (i0*ne11 + src1_col_0) * src1_padded_col_size*q8_1_ts/q8_1_bs;// for split tensors the data begins at i0 == i0_offset_low- char * src0_dd_i = src0_dd[id] + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;- float * src1_ddf_i = src1_ddf[id] + (i0*ne11 + src1_col_0) * ne10;- char * src1_ddq_i = src1_ddq[id] + src1_ddq_i_offset;- float * dst_dd_i = dst_dd[id] + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);+ char * src0_dd_i = dev[id].src0_dd + (i0/i02_divisor) * (ne01*ne00*src0_ts)/src0_bs;+ float * src1_ddf_i = dev[id].src1_ddf + (i0*ne11 + src1_col_0) * ne10;+ char * src1_ddq_i = dev[id].src1_ddq + src1_ddq_i_offset;+ float * dst_dd_i = dev[id].dst_dd + (i0*ne1 + src1_col_0) * (dst_on_device ? ne0 : row_diff);// the main device memory buffer can be on VRAM scratch, with space for all partial results// in that case an offset on dst_ddf_i is neededif (dst->backend == GGML_BACKEND_GPU && id == g_main_device) {- dst_dd_i += row_low[id]; // offset is 0 if no tensor split+ dst_dd_i += dev[id].row_low; // offset is 0 if no tensor split}// copy src0, src1 to device if necessaryif (src1->backend == GGML_BACKEND_GPU && src1_is_contiguous) {if (id != g_main_device) {if (convert_src1_to_q8_1) {- char * src1_ddq_i_source = src1_ddq[g_main_device] + src1_ddq_i_offset;- CUDA_CHECK(cudaMemcpyAsync(src1_ddq_i, src1_ddq_i_source, src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs,- cudaMemcpyDeviceToDevice, stream));+ char * src1_ddq_i_source = dev[g_main_device].src1_ddq + src1_ddq_i_offset;+ CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddq_i, id, src1_ddq_i_source, g_main_device,+ src1_ncols*src1_padded_col_size*q8_1_ts/q8_1_bs, stream));} else {float * src1_ddf_i_source = (float *) src1_extra->data_device[g_main_device];src1_ddf_i_source += (i0*ne11 + src1_col_0) * ne10;- CUDA_CHECK(cudaMemcpyAsync(src1_ddf_i, src1_ddf_i_source, src1_ncols*ne10*sizeof(float),- cudaMemcpyDeviceToDevice, stream));+ CUDA_CHECK(cudaMemcpyPeerAsync(src1_ddf_i, id, src1_ddf_i_source, g_main_device,+ src1_ncols*ne10*sizeof(float), stream));}}} else if (src1->backend == GGML_BACKEND_CPU || (src1_on_device && !src1_is_contiguous)) {CUDA_CHECK(ggml_cuda_cpy_tensor_2d(- src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));+ src1_ddf_i, src1, i03, i02, src1_col_0, src1_col_0+src1_ncols, stream));} else {GGML_ASSERT(false);}@@ -8484,12 +8667,12 @@ static void ggml_cuda_op_mul_mat(}if (src1_col_0 == 0 && (!src0_on_device || !src0_is_contiguous) && i02 % i02_divisor == 0) {- CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, row_low[id], row_high[id], stream));+ CUDA_CHECK(ggml_cuda_cpy_tensor_2d(src0_dd_i, src0, i03, i02/i02_divisor, dev[id].row_low, dev[id].row_high, stream));}// do the computationop(src0, src1, dst, src0_dd_i, src1_ddf_i, src1_ddq_i, dst_dd_i,- row_low[id], row_high[id], src1_ncols, src1_padded_col_size, stream);+ dev[id].row_low, dev[id].row_high, src1_ncols, src1_padded_col_size, stream);CUDA_CHECK(cudaGetLastError());// copy dst to host or other device if necessary@@ -8513,9 +8696,25 @@ static void ggml_cuda_op_mul_mat(// If dst is a vector with ne0 == 1 then you don't have to do this but it still produces correct results.float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));- dhf_dst_i += src1_col_0*ne0 + row_low[id];- CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float), dst_dd_i, row_diff*sizeof(float),- row_diff*sizeof(float), src1_ncols, kind, stream));+ dhf_dst_i += src1_col_0*ne0 + dev[id].row_low;+#if !defined(GGML_USE_HIPBLAS)+ if (kind == cudaMemcpyDeviceToDevice) {+ // cudaMemcpy2DAsync may fail with copies between vmm pools of different devices+ cudaMemcpy3DPeerParms p = {};+ p.dstDevice = g_main_device;+ p.dstPtr = make_cudaPitchedPtr(dhf_dst_i, ne0*sizeof(float), row_diff, src1_ncols);+ p.srcDevice = id;+ p.srcPtr = make_cudaPitchedPtr(dst_dd_i, row_diff*sizeof(float), row_diff, src1_ncols);+ p.extent = make_cudaExtent(row_diff*sizeof(float), src1_ncols, 1);+ CUDA_CHECK(cudaMemcpy3DPeerAsync(&p, stream));+ } else+#endif+ {+ CUDA_CHECK(cudaMemcpy2DAsync(dhf_dst_i, ne0*sizeof(float),+ dst_dd_i, row_diff*sizeof(float),+ row_diff*sizeof(float), src1_ncols,+ kind, stream));+ }} else {float * dhf_dst_i = (float *) ((char *) dst_off_device + i02*nb2 + i03*nb3);GGML_ASSERT(dst->nb[1] == ne0*sizeof(float));@@ -8532,35 +8731,14 @@ static void ggml_cuda_op_mul_mat(}}- for (int64_t id = 0; id < g_device_count; ++id) {- if ((!split && id != g_main_device) || row_low[id] == row_high[id]) {- continue;- }- CUDA_CHECK(ggml_cuda_set_device(id));-- // free buffers again when done- if (src0_as[id] > 0) {- ggml_cuda_pool_free(src0_dd[id], src0_as[id]);- }- if (src1_asf[id] > 0) {- ggml_cuda_pool_free(src1_ddf[id], src1_asf[id]);- }- if (src1_asq[id] > 0) {- ggml_cuda_pool_free(src1_ddq[id], src1_asq[id]);- }- if (dst_as[id] > 0) {- ggml_cuda_pool_free(dst_dd[id], dst_as[id]);- }- }-// main device waits for all other devices to be finishedif (split && g_device_count > 1) {int64_t is_max = (ne11 + MUL_MAT_SRC1_COL_STRIDE - 1) / MUL_MAT_SRC1_COL_STRIDE;is_max = is_max <= MAX_STREAMS ? is_max : MAX_STREAMS;- CUDA_CHECK(ggml_cuda_set_device(g_main_device));- for (int64_t id = 0; id < g_device_count; ++id) {- if (row_low[id] == row_high[id]) {+ ggml_cuda_set_device(g_main_device);+ for (int id = 0; id < g_device_count; ++id) {+ if (dev[id].row_low == dev[id].row_high) {continue;}for (int64_t is = 0; is < is_max; ++is) {@@ -8570,7 +8748,7 @@ static void ggml_cuda_op_mul_mat(}if (dst->backend == GGML_BACKEND_CPU) {- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);CUDA_CHECK(cudaDeviceSynchronize());}}@@ -8680,7 +8858,7 @@ static void ggml_cuda_mul_mat_vec_p021(const ggml_tensor * src0, const ggml_tensconst int64_t ne12 = src1->ne[2];- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;@@ -8712,7 +8890,7 @@ static void ggml_cuda_mul_mat_vec_nc(const ggml_tensor * src0, const ggml_tensorconst int64_t ne12 = src1->ne[2];- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;@@ -8749,9 +8927,9 @@ static __global__ void k_compute_batched_ptrs(int64_t i03 = i13 / r3;int64_t i02 = i12 / r2;- ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;- ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12/2 + i13*nb13/2;- ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;+ ptrs_src[0*ne23 + i12 + i13*ne12] = (const char *) src0_as_f16 + i02*nb02 + i03*nb03;+ ptrs_src[1*ne23 + i12 + i13*ne12] = (const char *) src1_as_f16 + i12*nb12 + i13*nb13;+ ptrs_dst[0*ne23 + i12 + i13*ne12] = ( char *) dst + i12*nbd2 + i13*nbd3;}static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {@@ -8761,37 +8939,19 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constGGML_ASSERT(src0->backend != GGML_BACKEND_GPU_SPLIT);GGML_ASSERT(src0->type == GGML_TYPE_F16);- GGML_ASSERT(src1->type == GGML_TYPE_F32);-- const int64_t ne00 = src0->ne[0]; GGML_UNUSED(ne00);- const int64_t ne01 = src0->ne[1];- const int64_t ne02 = src0->ne[2];- const int64_t ne03 = src0->ne[3];-- const int64_t nb01 = src0->nb[1];- const int64_t nb02 = src0->nb[2]; GGML_UNUSED(nb02);- const int64_t nb03 = src0->nb[3]; GGML_UNUSED(nb03);-- const int64_t ne10 = src1->ne[0];- const int64_t ne11 = src1->ne[1];- const int64_t ne12 = src1->ne[2];- const int64_t ne13 = src1->ne[3];- const int64_t nb11 = src1->nb[1];- const int64_t nb12 = src1->nb[2]; GGML_UNUSED(nb12);- const int64_t nb13 = src1->nb[3]; GGML_UNUSED(nb13);+ GGML_TENSOR_BINARY_OP_LOCALS- const int64_t ne1 = ggml_nelements(src1);- const int64_t ne = ggml_nelements(dst);+ const int64_t ne_dst = ggml_nelements(dst);- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];CUBLAS_SET_STREAM(g_main_device, main_stream);ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;void * src0_ddq = src0_extra->data_device[g_main_device];- half * src0_as_f16 = (half *) src0_ddq;+ half * src0_f16 = (half *) src0_ddq;ggml_tensor_extra_gpu * src1_extra = (ggml_tensor_extra_gpu *) src1->extra;float * src1_ddf = (float *) src1_extra->data_device[g_main_device];@@ -8800,17 +8960,18 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constfloat * dst_ddf = (float *) dst_extra->data_device[g_main_device];// convert src1 to fp16- const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);- GGML_ASSERT(to_fp16_cuda != nullptr);-- size_t src1_as = 0;- half * src1_as_f16 = (half *) ggml_cuda_pool_malloc(ne1 * sizeof(half), &src1_as);- to_fp16_cuda(src1_ddf, src1_as_f16, ne1, main_stream);-- size_t dst_as = 0;+ cuda_pool_alloc<half> src1_f16_alloc;+ if (src1->type != GGML_TYPE_F16) {+ const to_fp16_cuda_t to_fp16_cuda = ggml_get_to_fp16_cuda(src1->type);+ const int64_t ne_src1 = ggml_nelements(src1);+ src1_f16_alloc.alloc(ne_src1);+ GGML_ASSERT(to_fp16_cuda != nullptr);+ to_fp16_cuda(src1_ddf, src1_f16_alloc.get(), ne_src1, main_stream);+ }+ half * src1_f16 = src1->type == GGML_TYPE_F16 ? (half *) src1_ddf : src1_f16_alloc.get();- half * dst_f16 = nullptr;- char * dst_t = nullptr;+ cuda_pool_alloc<half> dst_f16;+ char * dst_t;cublasComputeType_t cu_compute_type = CUBLAS_COMPUTE_16F;cudaDataType_t cu_data_type = CUDA_R_16F;@@ -8829,8 +8990,7 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constconst void * beta = &beta_f16;if (dst->op_params[0] == GGML_PREC_DEFAULT) {- dst_f16 = (half *) ggml_cuda_pool_malloc(ne * sizeof(half), &dst_as);- dst_t = (char *) dst_f16;+ dst_t = (char *) dst_f16.alloc(ne_dst);nbd2 /= sizeof(float) / sizeof(half);nbd3 /= sizeof(float) / sizeof(half);@@ -8877,9 +9037,9 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constCUBLAS_CHECK(cublasGemmStridedBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,- alpha, (const char *) src0_as_f16, CUDA_R_16F, nb01/sizeof(half), src0->nb[2]/sizeof(half), // strideA- (const char *) src1_as_f16, CUDA_R_16F, nb11/sizeof(float), src1->nb[2]/sizeof(float), // strideB- beta, ( char *) dst_t, cu_data_type, ne01, dst->nb[2]/sizeof(float), // strideC+ alpha, (const char *) src0_f16, CUDA_R_16F, nb01/nb00, nb02/nb00, // strideA+ (const char *) src1_f16, CUDA_R_16F, nb11/nb10, nb12/nb10, // strideB+ beta, ( char *) dst_t, cu_data_type, ne01, nb2/nb0, // strideCne12*ne13,cu_compute_type,CUBLAS_GEMM_DEFAULT_TENSOR_OP));@@ -8887,23 +9047,18 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, const// use cublasGemmBatchedExconst int ne23 = ne12*ne13;- const void ** ptrs_src = nullptr;- void ** ptrs_dst = nullptr;-- size_t ptrs_src_s = 0;- size_t ptrs_dst_s = 0;-- ptrs_src = (const void **) ggml_cuda_pool_malloc(2*ne23*sizeof(void *), &ptrs_src_s);- ptrs_dst = ( void **) ggml_cuda_pool_malloc(1*ne23*sizeof(void *), &ptrs_dst_s);+ cuda_pool_alloc<const void *> ptrs_src(2*ne23);+ cuda_pool_alloc< void *> ptrs_dst(1*ne23);dim3 block_dims(ne13, ne12);k_compute_batched_ptrs<<<1, block_dims, 0, main_stream>>>(- src0_as_f16, src1_as_f16, dst_t,- ptrs_src, ptrs_dst,+ src0_f16, src1_f16, dst_t,+ ptrs_src.get(), ptrs_dst.get(),ne12, ne13,ne23,nb02, nb03,- nb12, nb13,+ src1->type == GGML_TYPE_F16 ? nb12 : nb12/2,+ src1->type == GGML_TYPE_F16 ? nb13 : nb13/2,nbd2, nbd3,r2, r3);CUDA_CHECK(cudaGetLastError());@@ -8911,30 +9066,19 @@ static void ggml_cuda_mul_mat_mat_batched_cublas(const ggml_tensor * src0, constCUBLAS_CHECK(cublasGemmBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,- alpha, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, nb01/sizeof(half),- (const void **) (ptrs_src + 1*ne23), CUDA_R_16F, nb11/sizeof(float),- beta, ( void **) (ptrs_dst + 0*ne23), cu_data_type, ne01,+ alpha, (const void **) (ptrs_src.get() + 0*ne23), CUDA_R_16F, nb01/nb00,+ (const void **) (ptrs_src.get() + 1*ne23), CUDA_R_16F, nb11/nb10,+ beta, ( void **) (ptrs_dst.get() + 0*ne23), cu_data_type, ne01,ne23,cu_compute_type,CUBLAS_GEMM_DEFAULT_TENSOR_OP));-- if (ptrs_src_s != 0) {- ggml_cuda_pool_free(ptrs_src, ptrs_src_s);- }- if (ptrs_dst_s != 0) {- ggml_cuda_pool_free(ptrs_dst, ptrs_dst_s);- }}#endifif (dst->op_params[0] == GGML_PREC_DEFAULT) {const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(GGML_TYPE_F16);- to_fp32_cuda(dst_f16, dst_ddf, ne, main_stream);-- ggml_cuda_pool_free(dst_f16, dst_as);+ to_fp32_cuda(dst_f16.get(), dst_ddf, ne_dst, main_stream);}-- ggml_cuda_pool_free(src1_as_f16, src1_as);}static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {@@ -8946,17 +9090,31 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1const bool split = src0->backend == GGML_BACKEND_GPU_SPLIT;int64_t min_compute_capability = INT_MAX;- for (int64_t id = 0; id < g_device_count; ++id) {- if (min_compute_capability > g_compute_capabilities[id] && g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) {- min_compute_capability = g_compute_capabilities[id];+ for (int id = 0; id < g_device_count; ++id) {+ if (min_compute_capability > g_device_caps[id].cc && g_tensor_split[id] < (id + 1 < g_device_count ? g_tensor_split[id + 1] : 1.0f)) {+ min_compute_capability = g_device_caps[id].cc;}}+#if defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)++ const bool fp16_performance_good = min_compute_capability >= CC_RDNA1;+ bool use_mul_mat_q = ggml_is_quantized(src0->type);#ifdef CUDA_USE_TENSOR_CORES- const bool use_tensor_cores = true;+ use_mul_mat_q = use_mul_mat_q && min_compute_capability < CC_RDNA3;+#endif // CUDA_USE_TENSOR_CORES+#else- const bool use_tensor_cores = false;-#endif++ const bool fp16_performance_good = min_compute_capability >= CC_VOLTA;+ bool use_mul_mat_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type);+#ifdef CUDA_USE_TENSOR_CORES+ // when tensor cores are available, use them for large batch size+ // ref: https://github.com/ggerganov/llama.cpp/pull/3776+ use_mul_mat_q = use_mul_mat_q && !(fp16_performance_good && src1->ne[1] > MMQ_MAX_BATCH_SIZE);+#endif // CUDA_USE_TENSOR_CORES++#endif // defined(GGML_USE_HIPBLAS) && defined(__HIP_PLATFORM_AMD__)// debug helpers//printf("src0: %8d %8d %8d %8d\n", src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3]);@@ -8966,19 +9124,19 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1//printf("src0 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src0), ggml_is_transposed(src0), ggml_type_name(src0->type), src0->name);//printf("src1 is contiguous %d, transposed %d, type = %s, name = %s\n", ggml_is_contiguous(src1), ggml_is_transposed(src1), ggml_type_name(src1->type), src1->name);- if (!split && all_on_device && !use_tensor_cores && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {+ if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && ggml_is_permuted(src0) && ggml_is_permuted(src1) && src1->ne[1] == 1) {// KQ single-batchggml_cuda_mul_mat_vec_p021(src0, src1, dst);- } else if (!split && all_on_device && !use_tensor_cores && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {+ } else if (!split && all_on_device && !fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_contiguous(src0) && !ggml_is_transposed(src1) && src1->ne[1] == 1) {// KQV single-batchggml_cuda_mul_mat_vec_nc(src0, src1, dst);- } else if (!split && all_on_device && use_tensor_cores && src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) {+ } else if (!split && all_on_device && fp16_performance_good && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1)) {// KQ + KQV multi-batchggml_cuda_mul_mat_mat_batched_cublas(src0, src1, dst);} else if (src0->type == GGML_TYPE_F32) {ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_cublas, false);} else if (ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F16) {- if (src1->ne[1] == 1 && src0->ne[0] % GGML_CUDA_DMMV_X == 0) {+ if (src1->ne[1] == 1 && src0->ne[0] % GGML_CUDA_DMMV_X == 0 && src1->type == GGML_TYPE_F32) {#ifdef GGML_CUDA_FORCE_DMMVconst bool use_mul_mat_vec_q = false;#else@@ -8992,14 +9150,6 @@ static void ggml_cuda_mul_mat(const ggml_tensor * src0, const ggml_tensor * src1ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_dequantize_mul_mat_vec, false);}} else {- bool use_mul_mat_q = min_compute_capability >= MIN_CC_DP4A && ggml_is_quantized(src0->type);-- // when tensor cores are available, use them for large batch size- // ref: https://github.com/ggerganov/llama.cpp/pull/3776- if (use_tensor_cores && min_compute_capability >= CC_VOLTA && src1->ne[1] > MMQ_MAX_BATCH_SIZE) {- use_mul_mat_q = false;- }-if (use_mul_mat_q) {ggml_cuda_op_mul_mat(src0, src1, dst, ggml_cuda_op_mul_mat_q, true);} else {@@ -9089,7 +9239,7 @@ static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) {const int64_t ne1 = ggml_nelements(src1);const int64_t ne = ggml_nelements(dst);- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];CUBLAS_SET_STREAM(id, main_stream);@@ -9165,7 +9315,7 @@ static void ggml_cuda_mul_mat_id_cublas(ggml_tensor * dst) {CUDA_CHECK(cudaGetLastError());CUBLAS_CHECK(- cublasGemmBatchedEx(g_cublas_handles[g_main_device], CUBLAS_OP_T, CUBLAS_OP_N,+ cublasGemmBatchedEx(CUBLAS_HANDLE(g_main_device), CUBLAS_OP_T, CUBLAS_OP_N,ne01, ne11, ne10,&alpha_f16, (const void **) (ptrs_src + 0*ne23), CUDA_R_16F, ne00,(const void **) (ptrs_src + 1*ne23), CUDA_R_16F, ne10,@@ -9198,7 +9348,8 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * s// TODO: mmq/mmv support#endif- GGML_ASSERT(dst->backend == GGML_BACKEND_GPU);+ const int64_t nb11 = src1->nb[1];+ const int64_t nb1 = dst->nb[1];const struct ggml_tensor * ids = src0;const int32_t id = ((int32_t *) dst->op_params)[0];@@ -9206,10 +9357,12 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * sstd::vector<char> ids_host(ggml_nbytes(ids));+ cudaStream_t stream = g_cudaStreams[g_main_device][0];+if (ids->backend == GGML_BACKEND_GPU) {const char * ids_dev = (const char *)((const ggml_tensor_extra_gpu *)ids->extra)->data_device[g_main_device];- CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));- CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));+ CUDA_CHECK(cudaMemcpyAsync(ids_host.data(), ids_dev, ggml_nbytes(ids), cudaMemcpyDeviceToHost, stream));+ CUDA_CHECK(cudaStreamSynchronize(stream));} else {memcpy(ids_host.data(), ids->data, ggml_nbytes(ids));}@@ -9223,37 +9376,106 @@ static void ggml_cuda_mul_mat_id(const ggml_tensor * src0, const ggml_tensor * sggml_tensor src1_row = *src1;ggml_tensor dst_row = *dst;- src1_row.ne[1] = 1;- dst_row.ne[1] = 1;-- src1_row.nb[2] = src1_row.nb[1];- dst_row.nb[2] = dst_row.nb[1];-- src1_row.nb[3] = src1_row.nb[1];- dst_row.nb[3] = dst_row.nb[1];+ src1_row.backend = GGML_BACKEND_GPU;+ dst_row.backend = GGML_BACKEND_GPU;src1_row.extra = &src1_row_extra;dst_row.extra = &dst_row_extra;+ char * src1_original = src1->backend == GGML_BACKEND_CPU ?+ (char *) src1->data : (char *) src1_extra->data_device[g_main_device];+ char * dst_original = dst->backend == GGML_BACKEND_CPU ?+ (char *) dst->data : (char *) dst_extra->data_device[g_main_device];++ if (src1->ne[1] == 1) {+ GGML_ASSERT(src1->backend == GGML_BACKEND_GPU);+ GGML_ASSERT(dst->backend == GGML_BACKEND_GPU);++ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ //int32_t row_id;+ //CUDA_CHECK(cudaMemcpyAsync(&row_id, ids_dev + i01*ids->nb[1] + id*ids->nb[0], sizeof(int32_t), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));+ //CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));++ const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);++ GGML_ASSERT(row_id >= 0 && row_id < n_as);++ const struct ggml_tensor * src0_row = dst->src[row_id + 2];++ src1_row_extra.data_device[g_main_device] = src1_original + i01*src1->nb[1];+ src1_row.data = (char *) src1->data + i01*src1->nb[1]; // TODO why is this set?++ dst_row_extra.data_device[g_main_device] = dst_original + i01*dst->nb[1];+ dst_row.data = (char *) dst->data + i01*dst->nb[1]; // TODO why is this set?- for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {- //int32_t row_id;- //CUDA_CHECK(cudaMemcpyAsync(&row_id, ids_dev + i01*ids->nb[1] + id*ids->nb[0], sizeof(int32_t), cudaMemcpyDeviceToHost, g_cudaStreams[g_main_device][0]));- //CUDA_CHECK(cudaStreamSynchronize(g_cudaStreams[g_main_device][0]));+ ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);+ }+ } else {+ cuda_pool_alloc<char> src1_contiguous(sizeof(float)*ggml_nelements(src1));+ cuda_pool_alloc<char> dst_contiguous(sizeof(float)*ggml_nelements(dst));++ src1_row_extra.data_device[g_main_device] = src1_contiguous.get();+ dst_row_extra.data_device[g_main_device] = dst_contiguous.get();++ const cudaMemcpyKind src1_kind = src1->backend == GGML_BACKEND_CPU ?+ cudaMemcpyHostToDevice : cudaMemcpyDeviceToDevice;+ const cudaMemcpyKind dst_kind = dst->backend == GGML_BACKEND_CPU ?+ cudaMemcpyDeviceToHost : cudaMemcpyDeviceToDevice;++ for (int32_t row_id = 0; row_id < n_as; ++row_id) {+ const struct ggml_tensor * src0_row = dst->src[row_id + 2];- const int32_t row_id = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);+ int64_t num_src1_rows = 0;+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);- GGML_ASSERT(row_id >= 0 && row_id < n_as);+ if (row_id_i != row_id) {+ continue;+ }++ GGML_ASSERT(row_id >= 0 && row_id < n_as);- const struct ggml_tensor * src0_row = dst->src[row_id + 2];+ CUDA_CHECK(cudaMemcpyAsync(src1_contiguous.get() + num_src1_rows*nb11, src1_original + i01*nb11,+ nb11, src1_kind, stream));+ num_src1_rows++;+ }- src1_row_extra.data_device[g_main_device] = (char *) src1_extra->data_device[g_main_device] + i01*src1->nb[1];- src1_row.data = (char *) src1->data + i01*src1->nb[1];+ if (num_src1_rows == 0) {+ continue;+ }- dst_row_extra.data_device[g_main_device] = (char *) dst_extra->data_device[g_main_device] + i01*dst->nb[1];- dst_row.data = (char *) dst->data + i01*dst->nb[1];+ src1_row.ne[1] = num_src1_rows;+ dst_row.ne[1] = num_src1_rows;- ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);+ src1_row.nb[1] = nb11;+ src1_row.nb[2] = num_src1_rows*nb11;+ src1_row.nb[3] = num_src1_rows*nb11;++ dst_row.nb[1] = nb1;+ dst_row.nb[2] = num_src1_rows*nb1;+ dst_row.nb[3] = num_src1_rows*nb1;++ ggml_cuda_mul_mat(src0_row, &src1_row, &dst_row);++ num_src1_rows = 0;+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id_i = *(const int32_t *) (ids_host.data() + i01*ids->nb[1] + id*ids->nb[0]);++ if (row_id_i != row_id) {+ continue;+ }++ GGML_ASSERT(row_id >= 0 && row_id < n_as);++ CUDA_CHECK(cudaMemcpyAsync(dst_original + i01*nb1, dst_contiguous.get() + num_src1_rows*nb1,+ nb1, dst_kind, stream));+ num_src1_rows++;+ }+ }+ }++ if (dst->backend == GGML_BACKEND_CPU) {+ CUDA_CHECK(cudaStreamSynchronize(stream));}}@@ -9291,7 +9513,7 @@ static void ggml_cuda_cpy(const ggml_tensor * src0, const ggml_tensor * src1, ggconst int64_t nb11 = src1->nb[1];const int64_t nb12 = src1->nb[2];- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);cudaStream_t main_stream = g_cudaStreams[g_main_device][0];const ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu *) src0->extra;@@ -9381,7 +9603,7 @@ void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {ggml_tensor_extra_gpu * extra = new struct ggml_tensor_extra_gpu;memset(extra, 0, sizeof(*extra));- for (int64_t id = 0; id < g_device_count; ++id) {+ for (int id = 0; id < g_device_count; ++id) {if (backend == GGML_BACKEND_GPU && id != g_main_device) {continue;}@@ -9419,13 +9641,12 @@ void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {// pad last row to a multiple of 512 elements to avoid out-of-bounds memory accessesif (ne0 % MATRIX_ROW_PADDING != 0) {- size += (MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING)- * ggml_type_size(tensor->type)/ggml_blck_size(tensor->type);+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);}char * buf;CUDA_CHECK(cudaMalloc(&buf, size));- char * buf_host = (char*)data + offset_split;+ char * buf_host = (char *)data + offset_split;// set padding to 0 to avoid possible NaN valuesif (size > original_size) {@@ -9447,21 +9668,20 @@ void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor) {}void ggml_cuda_free_data(struct ggml_tensor * tensor) {- if (!tensor || (tensor->backend != GGML_BACKEND_GPU && tensor->backend != GGML_BACKEND_GPU_SPLIT) ) {+ if (!tensor || !tensor->extra || (tensor->backend != GGML_BACKEND_GPU && tensor->backend != GGML_BACKEND_GPU_SPLIT) ) {return;}ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra;- for (int64_t id = 0; id < g_device_count; ++id) {+ for (int id = 0; id < g_device_count; ++id) {+ ggml_cuda_set_device(id);if (extra->data_device[id] != nullptr) {- CUDA_CHECK(ggml_cuda_set_device(id));CUDA_CHECK(cudaFree(extra->data_device[id]));}for (int64_t is = 0; is < MAX_STREAMS; ++is) {if (extra->events[id][is] != nullptr) {- CUDA_CHECK(ggml_cuda_set_device(id));CUDA_CHECK(cudaEventDestroy(extra->events[id][is]));}}@@ -9515,7 +9735,7 @@ static void ggml_cuda_assign_buffers_impl(struct ggml_tensor * tensor, bool scraforce_inplace;const size_t size = ggml_nbytes(tensor);- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);if (inplace && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) {ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra;char * src0_ddc = (char *) src0_extra->data_device[g_main_device];@@ -9570,11 +9790,10 @@ void ggml_cuda_assign_scratch_offset(struct ggml_tensor * tensor, size_t offset)ggml_tensor_extra_gpu * extra = ggml_cuda_alloc_temp_tensor_extra();- const bool inplace = (tensor->src[0] != nullptr && tensor->src[0]->data == tensor->data) ||- tensor->op == GGML_OP_VIEW;+ const bool inplace = tensor->view_src != nullptr;- if (inplace && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT)) {- ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->src[0]->extra;+ if (inplace && (tensor->view_src->backend == GGML_BACKEND_GPU || tensor->view_src->backend == GGML_BACKEND_GPU_SPLIT)) {+ ggml_tensor_extra_gpu * src0_extra = (ggml_tensor_extra_gpu * ) tensor->view_src->extra;char * src0_ddc = (char *) src0_extra->data_device[g_main_device];size_t view_offset = 0;if (tensor->op == GGML_OP_VIEW) {@@ -9593,7 +9812,7 @@ void ggml_cuda_copy_to_device(struct ggml_tensor * tensor) {GGML_ASSERT(ggml_is_contiguous(tensor));ggml_tensor_extra_gpu * extra = (ggml_tensor_extra_gpu *) tensor->extra;- CUDA_CHECK(ggml_cuda_set_device(g_main_device));+ ggml_cuda_set_device(g_main_device);CUDA_CHECK(cudaMemcpy(extra->data_device[g_main_device], tensor->data, ggml_nbytes(tensor), cudaMemcpyHostToDevice));}@@ -9654,14 +9873,14 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_|| (tensor->src[0] != nullptr && (tensor->src[0]->backend == GGML_BACKEND_GPU || tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT))|| (tensor->src[1] != nullptr && tensor->src[1]->backend == GGML_BACKEND_GPU);- if (!any_on_device && tensor->op != GGML_OP_MUL_MAT) {+ if (!any_on_device && tensor->op != GGML_OP_MUL_MAT && tensor->op != GGML_OP_MUL_MAT_ID) {return false;}if (tensor->op == GGML_OP_MUL_MAT) {if (tensor->src[0]->ne[3] != tensor->src[1]->ne[3]) {#ifndef NDEBUG- fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = " PRId64 ", src1->ne[3] = " PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);+ fprintf(stderr, "%s: cannot compute %s: src0->ne[3] = %" PRId64 ", src1->ne[3] = %" PRId64 " - fallback to CPU\n", __func__, tensor->name, tensor->src[0]->ne[3], tensor->src[1]->ne[3]);#endifreturn false;}@@ -9790,6 +10009,10 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_return false;}+ if (tensor->src[0] != nullptr && tensor->src[0]->backend == GGML_BACKEND_GPU_SPLIT) {+ ggml_cuda_set_peer_access(tensor->src[1]->ne[1]);+ }+if (params->ith != 0) {return true;}@@ -9863,7 +10086,7 @@ static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, gggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;if (tensor->view_src != NULL && tensor->view_offs == 0) {- assert(tensor->view_src->buffer->buft == buffer->buft); // TODO+ assert(tensor->view_src->buffer->buft == buffer->buft);tensor->backend = tensor->view_src->backend;tensor->extra = tensor->view_src->extra;return;@@ -9894,23 +10117,34 @@ static void ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer_t buffer, g}static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);- CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice));+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;- UNUSED(buffer);+ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());++ CUDA_CHECK(cudaMemcpy((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice));}static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;++ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());+CUDA_CHECK(cudaMemcpy(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost));+}- UNUSED(buffer);+static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;++ ggml_cuda_set_device(ctx->device);+ CUDA_CHECK(cudaDeviceSynchronize());++ CUDA_CHECK(cudaMemset(ctx->dev_ptr, value, buffer->size));}static struct ggml_backend_buffer_i cuda_backend_buffer_interface = {@@ -9921,6 +10155,7 @@ static struct ggml_backend_buffer_i cuda_backend_buffer_interface = {/* .get_tensor = */ ggml_backend_cuda_buffer_get_tensor,/* .cpy_tensor_from = */ NULL,/* .cpy_tensor_to = */ NULL,+ /* .clear = */ ggml_backend_cuda_buffer_clear,};// cuda buffer type@@ -9957,8 +10192,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_tif (ggml_is_quantized(tensor->type)) {if (ne0 % MATRIX_ROW_PADDING != 0) {- size += (MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING)- * ggml_type_size(tensor->type)/ggml_blck_size(tensor->type);+ size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING);}}@@ -9973,40 +10207,46 @@ static bool ggml_backend_cuda_buffer_type_supports_backend(ggml_backend_buffer_tUNUSED(buft);}-static ggml_backend_buffer_type_i cuda_backend_buffer_type_interface = {+static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = {/* .alloc_buffer = */ ggml_backend_cuda_buffer_type_alloc_buffer,/* .get_alignment = */ ggml_backend_cuda_buffer_type_get_alignment,/* .get_alloc_size = */ ggml_backend_cuda_buffer_type_get_alloc_size,/* .supports_backend = */ ggml_backend_cuda_buffer_type_supports_backend,+ /* .is_host = */ nullptr,};ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {- static struct ggml_backend_buffer_type ggml_backend_buffer_type_cuda[GGML_CUDA_MAX_DEVICES];- static bool ggml_backend_buffer_type_cuda_initialized = false;- if (!ggml_backend_buffer_type_cuda_initialized) {+ static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES];++ static bool ggml_backend_cuda_buffer_type_initialized = false;++ if (!ggml_backend_cuda_buffer_type_initialized) {for (int i = 0; i < GGML_CUDA_MAX_DEVICES; i++) {- ggml_backend_buffer_type_cuda[i] = {- /* .iface = */ cuda_backend_buffer_type_interface,+ ggml_backend_cuda_buffer_types[i] = {+ /* .iface = */ ggml_backend_cuda_buffer_type_interface,/* .context = */ (ggml_backend_buffer_type_context_t) (intptr_t) i,};}- ggml_backend_buffer_type_cuda_initialized = true;+ ggml_backend_cuda_buffer_type_initialized = true;}- return &ggml_backend_buffer_type_cuda[device];+ return &ggml_backend_cuda_buffer_types[device];}// host buffer typestatic void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {- ggml_backend_buffer_context_cuda * ctx = (ggml_backend_buffer_context_cuda *)buffer->context;- CUDA_CHECK(cudaFreeHost(ctx->dev_ptr));- delete ctx;+ ggml_cuda_host_free(buffer->context);}static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {- void * ptr;- CUDA_CHECK(cudaMallocHost(&ptr, size));+ void * ptr = ggml_cuda_host_malloc(size);++ if (ptr == nullptr) {+ // fallback to cpu buffer+ // return ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);+ GGML_ASSERT(!"ran out of gpu memory");+ }// FIXME: this is a hack to avoid having to implement a new buffer typeggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);@@ -10014,24 +10254,21 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggmbuffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer;return buffer;-- UNUSED(buft);}-struct ggml_backend_buffer_type_i cuda_backend_host_buffer_type_interface = {- /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,- /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,- /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,- /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,-};-ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {- static struct ggml_backend_buffer_type ggml_backend_buffer_type_cuda_host = {- /* .iface = */ cuda_backend_host_buffer_type_interface,+ static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_type_host = {+ /* .iface = */ {+ /* .alloc_buffer = */ ggml_backend_cuda_host_buffer_type_alloc_buffer,+ /* .get_alignment = */ ggml_backend_cpu_buffer_type()->iface.get_alignment,+ /* .get_alloc_size = */ ggml_backend_cpu_buffer_type()->iface.get_alloc_size,+ /* .supports_backend = */ ggml_backend_cpu_buffer_type()->iface.supports_backend,+ /* .is_host = */ ggml_backend_cpu_buffer_type()->iface.is_host,+ },/* .context = */ nullptr,};- return &ggml_backend_buffer_type_cuda_host;+ return &ggml_backend_cuda_buffer_type_host;}// backend@@ -10063,8 +10300,6 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, g_cudaStreams[cuda_ctx->device][0]));@@ -10074,8 +10309,6 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggmggml_backend_context_cuda * cuda_ctx = (ggml_backend_context_cuda *)backend->context;GGML_ASSERT(tensor->buffer->buft == ggml_backend_cuda_buffer_type(cuda_ctx->device) && "unsupported buffer type");- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");GGML_ASSERT(tensor->backend == GGML_BACKEND_GPU);CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, g_cudaStreams[cuda_ctx->device][0]));@@ -10241,14 +10474,19 @@ static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_ten}return false;} break;+ case GGML_OP_DUP:+ case GGML_OP_REPEAT:+ case GGML_OP_CONCAT:+ {+ ggml_type src0_type = op->src[0]->type;+ return src0_type != GGML_TYPE_I32 && src0_type != GGML_TYPE_I16;+ } break;case GGML_OP_NONE:case GGML_OP_RESHAPE:case GGML_OP_VIEW:case GGML_OP_PERMUTE:case GGML_OP_TRANSPOSE:case GGML_OP_NORM:- case GGML_OP_REPEAT:- case GGML_OP_DUP:case GGML_OP_ADD:case GGML_OP_MUL:case GGML_OP_DIV:@@ -10265,7 +10503,6 @@ static bool ggml_backend_cuda_supports_op(ggml_backend_t backend, const ggml_tencase GGML_OP_SUM_ROWS:case GGML_OP_ARGSORT:case GGML_OP_ACC:- case GGML_OP_CONCAT:case GGML_OP_GROUP_NORM:case GGML_OP_UPSCALE:case GGML_OP_PAD:@@ -10328,8 +10565,6 @@ ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data)UNUSED(params);}-extern "C" int ggml_backend_cuda_reg_devices();-static int ggml_blck_size(enum ggml_type type) {switch (type) {case GGML_TYPE_I8:diff --git a/llama.cpp/ggml-cuda.h b/llama.cpp/ggml-cuda.hindex bc3ede6a..23a67561 100644--- a/llama.cpp/ggml-cuda.h+++ b/llama.cpp/ggml-cuda.h@@ -3,13 +3,8 @@#include "ggml.h"#include "ggml-backend.h"-#ifdef GGML_USE_HIPBLAS-#define GGML_CUDA_NAME "ROCm"-#define GGML_CUBLAS_NAME "hipBLAS"-#else-#define GGML_CUDA_NAME "CUDA"-#define GGML_CUBLAS_NAME "cuBLAS"-#endif+#define GGML_CUDA_NAME "ROCm/CUDA"+#define GGML_CUBLAS_NAME "hipBLAS/cuBLAS/tinyBLAS"#ifdef __cplusplusextern "C" {diff --git a/llama.cpp/ggml-metal.h b/llama.cpp/ggml-metal.hindex 0ff777d3..db09b611 100644--- a/llama.cpp/ggml-metal.h+++ b/llama.cpp/ggml-metal.h@@ -44,6 +44,7 @@ extern "C" {struct ggml_metal_context;bool ggml_metal_supported(void);+void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);// number of command buffers to usestruct ggml_metal_context * ggml_metal_init(int n_cb);@@ -98,7 +99,10 @@ GGML_API ggml_backend_t ggml_backend_metal_init(void);GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);+GGML_API ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);+GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);+GGML_API ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);// helper to check if the device supports a specific familydiff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 40f09404..537b2512 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -324,6 +324,7 @@ struct ggml_metal_context {GGML_METAL_DECL_KERNEL(get_rows_q4_K);GGML_METAL_DECL_KERNEL(get_rows_q5_K);GGML_METAL_DECL_KERNEL(get_rows_q6_K);+ GGML_METAL_DECL_KERNEL(get_rows_i32);GGML_METAL_DECL_KERNEL(rms_norm);GGML_METAL_DECL_KERNEL(group_norm);GGML_METAL_DECL_KERNEL(norm);@@ -486,13 +487,14 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {bundle = [NSBundle bundleForClass:[GGMLMetalClass class]];#endifNSError * error = nil;- NSString * libPath = [bundle pathForResource:@"default" ofType:@"metallib"];+ NSString * libPath = [bundle pathForResource:@"ggml" ofType:@"metallib"];if (libPath != nil) {+ // pre-compiled library foundNSURL * libURL = [NSURL fileURLWithPath:libPath];GGML_METAL_LOG_INFO("%s: loading '%s'\n", __func__, [libPath UTF8String]);ctx->library = [ctx->device newLibraryWithURL:libURL error:&error];} else {- GGML_METAL_LOG_INFO("%s: default.metallib not found, loading from source\n", __func__);+ GGML_METAL_LOG_INFO("%s: ggml.metallib not found, loading from source\n", __func__);NSString * sourcePath;NSString * ggmlMetalPathResources = [[NSProcessInfo processInfo].environment objectForKey:@"GGML_METAL_PATH_RESOURCES"];@@ -520,6 +522,13 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {options = [MTLCompileOptions new];options.preprocessorMacros = @{ @"QK_K" : @(64) };#endif+ // try to disable fast-math+ // NOTE: this seems to have no effect whatsoever+ // instead, in order to disable fast-math, we have to build ggml.metallib from the command line+ // using xcrun -sdk macosx metal -fno-fast-math -c ggml-metal.metal -o ggml-metal.air+ // and go through the "pre-compiled library found" path above+ //[options setFastMathEnabled:false];+ctx->library = [ctx->device newLibraryWithSource:src options:options error:&error];}@@ -598,6 +607,7 @@ struct ggml_metal_context * ggml_metal_init(int n_cb) {GGML_METAL_ADD_KERNEL(get_rows_q4_K);GGML_METAL_ADD_KERNEL(get_rows_q5_K);GGML_METAL_ADD_KERNEL(get_rows_q6_K);+ GGML_METAL_ADD_KERNEL(get_rows_i32);GGML_METAL_ADD_KERNEL(rms_norm);GGML_METAL_ADD_KERNEL(group_norm);GGML_METAL_ADD_KERNEL(norm);@@ -720,6 +730,7 @@ void ggml_metal_free(struct ggml_metal_context * ctx) {GGML_METAL_DEL_KERNEL(get_rows_q4_K);GGML_METAL_DEL_KERNEL(get_rows_q5_K);GGML_METAL_DEL_KERNEL(get_rows_q6_K);+ GGML_METAL_DEL_KERNEL(get_rows_i32);GGML_METAL_DEL_KERNEL(rms_norm);GGML_METAL_DEL_KERNEL(group_norm);GGML_METAL_DEL_KERNEL(norm);@@ -844,12 +855,24 @@ int * ggml_metal_get_concur_list(struct ggml_metal_context * ctx) {}// temporarily defined here for compatibility between ggml-backend and the old API-struct ggml_backend_metal_buffer_context {- void * data;++struct ggml_backend_metal_buffer {+ void * data;+ size_t size;id<MTLBuffer> metal;};+struct ggml_backend_metal_buffer_context {+ void * all_data;+ size_t all_size;+ bool owned;++ // multiple buffers are used only to avoid the maximum buffer size limitation when using mmap+ int n_buffers;+ struct ggml_backend_metal_buffer buffers[GGML_METAL_MAX_BUFFERS];+};+// finds the Metal buffer that contains the tensor data on the GPU device// the assumption is that there is 1-to-1 mapping between the host and device memory buffers, so we can find the// Metal buffer based on the host memory pointer@@ -859,17 +882,29 @@ static id<MTLBuffer> ggml_metal_get_buffer(struct ggml_metal_context * ctx, struconst int64_t tsize = ggml_nbytes(t);+ ggml_backend_buffer_t buffer = t->view_src ? t->view_src->buffer : t->buffer;+// compatibility with ggml-backend- if (t->buffer && t->buffer->buft == ggml_backend_metal_buffer_type()) {- struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) t->buffer->context;+ if (buffer && buffer->buft == ggml_backend_metal_buffer_type()) {+ struct ggml_backend_metal_buffer_context * buf_ctx = (struct ggml_backend_metal_buffer_context *) buffer->context;++ // find the view that contains the tensor fully+ for (int i = 0; i < buf_ctx->n_buffers; ++i) {+ const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->buffers[i].data;- const int64_t ioffs = (int64_t) t->data - (int64_t) buf_ctx->data;+ //GGML_METAL_LOG_INFO("ioffs = %10ld, tsize = %10ld, sum = %10ld, buf_ctx->buffers[%d].size = %10ld\n", ioffs, tsize, ioffs + tsize, i, buf_ctx->buffers[i].size);+ if (ioffs >= 0 && ioffs + tsize <= (int64_t) buf_ctx->buffers[i].size) {+ *offs = (size_t) ioffs;- GGML_ASSERT(ioffs >= 0 && ioffs + tsize <= (int64_t) t->buffer->size);+ //GGML_METAL_LOG_INFO("%s: tensor '%16s', offs = %8ld\n", __func__, t->name, *offs);++ return buf_ctx->buffers[i].metal;+ }+ }- *offs = (size_t) ioffs;+ GGML_METAL_LOG_ERROR("%s: error: tensor '%s' buffer is nil\n", __func__, t->name);- return buf_ctx->metal;+ return nil;}// find the view that contains the tensor fully@@ -1242,7 +1277,7 @@ void ggml_metal_graph_compute(}if (!ggml_metal_supports_op(dst)) {- GGML_METAL_LOG_ERROR("%s: error: unsupported op '%d'\n", __func__, dst->op);+ GGML_METAL_LOG_ERROR("%s: error: unsupported op '%s'\n", __func__, ggml_op_desc(dst));GGML_ASSERT(!"unsupported op");}@@ -1435,7 +1470,7 @@ void ggml_metal_graph_compute(// not sure how to avoid this// TODO: make a simpler cpy_bytes kernel- const int nth = MIN(1024, ne00);+ const int nth = MIN((int) ctx->pipeline_cpy_f32_f32.maxTotalThreadsPerThreadgroup, ne00);[encoder setComputePipelineState:ctx->pipeline_cpy_f32_f32];[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];@@ -1490,7 +1525,7 @@ void ggml_metal_graph_compute([encoder setBytes:&pnb3 length:sizeof(pnb3) atIndex:26];[encoder setBytes:&offs length:sizeof(offs) atIndex:27];- const int nth = MIN(1024, ne0);+ const int nth = MIN((int) ctx->pipeline_add.maxTotalThreadsPerThreadgroup, ne00);[encoder dispatchThreadgroups:MTLSizeMake(ne11, ne12, ne13) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;@@ -1498,7 +1533,7 @@ void ggml_metal_graph_compute({GGML_ASSERT(ggml_is_contiguous(src0));- const float scale = *(const float *) src1->data;+ const float scale = *(const float *) dst->op_params;int64_t n = ggml_nelements(dst);@@ -1509,8 +1544,8 @@ void ggml_metal_graph_compute([encoder setComputePipelineState:ctx->pipeline_scale];}- [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];- [encoder setBuffer:id_dst offset:offs_dst atIndex:1];+ [encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];+ [encoder setBuffer:id_dst offset:offs_dst atIndex:1];[encoder setBytes:&scale length:sizeof(scale) atIndex:2];[encoder dispatchThreadgroups:MTLSizeMake(n, 1, 1) threadsPerThreadgroup:MTLSizeMake(1, 1, 1)];@@ -1854,6 +1889,10 @@ void ggml_metal_graph_compute(}};+ if (ggml_is_quantized(src0t)) {+ GGML_ASSERT(ne00 >= nth0*nth1);+ }+[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];[encoder setBuffer:id_dst offset:offs_dst atIndex:2];@@ -1912,6 +1951,9 @@ void ggml_metal_graph_compute(// TODO: make this more generalGGML_ASSERT(n_as <= 8);+ // max size of the src1ids array in the kernel stack+ GGML_ASSERT(ne11 <= 512);+struct ggml_tensor * src2 = gf->nodes[i]->src[2];const int64_t ne20 = src2 ? src2->ne[0] : 0;@@ -1929,9 +1971,6 @@ void ggml_metal_graph_compute(GGML_ASSERT(!ggml_is_transposed(src2));GGML_ASSERT(!ggml_is_transposed(src1));- GGML_ASSERT(ne20 % 32 == 0);- // !!!!!!!!! TODO: this assert is probably required but not sure!- //GGML_ASSERT(ne20 >= 64);GGML_ASSERT(src1t == GGML_TYPE_F32);const uint r2 = ne12/ne22;@@ -1939,22 +1978,22 @@ void ggml_metal_graph_compute(// find the break-even point where the matrix-matrix kernel becomes more efficient compared// to the matrix-vector kernel- int ne11_mm_min = 1;+ int ne11_mm_min = n_as;const int idx = ((int32_t *) dst->op_params)[0];// batch sizeGGML_ASSERT(ne01 == ne11);- const int64_t _ne1 = 1; // kernel_mul_mm_impl needs a reference in constant memory-// for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs// AMD GPU and older A-chips will reuse matrix-vector multiplication kernel// !!!// TODO: for now, always use mat-vec kernels until we figure out how to improve the// indirect matrix multiplication// !!!- if ([ctx->device supportsFamily:MTLGPUFamilyApple7] && _ne1 > ne11_mm_min) {+ if ([ctx->device supportsFamily:MTLGPUFamilyApple7] &&+ ne20 % 32 == 0 && ne20 >= 64 &&+ ne11 > ne11_mm_min) {switch (src2->type) {case GGML_TYPE_F32: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f32_f32]; break;case GGML_TYPE_F16: [encoder setComputePipelineState:ctx->pipeline_mul_mm_id_f16_f32]; break;@@ -1984,14 +2023,15 @@ void ggml_metal_graph_compute([encoder setBytes:&nb11 length:sizeof(nb11) atIndex:11];[encoder setBytes:&nb12 length:sizeof(nb12) atIndex:12];[encoder setBytes:&ne0 length:sizeof(ne0) atIndex:13];- [encoder setBytes:&_ne1 length:sizeof(_ne1) atIndex:14];+ [encoder setBytes:&ne1 length:sizeof(ne1) atIndex:14];[encoder setBytes:&nb1 length:sizeof(nb1) atIndex:15];[encoder setBytes:&r2 length:sizeof(r2) atIndex:16];[encoder setBytes:&r3 length:sizeof(r3) atIndex:17];[encoder setBytes:&idx length:sizeof(idx) atIndex:18];// TODO: how to make this an array? read Metal docs- for (int j = 0; j < n_as; ++j) {- struct ggml_tensor * src_cur = dst->src[2 + j];+ for (int j = 0; j < 8; ++j) {+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];size_t offs_src_cur = 0;id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);@@ -2001,8 +2041,7 @@ void ggml_metal_graph_compute([encoder setThreadgroupMemoryLength:8192 atIndex:0];- // TODO: processing one row at a time (ne11 -> 1) is not efficient- [encoder dispatchThreadgroups:MTLSizeMake( (_ne1 + 31)/32, (ne21 + 63)/64, ne01*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];+ [encoder dispatchThreadgroups:MTLSizeMake((ne11 + 31)/32, (ne21 + 63)/64, n_as*ne12*ne13) threadsPerThreadgroup:MTLSizeMake(128, 1, 1)];} else {int nth0 = 32;int nth1 = 1;@@ -2085,11 +2124,17 @@ void ggml_metal_graph_compute(} break;default:{- GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src0t);+ GGML_METAL_LOG_ERROR("Asserting on type %d\n", (int)src2t);GGML_ASSERT(false && "not implemented");}};+ if (ggml_is_quantized(src2t)) {+ GGML_ASSERT(ne20 >= nth0*nth1);+ }++ const int64_t _ne1 = 1; // kernels needs a reference in constant memory+[encoder setBuffer:id_src0 offset:offs_src0 atIndex:0];[encoder setBuffer:id_src1 offset:offs_src1 atIndex:1];[encoder setBuffer:id_dst offset:offs_dst atIndex:2];@@ -2114,8 +2159,9 @@ void ggml_metal_graph_compute([encoder setBytes:&r3 length:sizeof(r3) atIndex:21];[encoder setBytes:&idx length:sizeof(idx) atIndex:22];// TODO: how to make this an array? read Metal docs- for (int j = 0; j < n_as; ++j) {- struct ggml_tensor * src_cur = dst->src[2 + j];+ for (int j = 0; j < 8; ++j) {+ // NOTE: this is done like this to avoid uninitialized kernel arguments when n_as < 8+ struct ggml_tensor * src_cur = dst->src[2 + (j % n_as)];size_t offs_src_cur = 0;id<MTLBuffer> id_src_cur = ggml_metal_get_buffer(ctx, src_cur, &offs_src_cur);@@ -2164,6 +2210,7 @@ void ggml_metal_graph_compute(case GGML_TYPE_Q4_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q4_K]; break;case GGML_TYPE_Q5_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q5_K]; break;case GGML_TYPE_Q6_K: [encoder setComputePipelineState:ctx->pipeline_get_rows_q6_K]; break;+ case GGML_TYPE_I32: [encoder setComputePipelineState:ctx->pipeline_get_rows_i32]; break;default: GGML_ASSERT(false && "not implemented");}@@ -2434,7 +2481,7 @@ void ggml_metal_graph_compute([encoder setBytes:&nb3 length:sizeof(nb3) atIndex:17];[encoder setBytes:&sf length:sizeof(sf) atIndex:18];- const int nth = MIN(1024, ne0);+ const int nth = MIN((int) ctx->pipeline_upscale_f32.maxTotalThreadsPerThreadgroup, ne0);[encoder dispatchThreadgroups:MTLSizeMake(ne1, ne2, ne3) threadsPerThreadgroup:MTLSizeMake(nth, 1, 1)];} break;@@ -2598,6 +2645,7 @@ void ggml_metal_graph_compute(// backend interface+// default bufferstatic id<MTLDevice> g_backend_device = nil;static int g_backend_device_ref_count = 0;@@ -2625,34 +2673,31 @@ static void ggml_backend_metal_free_device(void) {static void * ggml_backend_metal_buffer_get_base(ggml_backend_buffer_t buffer) {struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;- return ctx->data;+ return ctx->all_data;}static void ggml_backend_metal_buffer_free_buffer(ggml_backend_buffer_t buffer) {struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;- [ctx->metal release];+ for (int i = 0; i < ctx->n_buffers; i++) {+ [ctx->buffers[i].metal release];+ }ggml_backend_metal_free_device();- free(ctx->data);- free(ctx);+ if (ctx->owned) {+ free(ctx->all_data);+ }- UNUSED(buffer);+ free(ctx);}static void ggml_backend_metal_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor write out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");-memcpy((char *)tensor->data + offset, data, size);UNUSED(buffer);}static void ggml_backend_metal_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {- GGML_ASSERT(offset + size <= ggml_nbytes(tensor) && "tensor read out of bounds");- GGML_ASSERT(tensor->data != NULL && "tensor not allocated");-memcpy(data, (const char *)tensor->data + offset, size);UNUSED(buffer);@@ -2670,7 +2715,13 @@ static void ggml_backend_metal_buffer_cpy_tensor_to(ggml_backend_buffer_t bufferUNUSED(buffer);}-static struct ggml_backend_buffer_i metal_backend_buffer_i = {+static void ggml_backend_metal_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ struct ggml_backend_metal_buffer_context * ctx = (struct ggml_backend_metal_buffer_context *)buffer->context;++ memset(ctx->all_data, value, ctx->all_size);+}++static struct ggml_backend_buffer_i ggml_backend_metal_buffer_i = {/* .free_buffer = */ ggml_backend_metal_buffer_free_buffer,/* .get_base = */ ggml_backend_metal_buffer_get_base,/* .init_tensor = */ NULL,@@ -2678,8 +2729,11 @@ static struct ggml_backend_buffer_i metal_backend_buffer_i = {/* .get_tensor = */ ggml_backend_metal_buffer_get_tensor,/* .cpy_tensor_from = */ ggml_backend_metal_buffer_cpy_tensor_from,/* .cpy_tensor_to = */ ggml_backend_metal_buffer_cpy_tensor_to,+ /* .clear = */ ggml_backend_metal_buffer_clear,};+// default buffer type+static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));@@ -2690,13 +2744,46 @@ static ggml_backend_buffer_t ggml_backend_metal_buffer_type_alloc_buffer(ggml_basize_aligned += (size_page - (size_aligned % size_page));}- ctx->data = ggml_metal_host_malloc(size);- ctx->metal = [ggml_backend_metal_get_device() newBufferWithBytesNoCopy:ctx->data+ id<MTLDevice> device = ggml_backend_metal_get_device();++ ctx->all_data = ggml_metal_host_malloc(size_aligned);+ ctx->all_size = size_aligned;+ ctx->owned = true;+ ctx->n_buffers = 1;++ ctx->buffers[0].data = ctx->all_data;+ ctx->buffers[0].size = size;+ ctx->buffers[0].metal = [device newBufferWithBytesNoCopy:ctx->all_datalength:size_alignedoptions:MTLResourceStorageModeShareddeallocator:nil];- return ggml_backend_buffer_init(buft, metal_backend_buffer_i, ctx, size);+ if (ctx->buffers[0].metal == nil) {+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);+ free(ctx);+ ggml_backend_metal_free_device();+ return NULL;+ }++ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);+++#if TARGET_OS_OSX+ GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",+ device.currentAllocatedSize / 1024.0 / 1024.0,+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);++ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);+ } else {+ GGML_METAL_LOG_INFO("\n");+ }+#else+ GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);+#endif+++ return ggml_backend_buffer_init(buft, ggml_backend_metal_buffer_i, ctx, size);}static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {@@ -2707,7 +2794,13 @@ static size_t ggml_backend_metal_buffer_type_get_alignment(ggml_backend_buffer_tstatic bool ggml_backend_metal_buffer_type_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend) {return ggml_backend_is_metal(backend) || ggml_backend_is_cpu(backend);- GGML_UNUSED(buft);+ UNUSED(buft);+}++static bool ggml_backend_metal_buffer_type_is_host(ggml_backend_buffer_type_t buft) {+ return true;++ UNUSED(buft);}ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {@@ -2717,6 +2810,7 @@ ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {/* .get_alignment = */ ggml_backend_metal_buffer_type_get_alignment,/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes/* .supports_backend = */ ggml_backend_metal_buffer_type_supports_backend,+ /* .is_host = */ ggml_backend_metal_buffer_type_is_host,},/* .context = */ NULL,};@@ -2724,6 +2818,87 @@ ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {return &ggml_backend_buffer_type_metal;}+// buffer from ptr++ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {+ struct ggml_backend_metal_buffer_context * ctx = malloc(sizeof(struct ggml_backend_metal_buffer_context));++ ctx->all_data = data;+ ctx->all_size = size;+ ctx->owned = false;+ ctx->n_buffers = 0;++ const size_t size_page = sysconf(_SC_PAGESIZE);+ size_t size_aligned = size;+ if ((size_aligned % size_page) != 0) {+ size_aligned += (size_page - (size_aligned % size_page));+ }++ id<MTLDevice> device = ggml_backend_metal_get_device();++ // the buffer fits into the max buffer size allowed by the device+ if (size_aligned <= device.maxBufferLength) {+ ctx->buffers[ctx->n_buffers].data = data;+ ctx->buffers[ctx->n_buffers].size = size;++ ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:data length:size_aligned options:MTLResourceStorageModeShared deallocator:nil];++ if (ctx->buffers[ctx->n_buffers].metal == nil) {+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_aligned / 1024.0 / 1024.0);+ return false;+ }++ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB", __func__, size_aligned / 1024.0 / 1024.0);++ ++ctx->n_buffers;+ } else {+ // this overlap between the views will guarantee that the tensor with the maximum size will fully fit into+ // one of the views+ const size_t size_ovlp = ((max_size + size_page - 1) / size_page + 1) * size_page; // round-up 2 pages just in case+ const size_t size_step = device.maxBufferLength - size_ovlp;+ const size_t size_view = device.maxBufferLength;++ for (size_t i = 0; i < size; i += size_step) {+ const size_t size_step_aligned = (i + size_view <= size) ? size_view : (size_aligned - i);++ ctx->buffers[ctx->n_buffers].data = (void *) ((uint8_t *) data + i);+ ctx->buffers[ctx->n_buffers].size = size_step_aligned;++ ctx->buffers[ctx->n_buffers].metal = [device newBufferWithBytesNoCopy:(void *) ((uint8_t *) data + i) length:size_step_aligned options:MTLResourceStorageModeShared deallocator:nil];++ if (ctx->buffers[ctx->n_buffers].metal == nil) {+ GGML_METAL_LOG_ERROR("%s: error: failed to allocate buffer, size = %8.2f MiB\n", __func__, size_step_aligned / 1024.0 / 1024.0);+ return false;+ }++ GGML_METAL_LOG_INFO("%s: allocated buffer, size = %8.2f MiB, offs = %12ld", __func__, size_step_aligned / 1024.0 / 1024.0, i);+ if (i + size_step < size) {+ GGML_METAL_LOG_INFO("\n");+ }++ ++ctx->n_buffers;+ }+ }++#if TARGET_OS_OSX+ GGML_METAL_LOG_INFO(", (%8.2f / %8.2f)",+ device.currentAllocatedSize / 1024.0 / 1024.0,+ device.recommendedMaxWorkingSetSize / 1024.0 / 1024.0);++ if (device.currentAllocatedSize > device.recommendedMaxWorkingSetSize) {+ GGML_METAL_LOG_WARN("%s: warning: current allocated size is greater than the recommended max working set size\n", __func__);+ } else {+ GGML_METAL_LOG_INFO("\n");+ }+#else+ GGML_METAL_LOG_INFO(", (%8.2f)\n", device.currentAllocatedSize / 1024.0 / 1024.0);+#endif++ return ggml_backend_buffer_init(ggml_backend_metal_buffer_type(), ggml_backend_metal_buffer_i, ctx, size);+}++// backend+static const char * ggml_backend_metal_name(ggml_backend_t backend) {return "Metal";@@ -2736,10 +2911,6 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {free(backend);}-static void ggml_backend_metal_synchronize(ggml_backend_t backend) {- UNUSED(backend);-}-static ggml_backend_buffer_type_t ggml_backend_metal_get_default_buffer_type(ggml_backend_t backend) {return ggml_backend_metal_buffer_type();@@ -2766,25 +2937,15 @@ static struct ggml_backend_i metal_backend_i = {/* .get_tensor_async = */ NULL,/* .cpy_tensor_from_async = */ NULL,/* .cpy_tensor_to_async = */ NULL,- /* .synchronize = */ ggml_backend_metal_synchronize,- /* .graph_plan_create = */ NULL, // the metal implementation does not require creating graph plans atm+ /* .synchronize = */ NULL,+ /* .graph_plan_create = */ NULL,/* .graph_plan_free = */ NULL,/* .graph_plan_compute = */ NULL,/* .graph_compute = */ ggml_backend_metal_graph_compute,/* .supports_op = */ ggml_backend_metal_supports_op,};-// TODO: make a common log callback for all backends in ggml-backend-static void ggml_backend_log_callback(enum ggml_log_level level, const char * msg, void * user_data) {- fprintf(stderr, "%s", msg);-- UNUSED(level);- UNUSED(user_data);-}-ggml_backend_t ggml_backend_metal_init(void) {- ggml_metal_log_set_callback(ggml_backend_log_callback, NULL);-struct ggml_metal_context * ctx = ggml_metal_init(GGML_DEFAULT_N_THREADS);if (ctx == NULL) {diff --git a/llama.cpp/ggml-metal.metal b/llama.cpp/ggml-metal.metalindex d5b54e11..a7d3f9ef 100644--- a/llama.cpp/ggml-metal.metal+++ b/llama.cpp/ggml-metal.metal@@ -59,26 +59,26 @@ kernel void kernel_add(constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant int64_t & nb00,- constant int64_t & nb01,- constant int64_t & nb02,- constant int64_t & nb03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,- constant int64_t & nb13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,constant int64_t & ne3,- constant int64_t & nb0,- constant int64_t & nb1,- constant int64_t & nb2,- constant int64_t & nb3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,constant int64_t & offs,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],@@ -109,26 +109,26 @@ kernel void kernel_mul(constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant int64_t & nb00,- constant int64_t & nb01,- constant int64_t & nb02,- constant int64_t & nb03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,- constant int64_t & nb13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,constant int64_t & ne3,- constant int64_t & nb0,- constant int64_t & nb1,- constant int64_t & nb2,- constant int64_t & nb3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],uint3 ntg[[threads_per_threadgroup]]) {@@ -158,26 +158,26 @@ kernel void kernel_div(constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant int64_t & nb00,- constant int64_t & nb01,- constant int64_t & nb02,- constant int64_t & nb03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,- constant int64_t & nb13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,constant int64_t & ne3,- constant int64_t & nb0,- constant int64_t & nb1,- constant int64_t & nb2,- constant int64_t & nb3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,uint3 tgpig[[threadgroup_position_in_grid]],uint3 tpitg[[thread_position_in_threadgroup]],uint3 ntg[[threads_per_threadgroup]]) {@@ -205,7 +205,7 @@ kernel void kernel_add_row(device const float4 * src0,device const float4 * src1,device float4 * dst,- constant int64_t & nb [[buffer(28)]],+ constant uint64_t & nb [[buffer(28)]],uint tpig[[thread_position_in_grid]]) {dst[tpig] = src0[tpig] + src1[tpig % nb];}@@ -214,7 +214,7 @@ kernel void kernel_mul_row(device const float4 * src0,device const float4 * src1,device float4 * dst,- constant int64_t & nb [[buffer(28)]],+ constant uint64_t & nb [[buffer(28)]],uint tpig[[thread_position_in_grid]]) {dst[tpig] = src0[tpig] * src1[tpig % nb];}@@ -223,7 +223,7 @@ kernel void kernel_div_row(device const float4 * src0,device const float4 * src1,device float4 * dst,- constant int64_t & nb [[buffer(28)]],+ constant uint64_t & nb [[buffer(28)]],uint tpig[[thread_position_in_grid]]) {dst[tpig] = src0[tpig] / src1[tpig % nb];}@@ -307,26 +307,26 @@ kernel void kernel_sum_rows(constant int64_t & ne01,constant int64_t & ne02,constant int64_t & ne03,- constant int64_t & nb00,- constant int64_t & nb01,- constant int64_t & nb02,- constant int64_t & nb03,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant uint64_t & nb03,constant int64_t & ne10,constant int64_t & ne11,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,- constant int64_t & nb13,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant uint64_t & nb13,constant int64_t & ne0,constant int64_t & ne1,constant int64_t & ne2,constant int64_t & ne3,- constant int64_t & nb0,- constant int64_t & nb1,- constant int64_t & nb2,- constant int64_t & nb3,+ constant uint64_t & nb0,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ constant uint64_t & nb3,uint3 tpig[[thread_position_in_grid]]) {int64_t i3 = tpig.z;int64_t i2 = tpig.y;@@ -846,7 +846,7 @@ inline float block_q_n_dot_y(device const block_q5_1 * qb_curr, float sumy, thre#define N_SIMDGROUP 2 // number of SIMD groups in a thread group//Note: This is a template, but strictly speaking it only applies to// quantizations where the block size is 32. It also does not-// giard against the number of rows not being divisible by+// guard against the number of rows not being divisible by// N_DST, so this is another explicit assumption of the implementation.template<typename block_q_type, int nr, int nsg, int nw>void mul_vec_q_n_f32_impl(@@ -920,14 +920,21 @@ kernel void kernel_mul_mv_q4_0_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -939,14 +946,21 @@ kernel void kernel_mul_mv_q4_1_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -958,14 +972,21 @@ kernel void kernel_mul_mv_q5_0_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -977,14 +998,21 @@ kernel void kernel_mul_mv_q5_1_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -1071,12 +1099,19 @@ kernel void kernel_mul_mv_q8_0_f32(constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne10,+ constant int64_t & ne11,constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -1182,8 +1217,8 @@ kernel void kernel_mul_mv_f32_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {kernel_mul_mv_f32_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);@@ -1209,8 +1244,8 @@ kernel void kernel_mul_mv_f16_f16(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {@@ -1346,8 +1381,8 @@ kernel void kernel_mul_mv_f16_f32_1row(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {kernel_mul_mv_f16_f32_1row_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);@@ -1452,8 +1487,8 @@ kernel void kernel_mul_mv_f16_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {kernel_mul_mv_f16_f32_impl(src0, src1, dst, ne00, ne01, ne02, nb00, nb01, nb02, ne10, ne11, ne12, nb10, nb11, nb12, ne0, ne1, r2, r3, tgpig, tiisg);@@ -1478,8 +1513,8 @@ kernel void kernel_mul_mv_f16_f32_l4(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]]) {@@ -1543,7 +1578,8 @@ kernel void kernel_alibi_f32(const int64_t i3 = n / (ne2*ne1*ne0);const int64_t i2 = (n - i3*ne2*ne1*ne0) / (ne1*ne0);const int64_t i1 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0) / ne0;- const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);+ //const int64_t i0 = (n - i3*ne2*ne1*ne0 - i2*ne1*ne0 - i1*ne0);+const int64_t k = i3*ne3 + i2;float m_k;@@ -2410,22 +2446,6 @@ typedef struct {} block_q6_K;// 210 bytes / block-static inline uchar4 get_scale_min_k4(int j, device const uint8_t * q) {- uchar4 r;- if (j < 4) {- r[0] = q[j+0] & 63;- r[2] = q[j+1] & 63;- r[1] = q[j+4] & 63;- r[3] = q[j+5] & 63;- } else {- r[0] = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4);- r[2] = (q[j+5] & 0xF) | ((q[j-3] >> 6) << 4);- r[1] = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4);- r[3] = (q[j+5] >> 4) | ((q[j+1] >> 6) << 4);- }- return r;-}-//====================================== dot products =========================void kernel_mul_mv_q2_K_f32_impl(@@ -2584,14 +2604,21 @@ kernel void kernel_mul_mv_q2_K_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -2841,14 +2868,21 @@ kernel void kernel_mul_mv_q3_K_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -2984,8 +3018,8 @@ void kernel_mul_mv_q4_K_f32_impl(constant uint & r2,constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],- uint tiisg[[thread_index_in_simdgroup]],- uint sgitg[[simdgroup_index_in_threadgroup]]) {+ uint tiisg[[thread_index_in_simdgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {const int ix = tiisg/4; // 0...7const int it = tiisg%4; // 0...3@@ -2994,7 +3028,7 @@ void kernel_mul_mv_q4_K_f32_impl(const int r0 = tgpig.x;const int r1 = tgpig.y;const int im = tgpig.z;- const int first_row = (r0 * N_SIMDGROUP + sgitg) * N_DST;+ const int first_row = r0 * N_DST;const int ib_row = first_row * nb;const uint i12 = im%ne12;@@ -3060,7 +3094,7 @@ void kernel_mul_mv_q4_K_f32_impl(for (int row = 0; row < N_DST; ++row) {all_sum = simd_sum(sumf[row]);if (tiisg == 0) {- dst[r1*ne0+ im*ne0*ne1 + first_row + row] = all_sum;+ dst[r1*ne0 + im*ne0*ne1 + first_row + row] = all_sum;}}}@@ -3072,14 +3106,21 @@ kernel void kernel_mul_mv_q4_K_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -3271,14 +3312,21 @@ kernel void kernel_mul_mv_q5_K_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -3398,14 +3446,21 @@ kernel void kernel_mul_mv_q6_K_f32(device const float * src1,device float * dst,constant int64_t & ne00,- constant int64_t & ne01[[buffer(4)]],- constant int64_t & ne02[[buffer(5)]],- constant int64_t & ne10[[buffer(9)]],- constant int64_t & ne12[[buffer(11)]],- constant int64_t & ne0 [[buffer(15)]],- constant int64_t & ne1 [[buffer(16)]],- constant uint & r2 [[buffer(17)]],- constant uint & r3 [[buffer(18)]],+ constant int64_t & ne01,+ constant int64_t & ne02,+ constant uint64_t & nb00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant int64_t & ne11,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ constant int64_t & ne1,+ constant uint & r2,+ constant uint & r3,uint3 tgpig[[threadgroup_position_in_grid]],uint tiisg[[thread_index_in_simdgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {@@ -3523,7 +3578,7 @@ void dequantize_q8_0(device const block_q8_0 *xb, short il, thread type4x4 & regdevice const int8_t * qs = ((device const int8_t *)xb->qs);const half d = xb->d;- for (int i=0;i<16;i++) {+ for (int i = 0; i < 16; i++) {reg[i/4][i%4] = (qs[i + 16*il] * d);}}@@ -3774,6 +3829,35 @@ kernel void kernel_get_rows_f16(}}+kernel void kernel_get_rows_i32(+ device const void * src0,+ device const char * src1,+ device int32_t * dst,+ constant int64_t & ne00,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne10,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb1,+ constant uint64_t & nb2,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint3 tptg [[threads_per_threadgroup]]) {+ const int64_t i10 = tgpig.x;+ const int64_t i11 = tgpig.y;++ const int64_t r = ((device int32_t *) ((device char *) src1 + i11*nb11 + i10*nb10))[0];++ const int64_t i02 = i11;++ for (int ind = tiitg; ind < ne00; ind += tptg.x) {+ ((device int32_t *) ((device char *) dst + i11*nb2 + i10*nb1))[ind] =+ ((device int32_t *) ((device char *) src0 + r*nb01 + i02*nb02))[ind];+ }+}++#define BLOCK_SIZE_M 64 // 8 simdgroup matrices from matrix A#define BLOCK_SIZE_N 32 // 4 simdgroup matrices from matrix B#define BLOCK_SIZE_K 32@@ -3792,12 +3876,12 @@ void kernel_mul_mm_impl(device const uchar * src0,device float * dst,constant int64_t & ne00,constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne12,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,constant uint & r2,@@ -3918,18 +4002,143 @@ void kernel_mul_mm_impl(device const uchar * src0,}}+// same as kernel_mul_mm_impl, but src1 and dst are accessed via indices stored in src1ids+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>+void kernel_mul_mm_id_impl(+ device const uchar * src0,+ device const uchar * src1,+ thread short * src1ids,+ device float * dst,+ constant int64_t & ne00,+ constant int64_t & ne02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,+ constant int64_t & ne12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,+ constant int64_t & ne0,+ int64_t ne1,+ constant uint & r2,+ constant uint & r3,+ threadgroup uchar * shared_memory,+ uint3 tgpig[[threadgroup_position_in_grid]],+ uint tiitg[[thread_index_in_threadgroup]],+ uint sgitg[[simdgroup_index_in_threadgroup]]) {++ threadgroup half * sa = (threadgroup half *)(shared_memory);+ threadgroup float * sb = (threadgroup float *)(shared_memory + 4096);++ const uint r0 = tgpig.y;+ const uint r1 = tgpig.x;+ const uint im = tgpig.z;++ if (r1 * BLOCK_SIZE_N >= ne1) return;++ // if this block is of 64x32 shape or smaller+ short n_rows = (ne0 - r0 * BLOCK_SIZE_M < BLOCK_SIZE_M) ? (ne0 - r0 * BLOCK_SIZE_M) : BLOCK_SIZE_M;+ short n_cols = (ne1 - r1 * BLOCK_SIZE_N < BLOCK_SIZE_N) ? (ne1 - r1 * BLOCK_SIZE_N) : BLOCK_SIZE_N;++ // a thread shouldn't load data outside of the matrix+ short thread_row = ((short)tiitg/THREAD_PER_ROW) < n_rows ? ((short)tiitg/THREAD_PER_ROW) : n_rows - 1;+ short thread_col = ((short)tiitg/THREAD_PER_COL) < n_cols ? ((short)tiitg/THREAD_PER_COL) : n_cols - 1;++ simdgroup_half8x8 ma[4];+ simdgroup_float8x8 mb[2];+ simdgroup_float8x8 c_res[8];+ for (int i = 0; i < 8; i++){+ c_res[i] = make_filled_simdgroup_matrix<float, 8>(0.f);+ }++ short il = (tiitg % THREAD_PER_ROW);++ const uint i12 = im%ne12;+ const uint i13 = im/ne12;++ uint offset0 = (i12/r2)*nb02 + (i13/r3)*(nb02*ne02);+ ushort offset1 = il/nl;++ device const block_q * x = (device const block_q *)(src0 + (r0 * BLOCK_SIZE_M + thread_row) * nb01 + offset0) + offset1;+ device const float * y = (device const float *)(src1+ + nb12 * im+ + nb11 * src1ids[r1 * BLOCK_SIZE_N + thread_col]+ + nb10 * (BLOCK_SIZE_K / THREAD_PER_COL * (tiitg % THREAD_PER_COL)));++ for (int loop_k = 0; loop_k < ne00; loop_k += BLOCK_SIZE_K) {+ // load data and store to threadgroup memory+ half4x4 temp_a;+ dequantize_func(x, il, temp_a);+ threadgroup_barrier(mem_flags::mem_threadgroup);++ for (int i = 0; i < 16; i++) {+ *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \+ + (tiitg % THREAD_PER_ROW) * 16 + (i / 8) * 8) \+ + (tiitg / THREAD_PER_ROW) % 8 + (i & 7) * 8) = temp_a[i/4][i%4];+ }++ *(threadgroup float2x4 *)(sb + (tiitg % THREAD_PER_COL) * 8 * 32 + 8 * (tiitg / THREAD_PER_COL)) = *((device float2x4 *)y);++ il = (il + 2 < nl) ? il + 2 : il % 2;+ x = (il < 2) ? x + (2+nl-1)/nl : x;+ y += BLOCK_SIZE_K;++ threadgroup_barrier(mem_flags::mem_threadgroup);++ // load matrices from threadgroup memory and conduct outer products+ threadgroup half * lsma = (sa + THREAD_MAT_M * SG_MAT_SIZE * (sgitg % 2));+ threadgroup float * lsmb = (sb + THREAD_MAT_N * SG_MAT_SIZE * (sgitg / 2));++ for (int ik = 0; ik < BLOCK_SIZE_K / 8; ik++) {+ for (int i = 0; i < 4; i++) {+ simdgroup_load(ma[i],lsma + SG_MAT_SIZE * i);+ }+ simdgroup_barrier(mem_flags::mem_none);+ for (int i = 0; i < 2; i++) {+ simdgroup_load(mb[i],lsmb + SG_MAT_SIZE * i);+ }++ lsma += BLOCK_SIZE_M / SG_MAT_ROW * SG_MAT_SIZE;+ lsmb += BLOCK_SIZE_N / SG_MAT_ROW * SG_MAT_SIZE;++ for (int i = 0; i < 8; i++){+ simdgroup_multiply_accumulate(c_res[i], mb[i/4], ma[i%4], c_res[i]);+ }+ }+ }++ {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ threadgroup float * temp_str = ((threadgroup float *)shared_memory) \+ + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;+ for (int i = 0; i < 8; i++) {+ simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);+ }++ threadgroup_barrier(mem_flags::mem_threadgroup);++ device float * C = dst + (BLOCK_SIZE_M * r0) + im*ne1*ne0;+ if (sgitg == 0) {+ for (int i = 0; i < n_rows; i++) {+ for (int j = tiitg; j < n_cols; j += BLOCK_SIZE_N) {+ *(C + i + src1ids[j + r1*BLOCK_SIZE_N] * ne0) = *(temp_str + i + j * BLOCK_SIZE_M);+ }+ }+ }+ }+}+template<typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread half4x4 &)>kernel void kernel_mul_mm(device const uchar * src0,device const uchar * src1,device float * dst,constant int64_t & ne00,constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne12,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,constant uint & r2,@@ -3964,20 +4173,20 @@ template<typename block_q, short nl, void (*dequantize_func)(device const block_kernel void kernel_mul_mm_id(device const uchar * ids,device const uchar * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -3993,18 +4202,28 @@ kernel void kernel_mul_mm_id(uint3 tgpig[[threadgroup_position_in_grid]],uint tiitg[[thread_index_in_threadgroup]],uint sgitg[[simdgroup_index_in_threadgroup]]) {- device const uchar * src0[8] = {src00, src01, src02, src03, src04, src05, src06, src07};+ device const uchar * src0s[8] = {src00, src01, src02, src03, src04, src05, src06, src07};- const int64_t bid = tgpig.z/(ne12*ne13);+ // expert id+ const int32_t id = tgpig.z/(ne12*ne13);tgpig.z = tgpig.z%(ne12*ne13);- const int32_t id = ((device int32_t *) (ids + bid*nbi1))[idx];+ // row indices of src1 for expert id+ int64_t _ne1 = 0;+ short src1ids[512];- kernel_mul_mm_impl<block_q, nl, dequantize_func>(- src0[id],- src1 + bid*nb11,- (device float *) (dst + bid*nb1),+ for (int64_t i1 = 0; i1 < ne1; i1++) {+ if (((device int32_t *) (ids + i1*nbi1))[idx] == id) {+ src1ids[_ne1++] = i1;+ }+ }++ kernel_mul_mm_id_impl<block_q, nl, dequantize_func>(+ src0s[id],+ src1,+ src1ids,+ dst,ne00,ne02,nb01,@@ -4014,7 +4233,7 @@ kernel void kernel_mul_mm_id(nb11,nb12,ne0,- ne1,+ _ne1,r2,r3,shared_memory,@@ -4070,12 +4289,12 @@ typedef void (mat_mm_t)(device float * dst,constant int64_t & ne00,constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne12,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,constant uint & r2,@@ -4103,20 +4322,20 @@ template [[host_name("kernel_mul_mm_q6_K_f32")]] kernel mat_mm_t kernel_mul_mm<btypedef void (mat_mm_id_t)(device const uchar * ids,device const uchar * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne02,- constant int64_t & nb01,- constant int64_t & nb02,+ constant uint64_t & nb01,+ constant uint64_t & nb02,constant int64_t & ne12,constant int64_t & ne13,- constant int64_t & nb10,- constant int64_t & nb11,- constant int64_t & nb12,+ constant uint64_t & nb10,+ constant uint64_t & nb11,+ constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4152,8 +4371,8 @@ template [[host_name("kernel_mul_mm_id_q6_K_f32")]] kernel mat_mm_id_t kernel_mukernel void kernel_mul_mv_id_f32_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4169,7 +4388,7 @@ kernel void kernel_mul_mv_id_f32_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4196,7 +4415,7 @@ kernel void kernel_mul_mv_id_f32_f32(kernel_mul_mv_f32_f32_impl(src0[id],src1 + bid*nb11,- (device float *) (dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4221,8 +4440,8 @@ kernel void kernel_mul_mv_id_f32_f32(kernel void kernel_mul_mv_id_f16_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4238,7 +4457,7 @@ kernel void kernel_mul_mv_id_f16_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4265,7 +4484,7 @@ kernel void kernel_mul_mv_id_f16_f32(kernel_mul_mv_f16_f32_impl(src0[id],src1 + bid*nb11,- (device float *) (dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4290,8 +4509,8 @@ kernel void kernel_mul_mv_id_f16_f32(kernel void kernel_mul_mv_id_q8_0_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4307,7 +4526,7 @@ kernel void kernel_mul_mv_id_q8_0_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4334,7 +4553,7 @@ kernel void kernel_mul_mv_id_q8_0_f32(kernel_mul_mv_q8_0_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4353,8 +4572,8 @@ kernel void kernel_mul_mv_id_q8_0_f32(kernel void kernel_mul_mv_id_q4_0_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4370,7 +4589,7 @@ kernel void kernel_mul_mv_id_q4_0_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4397,7 +4616,7 @@ kernel void kernel_mul_mv_id_q4_0_f32(mul_vec_q_n_f32_impl<block_q4_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4416,8 +4635,8 @@ kernel void kernel_mul_mv_id_q4_0_f32(kernel void kernel_mul_mv_id_q4_1_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4433,7 +4652,7 @@ kernel void kernel_mul_mv_id_q4_1_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4460,7 +4679,7 @@ kernel void kernel_mul_mv_id_q4_1_f32(mul_vec_q_n_f32_impl<block_q4_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4479,8 +4698,8 @@ kernel void kernel_mul_mv_id_q4_1_f32(kernel void kernel_mul_mv_id_q5_0_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4496,7 +4715,7 @@ kernel void kernel_mul_mv_id_q5_0_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4523,7 +4742,7 @@ kernel void kernel_mul_mv_id_q5_0_f32(mul_vec_q_n_f32_impl<block_q5_0, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4542,8 +4761,8 @@ kernel void kernel_mul_mv_id_q5_0_f32(kernel void kernel_mul_mv_id_q5_1_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4559,7 +4778,7 @@ kernel void kernel_mul_mv_id_q5_1_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4586,7 +4805,7 @@ kernel void kernel_mul_mv_id_q5_1_f32(mul_vec_q_n_f32_impl<block_q5_1, N_DST, N_SIMDGROUP, N_SIMDWIDTH>(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4605,8 +4824,8 @@ kernel void kernel_mul_mv_id_q5_1_f32(kernel void kernel_mul_mv_id_q2_K_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4622,7 +4841,7 @@ kernel void kernel_mul_mv_id_q2_K_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4649,7 +4868,7 @@ kernel void kernel_mul_mv_id_q2_K_f32(kernel_mul_mv_q2_K_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4668,8 +4887,8 @@ kernel void kernel_mul_mv_id_q2_K_f32(kernel void kernel_mul_mv_id_q3_K_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4685,7 +4904,7 @@ kernel void kernel_mul_mv_id_q3_K_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4712,7 +4931,7 @@ kernel void kernel_mul_mv_id_q3_K_f32(kernel_mul_mv_q3_K_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4731,8 +4950,8 @@ kernel void kernel_mul_mv_id_q3_K_f32(kernel void kernel_mul_mv_id_q4_K_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4748,7 +4967,7 @@ kernel void kernel_mul_mv_id_q4_K_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4775,7 +4994,7 @@ kernel void kernel_mul_mv_id_q4_K_f32(kernel_mul_mv_q4_K_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4794,8 +5013,8 @@ kernel void kernel_mul_mv_id_q4_K_f32(kernel void kernel_mul_mv_id_q5_K_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4811,7 +5030,7 @@ kernel void kernel_mul_mv_id_q5_K_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4838,7 +5057,7 @@ kernel void kernel_mul_mv_id_q5_K_f32(kernel_mul_mv_q5_K_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,@@ -4857,8 +5076,8 @@ kernel void kernel_mul_mv_id_q5_K_f32(kernel void kernel_mul_mv_id_q6_K_f32(device const char * ids,device const char * src1,- device uchar * dst,- constant int64_t & nbi1,+ device float * dst,+ constant uint64_t & nbi1,constant int64_t & ne00,constant int64_t & ne01,constant int64_t & ne02,@@ -4874,7 +5093,7 @@ kernel void kernel_mul_mv_id_q6_K_f32(constant uint64_t & nb12,constant int64_t & ne0,constant int64_t & ne1,- constant int64_t & nb1,+ constant uint64_t & nb1,constant uint & r2,constant uint & r3,constant int & idx,@@ -4901,7 +5120,7 @@ kernel void kernel_mul_mv_id_q6_K_f32(kernel_mul_mv_q6_K_f32_impl(src0[id],(device const float *) (src1 + bid*nb11),- (device float *) ( dst + bid*nb1),+ dst + bid*ne0,ne00,ne01,ne02,diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 4165e460..8fbf10f1 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -206,23 +206,6 @@ inline static void * ggml_aligned_malloc(size_t size) {#define UNUSED GGML_UNUSED#define SWAP(x, y, T) do { T SWAP = x; x = y; y = SWAP; } while (0)-#if defined(GGML_USE_ACCELERATE)-#include <Accelerate/Accelerate.h>-#if defined(GGML_USE_CLBLAST) // allow usage of CLBlast alongside Accelerate functions-#include "ggml-opencl.h"-#endif-#elif defined(GGML_USE_OPENBLAS)-#if defined(GGML_BLAS_USE_MKL)-#include <mkl.h>-#else-#include <cblas.h>-#endif-#elif defined(GGML_USE_CUBLAS)-#include "ggml-cuda.h"-#elif defined(GGML_USE_CLBLAST)-#include "ggml-opencl.h"-#endif-// floating point type used to accumulate sumstypedef double ggml_float;@@ -2064,8 +2047,13 @@ size_t ggml_type_size(enum ggml_type type) {return type_traits[type].type_size;}-float ggml_type_sizef(enum ggml_type type) {- return ((float)(type_traits[type].type_size))/type_traits[type].blck_size;+size_t ggml_row_size(enum ggml_type type, int64_t ne) {+ assert(ne % ggml_blck_size(type) == 0);+ return ggml_type_size(type)*ne/ggml_blck_size(type);+}++double ggml_type_sizef(enum ggml_type type) {+ return ((double)(type_traits[type].type_size))/type_traits[type].blck_size;}const char * ggml_type_name(enum ggml_type type) {@@ -2102,24 +2090,37 @@ size_t ggml_element_size(const struct ggml_tensor * tensor) {return ggml_type_size(tensor->type);}-static inline bool ggml_is_scalar(const struct ggml_tensor * tensor) {+bool ggml_is_scalar(const struct ggml_tensor * tensor) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");return tensor->ne[0] == 1 && tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;}-static inline bool ggml_is_vector(const struct ggml_tensor * tensor) {+bool ggml_is_vector(const struct ggml_tensor * tensor) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");return tensor->ne[1] == 1 && tensor->ne[2] == 1 && tensor->ne[3] == 1;}-static inline bool ggml_is_matrix(const struct ggml_tensor * tensor) {+bool ggml_is_matrix(const struct ggml_tensor * tensor) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");return tensor->ne[2] == 1 && tensor->ne[3] == 1;}+bool ggml_is_3d(const struct ggml_tensor * tensor) {+ return tensor->ne[3] == 1;+}++int ggml_n_dims(const struct ggml_tensor * tensor) {+ for (int i = GGML_MAX_DIMS - 1; i >= 1; --i) {+ if (tensor->ne[i] > 1) {+ return i + 1;+ }+ }+ return 1;+}+static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");@@ -2423,20 +2424,8 @@ size_t ggml_get_mem_size(const struct ggml_context * ctx) {size_t ggml_get_max_tensor_size(const struct ggml_context * ctx) {size_t max_size = 0;- struct ggml_object * obj = ctx->objects_begin;-- while (obj != NULL) {- if (obj->type == GGML_OBJECT_TENSOR) {- struct ggml_tensor * tensor = (struct ggml_tensor *) ((char *) ctx->mem_buffer + obj->offs);-- const size_t size = ggml_nbytes(tensor);-- if (max_size < size) {- max_size = size;- }- }-- obj = obj->next;+ for (struct ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor != NULL; tensor = ggml_get_next_tensor(ctx, tensor)) {+ max_size = MAX(max_size, ggml_nbytes(tensor));}return max_size;@@ -2525,7 +2514,7 @@ static struct ggml_tensor * ggml_new_tensor_impl(view_src = view_src->view_src;}- size_t data_size = ggml_type_size(type)*(ne[0]/ggml_blck_size(type));+ size_t data_size = ggml_row_size(type, ne[0]);for (int i = 1; i < n_dims; i++) {data_size *= ne[i];}@@ -2568,7 +2557,6 @@ static struct ggml_tensor * ggml_new_tensor_impl(/*.type =*/ type,/*.backend =*/ GGML_BACKEND_CPU,/*.buffer =*/ NULL,- /*.n_dims =*/ n_dims,/*.ne =*/ { 1, 1, 1, 1 },/*.nb =*/ { 0, 0, 0, 0 },/*.op =*/ GGML_OP_NONE,@@ -2675,7 +2663,7 @@ struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value) {}struct ggml_tensor * ggml_dup_tensor(struct ggml_context * ctx, const struct ggml_tensor * src) {- return ggml_new_tensor(ctx, src->type, src->n_dims, src->ne);+ return ggml_new_tensor(ctx, src->type, GGML_MAX_DIMS, src->ne);}static void ggml_set_op_params(struct ggml_tensor * tensor, const void * params, size_t params_size) {@@ -3124,7 +3112,7 @@ struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char *struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx,struct ggml_tensor * src) {- struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, src->n_dims, src->ne, src, 0);+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src, 0);ggml_format_name(result, "%s (view)", src->name);for (int i = 0; i < GGML_MAX_DIMS; i++) {@@ -3134,7 +3122,7 @@ struct ggml_tensor * ggml_view_tensor(return result;}-struct ggml_tensor * ggml_get_first_tensor(struct ggml_context * ctx) {+struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx) {struct ggml_object * obj = ctx->objects_begin;char * const mem_buffer = ctx->mem_buffer;@@ -3150,7 +3138,7 @@ struct ggml_tensor * ggml_get_first_tensor(struct ggml_context * ctx) {return NULL;}-struct ggml_tensor * ggml_get_next_tensor(struct ggml_context * ctx, struct ggml_tensor * tensor) {+struct ggml_tensor * ggml_get_next_tensor(const struct ggml_context * ctx, struct ggml_tensor * tensor) {struct ggml_object * obj = (struct ggml_object *) ((char *)tensor - GGML_OBJECT_SIZE);obj = obj->next;@@ -3282,10 +3270,10 @@ static struct ggml_tensor * ggml_add_cast_impl(is_node = true;}- struct ggml_tensor * result = ggml_new_tensor(ctx, type, a->n_dims, a->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, type, GGML_MAX_DIMS, a->ne);result->op = GGML_OP_ADD;- result->grad = is_node ? ggml_new_tensor(ctx, GGML_TYPE_F32, a->n_dims, a->ne) : NULL;+ result->grad = is_node ? ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne) : NULL;result->src[0] = a;result->src[1] = b;@@ -3654,12 +3642,12 @@ struct ggml_tensor * ggml_sum_rows(is_node = true;}- int64_t ne[4] = {1,1,1,1};- for (int i=1; i<a->n_dims; ++i) {+ int64_t ne[GGML_MAX_DIMS] = { 1 };+ for (int i = 1; i < GGML_MAX_DIMS; ++i) {ne[i] = a->ne[i];}- struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, a->n_dims, ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, ne);result->op = GGML_OP_SUM_ROWS;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -3680,8 +3668,8 @@ struct ggml_tensor * ggml_mean(is_node = true;}- int64_t ne[GGML_MAX_DIMS] = { 1, a->ne[1], a->ne[2], a->ne[3] };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, a->n_dims, ne);+ int64_t ne[4] = { 1, a->ne[1], a->ne[2], a->ne[3] };+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);result->op = GGML_OP_MEAN;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -3703,8 +3691,7 @@ struct ggml_tensor * ggml_argmax(is_node = true;}- int64_t ne[GGML_MAX_DIMS] = { a->ne[1], 1, 1, 1 };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, a->n_dims, ne);+ struct ggml_tensor * result = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, a->ne[1]);result->op = GGML_OP_ARGMAX;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -3727,7 +3714,7 @@ struct ggml_tensor * ggml_repeat(is_node = true;}- struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, b->n_dims, b->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, b->ne);result->op = GGML_OP_REPEAT;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -3754,7 +3741,7 @@ struct ggml_tensor * ggml_repeat_back(return a;}- struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, b->n_dims, b->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, GGML_MAX_DIMS, b->ne);result->op = GGML_OP_REPEAT_BACK;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4095,7 +4082,6 @@ static struct ggml_tensor * ggml_group_norm_impl(result->op = GGML_OP_GROUP_NORM;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;- result->src[1] = NULL; // TODO: maybe store epsilon here?return result;}@@ -4130,7 +4116,7 @@ struct ggml_tensor * ggml_mul_mat(}const int64_t ne[4] = { a->ne[1], b->ne[1], b->ne[2], b->ne[3] };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, MAX(a->n_dims, b->n_dims), ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);result->op = GGML_OP_MUL_MAT;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4172,7 +4158,7 @@ struct ggml_tensor * ggml_mul_mat_id(}const int64_t ne[4] = { as[0]->ne[1], b->ne[1], b->ne[2], b->ne[3] };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, MAX(as[0]->n_dims, b->n_dims), ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);ggml_set_op_params_i32(result, 0, id);ggml_set_op_params_i32(result, 1, n_as);@@ -4210,7 +4196,7 @@ struct ggml_tensor * ggml_out_prod(// a is broadcastable to b for ne[2] and ne[3] -> use b->ne[2] and b->ne[3]const int64_t ne[4] = { a->ne[0], b->ne[0], b->ne[2], b->ne[3] };- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, MAX(a->n_dims, b->n_dims), ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);result->op = GGML_OP_OUT_PROD;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4225,23 +4211,23 @@ struct ggml_tensor * ggml_out_prod(static struct ggml_tensor * ggml_scale_impl(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b,+ float s,bool inplace) {- GGML_ASSERT(ggml_is_scalar(b));GGML_ASSERT(ggml_is_padded_1d(a));bool is_node = false;- if (a->grad || b->grad) {+ if (a->grad) {is_node = true;}struct ggml_tensor * result = inplace ? ggml_view_tensor(ctx, a) : ggml_dup_tensor(ctx, a);+ ggml_set_op_params(result, &s, sizeof(s));+result->op = GGML_OP_SCALE;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;- result->src[1] = b;return result;}@@ -4249,15 +4235,15 @@ static struct ggml_tensor * ggml_scale_impl(struct ggml_tensor * ggml_scale(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b) {- return ggml_scale_impl(ctx, a, b, false);+ float s) {+ return ggml_scale_impl(ctx, a, s, false);}struct ggml_tensor * ggml_scale_inplace(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b) {- return ggml_scale_impl(ctx, a, b, true);+ float s) {+ return ggml_scale_impl(ctx, a, s, true);}// ggml_set@@ -4495,7 +4481,7 @@ struct ggml_tensor * ggml_reshape(//GGML_ASSERT(false);}- struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, b->n_dims, b->ne, a, 0);+ struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, a, 0);ggml_format_name(result, "%s (reshaped)", a->name);result->op = GGML_OP_RESHAPE;@@ -4821,8 +4807,11 @@ struct ggml_tensor * ggml_get_rows(}// TODO: implement non F32 return- //struct ggml_tensor * result = ggml_new_tensor_2d(ctx, a->type, a->ne[0], b->ne[0]);- struct ggml_tensor * result = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, a->ne[0], b->ne[0], b->ne[1], b->ne[2]);+ enum ggml_type type = GGML_TYPE_F32;+ if (a->type == GGML_TYPE_I32) {+ type = a->type;+ }+ struct ggml_tensor * result = ggml_new_tensor_4d(ctx, type, a->ne[0], b->ne[0], b->ne[1], b->ne[2]);result->op = GGML_OP_GET_ROWS;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -4873,7 +4862,7 @@ struct ggml_tensor * ggml_diag(}const int64_t ne[4] = { a->ne[0], a->ne[0], a->ne[2], a->ne[3] };- struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, MAX(a->n_dims, 2), ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, a->type, 4, ne);result->op = GGML_OP_DIAG;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -5520,7 +5509,7 @@ struct ggml_tensor * ggml_pool_1d(is_node = true;}- const int64_t ne[3] = {+ const int64_t ne[2] = {ggml_calc_pool_output_size(a->ne[0], k0, s0, p0),a->ne[1],};@@ -5595,7 +5584,6 @@ static struct ggml_tensor * ggml_upscale_impl(result->op_params[0] = scale_factor;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;- result->src[1] = NULL;return result;}@@ -5639,7 +5627,7 @@ struct ggml_tensor * ggml_argsort(enum ggml_sort_order order) {bool is_node = false;- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, a->n_dims, a->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_I32, GGML_MAX_DIMS, a->ne);ggml_set_op_params_i32(result, 0, (int32_t) order);@@ -5686,7 +5674,7 @@ struct ggml_tensor * ggml_flash_attn(}//struct ggml_tensor * result = ggml_dup_tensor(ctx, q);- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, q->n_dims, q->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, q->ne);int32_t t = masked ? 1 : 0;ggml_set_op_params(result, &t, sizeof(t));@@ -5719,7 +5707,7 @@ struct ggml_tensor * ggml_flash_ff(}//struct ggml_tensor * result = ggml_dup_tensor(ctx, a);- struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, a->n_dims, a->ne);+ struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne);result->op = GGML_OP_FLASH_FF;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;@@ -5835,7 +5823,6 @@ struct ggml_tensor * ggml_win_part(const int np = npx*npy;const int64_t ne[4] = { a->ne[0], w, w, np, };-struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);int32_t params[] = { npx, npy, w };@@ -5901,7 +5888,6 @@ struct ggml_tensor * ggml_get_rel_pos(result->op = GGML_OP_GET_REL_POS;result->grad = is_node ? ggml_dup_tensor(ctx, result) : NULL;result->src[0] = a;- result->src[1] = NULL;return result;}@@ -6996,14 +6982,165 @@ static void ggml_compute_forward_dup_f32(}}-static void ggml_compute_forward_dup(+// A simplified version of ggml_compute_forward_dup that doesn't do float upcasting, and just plain old memcpy.+static void ggml_compute_forward_dup_bytes(const struct ggml_compute_params * params,const struct ggml_tensor * src0,struct ggml_tensor * dst) {- if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst) && src0->type == dst->type) {+ GGML_ASSERT(ggml_nelements(dst) == ggml_nelements(src0));+ GGML_ASSERT(src0->type == dst->type);++ if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {+ return;+ }++ if (ggml_is_contiguous(src0) && ggml_is_contiguous(dst)) {ggml_compute_forward_dup_same_cont(params, src0, dst);return;}++ GGML_TENSOR_UNARY_OP_LOCALS;++ const size_t type_size = ggml_type_size(src0->type);+ const int ith = params->ith; // thread index+ const int nth = params->nth; // number of threads+++ // parallelize by rows+ const int nr = ne01;+ // number of rows per thread+ const int dr = (nr + nth - 1) / nth;+ // row range for this thread+ const int ir0 = dr * ith;+ const int ir1 = MIN(ir0 + dr, nr);++ if (src0->type == dst->type &&+ ne00 == ne0 &&+ nb00 == type_size && nb0 == type_size) {+ // copy by rows+ const size_t rs = ne00 * type_size;+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ memcpy(+ ((char *) dst->data + i01*nb1 + i02*nb2 + i03*nb3),+ ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03),+ rs);+ }+ }+ }+ return;+ }++ if (ggml_is_contiguous(dst)) {+ size_t id = 0;+ char * dst_ptr = (char *) dst->data;+ const size_t rs = ne00 * type_size;++ if (nb00 == type_size) {+ // src0 is contigous on first dimension, copy by rows+ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ id += rs * ir0;+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ const char * src0_ptr = (char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;+ memcpy(dst_ptr + id, src0_ptr, rs);+ id += rs;+ }+ id += rs * (ne01 - ir1);+ }+ }+ } else {+ //printf("%s: this is not optimal - fix me\n", __func__);++ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ id += rs * ir0;+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = (char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03;+ memcpy(dst_ptr + id, src0_ptr, type_size);++ id += type_size;+ }+ }+ id += rs * (ne01 - ir1);+ }+ }+ }++ return;+ }++ // dst counters++ int64_t i10 = 0;+ int64_t i11 = 0;+ int64_t i12 = 0;+ int64_t i13 = 0;++ for (int64_t i03 = 0; i03 < ne03; i03++) {+ for (int64_t i02 = 0; i02 < ne02; i02++) {+ i10 += ne00 * ir0;+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ for (int64_t i01 = ir0; i01 < ir1; i01++) {+ for (int64_t i00 = 0; i00 < ne00; i00++) {+ const char * src0_ptr = ((char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03);+ char * dst_ptr = ((char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3);++ memcpy(dst_ptr, src0_ptr, type_size);++ if (++i10 == ne0) {+ i10 = 0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+ i10 += ne00 * (ne01 - ir1);+ while (i10 >= ne0) {+ i10 -= ne0;+ if (++i11 == ne1) {+ i11 = 0;+ if (++i12 == ne2) {+ i12 = 0;+ if (++i13 == ne3) {+ i13 = 0;+ }+ }+ }+ }+ }+ }+}++static void ggml_compute_forward_dup(+ const struct ggml_compute_params * params,+ const struct ggml_tensor * src0,+ struct ggml_tensor * dst) {+ if (src0->type == dst->type) {+ ggml_compute_forward_dup_bytes(params, src0, dst);+ return;+ }+switch (src0->type) {case GGML_TYPE_F16:{@@ -7625,7 +7762,7 @@ static void ggml_compute_forward_acc_f32(GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));// view src0 and dst with these strides and data offset inbytes during acc- // nb0 is implicitely element_size because src0 and dst are contiguous+ // nb0 is implicitly element_size because src0 and dst are contiguoussize_t nb1 = ((int32_t *) dst->op_params)[0];size_t nb2 = ((int32_t *) dst->op_params)[1];size_t nb3 = ((int32_t *) dst->op_params)[2];@@ -7819,10 +7956,10 @@ static void ggml_compute_forward_mul_f32(const int ith = params->ith;const int nth = params->nth;-// TODO: OpenCL kernel support broadcast#ifdef GGML_USE_CLBLASTif (src1->backend == GGML_BACKEND_GPU) {- GGML_ASSERT(ggml_are_same_shape(src0, src1));+ // TODO: OpenCL kernel support full broadcast+ GGML_ASSERT(ggml_can_repeat_rows(src1, src0));if (ith == 0) {ggml_cl_mul(src0, src1, dst);}@@ -8462,10 +8599,12 @@ static void ggml_compute_forward_repeat(struct ggml_tensor * dst) {switch (src0->type) {case GGML_TYPE_F16:+ case GGML_TYPE_I16:{ggml_compute_forward_repeat_f16(params, src0, dst);} break;case GGML_TYPE_F32:+ case GGML_TYPE_I32:{ggml_compute_forward_repeat_f32(params, src0, dst);} break;@@ -8608,6 +8747,7 @@ static void ggml_compute_forward_concat(struct ggml_tensor* dst) {switch (src0->type) {case GGML_TYPE_F32:+ case GGML_TYPE_I32:{ggml_compute_forward_concat_f32(params, src0, src1, dst);} break;@@ -9635,16 +9775,11 @@ static bool ggml_compute_forward_mul_mat_use_blas(}#endif-// off1 = offset in i11 and i1-// cne1 = ne11 and ne1-// in a normal matrix multiplication, off1 = 0 and cne1 = ne1-// during GGML_TASK_INIT, the full src1 is converted regardless of off1 and cne1static void ggml_compute_forward_mul_mat(const struct ggml_compute_params * params,const struct ggml_tensor * src0,const struct ggml_tensor * src1,- struct ggml_tensor * dst,- int64_t off1, int64_t cne1) {+ struct ggml_tensor * dst) {int64_t t0 = ggml_perf_time_us();UNUSED(t0);@@ -9712,9 +9847,9 @@ static void ggml_compute_forward_mul_mat(const int64_t i03 = i13/r3;const int64_t i02 = i12/r2;- const void * x = (char *) src0->data + i02*nb02 + i03*nb03;- const float * y = (float *) ((char *) src1->data + off1*nb11 + i12*nb12 + i13*nb13);- float * d = (float *) ((char *) dst->data + off1*nb1 + i12*nb2 + i13*nb3);+ const void * x = (char *) src0->data + i02*nb02 + i03*nb03;+ const float * y = (float *) ((char *) src1->data + i12*nb12 + i13*nb13);+ float * d = (float *) ((char *) dst->data + i12*nb2 + i13*nb3);if (type != GGML_TYPE_F32) {float * const wdata = params->wdata;@@ -9731,7 +9866,7 @@ static void ggml_compute_forward_mul_mat(}cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans,- cne1, ne01, ne10,+ ne1, ne01, ne10,1.0f, y, ne10,x, ne00,0.0f, d, ne01);@@ -9747,10 +9882,10 @@ static void ggml_compute_forward_mul_mat(if (params->type == GGML_TASK_INIT) {if (src1->type != vec_dot_type) {char * wdata = params->wdata;- const size_t row_size = ne10*ggml_type_size(vec_dot_type)/ggml_blck_size(vec_dot_type);+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);assert(params->wsize >= ne11*ne12*ne13*row_size);- assert(src1->type == GGML_TYPE_F32);+ GGML_ASSERT(src1->type == GGML_TYPE_F32);for (int64_t i13 = 0; i13 < ne13; ++i13) {for (int64_t i12 = 0; i12 < ne12; ++i12) {@@ -9770,10 +9905,10 @@ static void ggml_compute_forward_mul_mat(}const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;- const size_t row_size = ne10*ggml_type_size(vec_dot_type)/ggml_blck_size(vec_dot_type);+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);- const int64_t nr0 = ne01; // src0 rows- const int64_t nr1 = cne1*ne12*ne13; // src1 rows+ const int64_t nr0 = ne01; // src0 rows+ const int64_t nr1 = ne1*ne12*ne13; // src1 rows//printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1);@@ -9815,9 +9950,9 @@ static void ggml_compute_forward_mul_mat(for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) {for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) {for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) {- const int64_t i13 = (ir1/(ne12*cne1));- const int64_t i12 = (ir1 - i13*ne12*cne1)/cne1;- const int64_t i11 = (ir1 - i13*ne12*cne1 - i12*cne1) + off1;+ const int64_t i13 = (ir1/(ne12*ne1));+ const int64_t i12 = (ir1 - i13*ne12*ne1)/ne1;+ const int64_t i11 = (ir1 - i13*ne12*ne1 - i12*ne1);// broadcast src0 into src1const int64_t i03 = i13/r3;@@ -9857,28 +9992,191 @@ static void ggml_compute_forward_mul_mat(static void ggml_compute_forward_mul_mat_id(const struct ggml_compute_params * params,- const struct ggml_tensor * src0,+ const struct ggml_tensor * ids,const struct ggml_tensor * src1,struct ggml_tensor * dst) {- if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {- // during GGML_TASK_INIT the entire src1 is converted to vec_dot_type- ggml_compute_forward_mul_mat(params, dst->src[2], src1, dst, 0, dst->ne[1]);- return;- }+ const struct ggml_tensor * src0 = dst->src[2]; // only for GGML_TENSOR_BINARY_OP_LOCALS++ GGML_TENSOR_BINARY_OP_LOCALS++ const int ith = params->ith;+ const int nth = params->nth;++ const enum ggml_type type = src0->type;++ const bool src1_cont = ggml_is_contiguous(src1);++ ggml_vec_dot_t const vec_dot = type_traits[type].vec_dot;+ enum ggml_type const vec_dot_type = type_traits[type].vec_dot_type;+ ggml_from_float_t const from_float_to_vec_dot = type_traits[vec_dot_type].from_float;++ GGML_ASSERT(ne0 == ne01);+ GGML_ASSERT(ne1 == ne11);+ GGML_ASSERT(ne2 == ne12);+ GGML_ASSERT(ne3 == ne13);++ // we don't support permuted src0 or src1+ GGML_ASSERT(nb00 == ggml_type_size(type));+ GGML_ASSERT(nb10 == ggml_type_size(src1->type));++ // dst cannot be transposed or permuted+ GGML_ASSERT(nb0 == sizeof(float));+ GGML_ASSERT(nb0 <= nb1);+ GGML_ASSERT(nb1 <= nb2);+ GGML_ASSERT(nb2 <= nb3);++ // broadcast factors+ const int64_t r2 = ne12/ne02;+ const int64_t r3 = ne13/ne03;- const struct ggml_tensor * ids = src0;+ // row groupsconst int id = ggml_get_op_params_i32(dst, 0);const int n_as = ggml_get_op_params_i32(dst, 1);- for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {- const int32_t row_id = *(const int32_t *) ((const char *) ids->data + i01*ids->nb[1] + id*ids->nb[0]);+ char * wdata_src1_end = (src1->type == vec_dot_type) ?+ (char *) params->wdata :+ (char *) params->wdata + GGML_PAD(ggml_row_size(vec_dot_type, ggml_nelements(src1)), sizeof(int64_t));++ int64_t * matrix_row_counts = (int64_t *) (wdata_src1_end); // [n_as]+ int64_t * matrix_rows = matrix_row_counts + n_as; // [n_as][ne11]++ #define MMID_MATRIX_ROW(row_id, i1) matrix_rows[(row_id)*ne11 + (i1)]++ if (params->type == GGML_TASK_INIT) {+ char * wdata = params->wdata;+ if (src1->type != vec_dot_type) {+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);++ assert(params->wsize >= ne11*ne12*ne13*row_size);+ assert(src1->type == GGML_TYPE_F32);++ for (int64_t i13 = 0; i13 < ne13; ++i13) {+ for (int64_t i12 = 0; i12 < ne12; ++i12) {+ for (int64_t i11 = 0; i11 < ne11; ++i11) {+ from_float_to_vec_dot((float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11), (void *) wdata, ne10);+ wdata += row_size;+ }+ }+ }+ }++ // initialize matrix_row_counts+ GGML_ASSERT(wdata == wdata_src1_end);+ memset(matrix_row_counts, 0, n_as*sizeof(int64_t));++ // group rows by src0 matrix+ for (int64_t i01 = 0; i01 < ids->ne[1]; i01++) {+ const int32_t row_id = *(const int32_t *) ((const char *) ids->data + i01*ids->nb[1] + id*ids->nb[0]);++ GGML_ASSERT(row_id >= 0 && row_id < n_as);+ MMID_MATRIX_ROW(row_id, matrix_row_counts[row_id]) = i01;+ matrix_row_counts[row_id] += 1;+ }++ return;+ }++ if (params->type == GGML_TASK_FINALIZE) {+ return;+ }++ // compute each matrix multiplication in sequence+ for (int cur_a = 0; cur_a < n_as; ++cur_a) {+ const int64_t cne1 = matrix_row_counts[cur_a];++ if (cne1 == 0) {+ continue;+ }++ const struct ggml_tensor * src0_cur = dst->src[cur_a + 2];++ const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;+ const size_t row_size = ggml_row_size(vec_dot_type, ne10);++ const int64_t nr0 = ne01; // src0 rows+ const int64_t nr1 = cne1*ne12*ne13; // src1 rows++ //printf("nr0 = %lld, nr1 = %lld\n", nr0, nr1);++ // distribute the thread work across the inner or outer loop based on which one is larger++ const int64_t nth0 = nr0 > nr1 ? nth : 1; // parallelize by src0 rows+ const int64_t nth1 = nr0 > nr1 ? 1 : nth; // parallelize by src1 rows++ const int64_t ith0 = ith % nth0;+ const int64_t ith1 = ith / nth0;++ const int64_t dr0 = (nr0 + nth0 - 1)/nth0;+ const int64_t dr1 = (nr1 + nth1 - 1)/nth1;++ const int64_t ir010 = dr0*ith0;+ const int64_t ir011 = MIN(ir010 + dr0, nr0);++ const int64_t ir110 = dr1*ith1;+ const int64_t ir111 = MIN(ir110 + dr1, nr1);- GGML_ASSERT(row_id >= 0 && row_id < n_as);+ //printf("ir010 = %6lld, ir011 = %6lld, ir110 = %6lld, ir111 = %6lld\n", ir010, ir011, ir110, ir111);- const struct ggml_tensor * src0_row = dst->src[row_id + 2];- ggml_compute_forward_mul_mat(params, src0_row, src1, dst, i01, 1);+ // threads with no work simply yield (not sure if it helps)+ if (ir010 >= ir011 || ir110 >= ir111) {+ sched_yield();+ continue;+ }++ assert(ne12 % ne02 == 0);+ assert(ne13 % ne03 == 0);++ // block-tiling attempt+ const int64_t blck_0 = 16;+ const int64_t blck_1 = 16;++ // attempt to reduce false-sharing (does not seem to make a difference)+ float tmp[16];++ for (int64_t iir1 = ir110; iir1 < ir111; iir1 += blck_1) {+ for (int64_t iir0 = ir010; iir0 < ir011; iir0 += blck_0) {+ for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir111; ++ir1) {+ const int64_t i13 = (ir1/(ne12*cne1)); // Note: currently, src1 is always a matrix+ const int64_t i12 = (ir1 - i13*ne12*cne1)/cne1;+ const int64_t _i11 = (ir1 - i13*ne12*cne1 - i12*cne1);+ const int64_t i11 = MMID_MATRIX_ROW(cur_a, _i11);++ // broadcast src0 into src1+ const int64_t i03 = i13/r3;+ const int64_t i02 = i12/r2;++ const int64_t i1 = i11;+ const int64_t i2 = i12;+ const int64_t i3 = i13;++ const char * src0_row = (const char *) src0_cur->data + (0 + i02*nb02 + i03*nb03);++ // desc: when src1 is not a contiguous memory block we have to calculate the offset using the strides+ // if it is, then we have either copied the data to params->wdata and made it contiguous or we are using+ // the original src1 data pointer, so we should index using the indices directly+ // TODO: this is a bit of a hack, we should probably have a better way to handle this+ const char * src1_col = (const char *) wdata ++ (src1_cont || src1->type != vec_dot_type+ ? (i11 + i12*ne11 + i13*ne12*ne11)*row_size+ : (i11*nb11 + i12*nb12 + i13*nb13));++ float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2 + i3*nb3));++ //for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {+ // vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);+ //}++ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir011; ++ir0) {+ vec_dot(ne00, &tmp[ir0 - iir0], src0_row + ir0*nb01, src1_col);+ }+ memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir011) - iir0)*sizeof(float));+ }+ }+ }}++ #undef MMID_MATRIX_ROW}// ggml_compute_forward_out_prod@@ -10222,19 +10520,18 @@ static void ggml_compute_forward_out_prod(static void ggml_compute_forward_scale_f32(const struct ggml_compute_params * params,const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(ggml_is_contiguous(dst));GGML_ASSERT(ggml_are_same_shape(src0, dst));- GGML_ASSERT(ggml_is_scalar(src1));if (params->type == GGML_TASK_INIT || params->type == GGML_TASK_FINALIZE) {return;}// scale factor- const float v = *(float *) src1->data;+ float v;+ memcpy(&v, dst->op_params, sizeof(float));const int ith = params->ith;const int nth = params->nth;@@ -10265,12 +10562,11 @@ static void ggml_compute_forward_scale_f32(static void ggml_compute_forward_scale(const struct ggml_compute_params * params,const struct ggml_tensor * src0,- const struct ggml_tensor * src1,struct ggml_tensor * dst) {switch (src0->type) {case GGML_TYPE_F32:{- ggml_compute_forward_scale_f32(params, src0, src1, dst);+ ggml_compute_forward_scale_f32(params, src0, dst);} break;default:{@@ -10290,7 +10586,7 @@ static void ggml_compute_forward_set_f32(GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));// view src0 and dst with these strides and data offset inbytes during set- // nb0 is implicitely element_size because src0 and dst are contiguous+ // nb0 is implicitly element_size because src0 and dst are contiguoussize_t nb1 = ((int32_t *) dst->op_params)[0];size_t nb2 = ((int32_t *) dst->op_params)[1];size_t nb3 = ((int32_t *) dst->op_params)[2];@@ -10576,6 +10872,7 @@ static void ggml_compute_forward_get_rows(ggml_compute_forward_get_rows_f16(params, src0, src1, dst);} break;case GGML_TYPE_F32:+ case GGML_TYPE_I32:{ggml_compute_forward_get_rows_f32(params, src0, src1, dst);} break;@@ -14268,7 +14565,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm} break;case GGML_OP_MUL_MAT:{- ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor, 0, tensor->ne[1]);+ ggml_compute_forward_mul_mat(params, tensor->src[0], tensor->src[1], tensor);} break;case GGML_OP_MUL_MAT_ID:{@@ -14280,7 +14577,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm} break;case GGML_OP_SCALE:{- ggml_compute_forward_scale(params, tensor->src[0], tensor->src[1], tensor);+ ggml_compute_forward_scale(params, tensor->src[0], tensor);} break;case GGML_OP_SET:{@@ -14644,7 +14941,7 @@ static struct ggml_tensor * ggml_recompute_graph_node(return replacements->vals[i];}- struct ggml_tensor * clone = ggml_new_tensor(ctx, node->type, node->n_dims, node->ne);+ struct ggml_tensor * clone = ggml_new_tensor(ctx, node->type, GGML_MAX_DIMS, node->ne);// insert clone into replacementsGGML_ASSERT(replacements->set.keys[i] == NULL); // assert that we don't overwrite@@ -14714,7 +15011,7 @@ void ggml_build_backward_gradient_checkpointing(// insert new tensors recomputing src, reusing already made replacements,// remember replacements: remember new tensors with mapping from corresponding gf nodes// recurse for input tensors,- // unless (i.e. terminating when) input tensors are replacments (like checkpoints)+ // unless (i.e. terminating when) input tensors are replacements (like checkpoints)node->src[k] = ggml_recompute_graph_node(ctx, gf, replacements, node->src[k]);}// insert rewritten backward node with replacements made into resulting backward graph gb@@ -14736,7 +15033,7 @@ static struct ggml_tensor * ggml_add_or_set(struct ggml_context * ctx, struct ggstatic struct ggml_tensor * ggml_acc_or_set(struct ggml_context * ctx, struct ggml_tensor * a, struct ggml_tensor * b, size_t nb1, size_t nb2, size_t nb3, size_t offset, struct ggml_hash_set zero_table) {if (ggml_hash_contains(zero_table, a)) {- struct ggml_tensor * a_zero = ggml_scale(ctx, a, ggml_new_f32(ctx, 0));+ struct ggml_tensor * a_zero = ggml_scale(ctx, a, 0.0f);return ggml_acc_impl(ctx, a_zero, b, nb1, nb2, nb3, offset, false);} else {return ggml_acc_impl(ctx, a, b, nb1, nb2, nb3, offset, false);@@ -14872,7 +15169,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorsrc0->grad,ggml_scale(ctx,ggml_mul(ctx, src0, tensor->grad),- ggml_new_f32(ctx, 2.0f)),+ 2.0f),zero_table);}} break;@@ -14886,7 +15183,7 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorggml_div(ctx,tensor->grad,tensor),- ggml_new_f32(ctx, 0.5f)),+ 0.5f),zero_table);}} break;@@ -15052,17 +15349,13 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensor{// necessary for llamaif (src0->grad) {+ float s;+ memcpy(&s, tensor->op_params, sizeof(float));+src0->grad =ggml_add_or_set(ctx,src0->grad,- ggml_scale_impl(ctx, tensor->grad, src1, false),- zero_table);- }- if (src1->grad) {- src1->grad =- ggml_add_or_set(ctx,- src1->grad,- ggml_sum(ctx, ggml_mul_impl(ctx, tensor->grad, src0, false)),+ ggml_scale_impl(ctx, tensor->grad, s, false),zero_table);}} break;@@ -15240,6 +15533,8 @@ static void ggml_compute_backward(struct ggml_context * ctx, struct ggml_tensorconst int n_past = ((int32_t *) tensor->op_params)[0];src0->grad =ggml_add_or_set(ctx, src0->grad,+ /* ggml_diag_mask_inf_impl() shouldn't be here */+ /* ref: https://github.com/ggerganov/llama.cpp/pull/4203#discussion_r1412377992 */ggml_diag_mask_zero_impl(ctx, tensor->grad, n_past, false),zero_table);}@@ -16069,7 +16364,6 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {} break;case GGML_OP_MUL_MAT_ID:{- // FIXME: blasn_tasks = n_threads;} break;case GGML_OP_OUT_PROD:@@ -16398,25 +16692,21 @@ struct ggml_cplan ggml_graph_plan(struct ggml_cgraph * cgraph, int n_threads) {} else#endifif (node->src[1]->type != vec_dot_type) {- cur = ggml_type_size(vec_dot_type)*ggml_nelements(node->src[1])/ggml_blck_size(vec_dot_type);+ cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));}} break;case GGML_OP_MUL_MAT_ID:{- const struct ggml_tensor * a = node->src[2];- const struct ggml_tensor * b = node->src[1];- const enum ggml_type vec_dot_type = type_traits[a->type].vec_dot_type;-#if defined(GGML_USE_ACCELERATE) || defined(GGML_USE_OPENBLAS)- if (ggml_compute_forward_mul_mat_use_blas(a, b, node)) {- if (a->type != GGML_TYPE_F32) {- // here we need memory just for single 2D matrix from src0- cur = ggml_type_size(GGML_TYPE_F32)*(a->ne[0]*a->ne[1]);- }- } else-#endif- if (b->type != vec_dot_type) {- cur = ggml_type_size(vec_dot_type)*ggml_nelements(b)/ggml_blck_size(vec_dot_type);+ const struct ggml_tensor * src0 = node->src[2];+ const struct ggml_tensor * src1 = node->src[1];+ const enum ggml_type vec_dot_type = type_traits[src0->type].vec_dot_type;+ if (src1->type != vec_dot_type) {+ cur = ggml_row_size(vec_dot_type, ggml_nelements(src1));}+ const int n_as = ggml_get_op_params_i32(node, 1);+ cur = GGML_PAD(cur, sizeof(int64_t)); // align+ cur += n_as * sizeof(int64_t); // matrix_row_counts+ cur += n_as * src1->ne[1] * sizeof(int64_t); // matrix_rows} break;case GGML_OP_OUT_PROD:{@@ -16646,7 +16936,7 @@ static void ggml_graph_export_leaf(const struct ggml_tensor * tensor, FILE * foufprintf(fout, "%-6s %-12s %8d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %16zu %16zu %16zu %16zu %16p %32s\n",ggml_type_name(tensor->type),ggml_op_name (tensor->op),- tensor->n_dims,+ ggml_n_dims(tensor),ne[0], ne[1], ne[2], ne[3],nb[0], nb[1], nb[2], nb[3],tensor->data,@@ -16661,7 +16951,7 @@ static void ggml_graph_export_node(const struct ggml_tensor * tensor, const chararg,ggml_type_name(tensor->type),ggml_op_name (tensor->op),- tensor->n_dims,+ ggml_n_dims(tensor),ne[0], ne[1], ne[2], ne[3],nb[0], nb[1], nb[2], nb[3],tensor->data,@@ -16751,11 +17041,9 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {const uint32_t type = tensor->type;const uint32_t op = tensor->op;- const uint32_t n_dims = tensor->n_dims;fwrite(&type, sizeof(uint32_t), 1, fout);fwrite(&op, sizeof(uint32_t), 1, fout);- fwrite(&n_dims, sizeof(uint32_t), 1, fout);for (int j = 0; j < GGML_MAX_DIMS; ++j) {const uint64_t ne = tensor->ne[j];@@ -16785,11 +17073,9 @@ void ggml_graph_export(const struct ggml_cgraph * cgraph, const char * fname) {const uint32_t type = tensor->type;const uint32_t op = tensor->op;- const uint32_t n_dims = tensor->n_dims;fwrite(&type, sizeof(uint32_t), 1, fout);fwrite(&op, sizeof(uint32_t), 1, fout);- fwrite(&n_dims, sizeof(uint32_t), 1, fout);for (int j = 0; j < GGML_MAX_DIMS; ++j) {const uint64_t ne = tensor->ne[j];@@ -16961,12 +17247,10 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *{uint32_t type;uint32_t op;- uint32_t n_dims;for (uint32_t i = 0; i < n_leafs; ++i) {type = *(const uint32_t *) ptr; ptr += sizeof(type);op = *(const uint32_t *) ptr; ptr += sizeof(op);- n_dims = *(const uint32_t *) ptr; ptr += sizeof(n_dims);int64_t ne[GGML_MAX_DIMS];size_t nb[GGML_MAX_DIMS];@@ -16982,7 +17266,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *nb[j] = nb_cur;}- struct ggml_tensor * tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, n_dims, ne);+ struct ggml_tensor * tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, GGML_MAX_DIMS, ne);tensor->op = (enum ggml_op) op;@@ -16999,7 +17283,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *ptr += ggml_nbytes(tensor);- fprintf(stderr, "%s: loaded leaf %d: '%16s', %3d dims, %9zu bytes\n", __func__, i, tensor->name, n_dims, ggml_nbytes(tensor));+ fprintf(stderr, "%s: loaded leaf %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));}}@@ -17009,12 +17293,10 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *{uint32_t type;uint32_t op;- uint32_t n_dims;for (uint32_t i = 0; i < n_nodes; ++i) {type = *(const uint32_t *) ptr; ptr += sizeof(type);op = *(const uint32_t *) ptr; ptr += sizeof(op);- n_dims = *(const uint32_t *) ptr; ptr += sizeof(n_dims);enum ggml_op eop = (enum ggml_op) op;@@ -17085,7 +17367,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *} break;default:{- tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, n_dims, ne);+ tensor = ggml_new_tensor(*ctx_eval, (enum ggml_type) type, GGML_MAX_DIMS, ne);tensor->op = eop;} break;@@ -17104,7 +17386,7 @@ struct ggml_cgraph * ggml_graph_import(const char * fname, struct ggml_context *result->nodes[i] = tensor;- fprintf(stderr, "%s: loaded node %d: '%16s', %3d dims, %9zu bytes\n", __func__, i, tensor->name, n_dims, ggml_nbytes(tensor));+ fprintf(stderr, "%s: loaded node %d: '%16s', %9zu bytes\n", __func__, i, tensor->name, ggml_nbytes(tensor));}}}@@ -17242,7 +17524,7 @@ void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraphfprintf(fp, "(%s)|", ggml_type_name(node->type));}- if (node->n_dims == 2) {+ if (ggml_is_matrix(node)) {fprintf(fp, "%d [%" PRId64 ", %" PRId64 "] | <x>%s", i, node->ne[0], node->ne[1], ggml_op_symbol(node->op));} else {fprintf(fp, "%d [%" PRId64 ", %" PRId64 ", %" PRId64 "] | <x>%s", i, node->ne[0], node->ne[1], node->ne[2], ggml_op_symbol(node->op));@@ -17371,9 +17653,9 @@ static void ggml_opt_acc_grad(int np, struct ggml_tensor * const ps[], float * g}//-// ADAM+// Using AdamW - ref: https://arxiv.org/pdf/1711.05101v3.pdf//-// ref: https://arxiv.org/pdf/1412.6980.pdf+// (Original Adam - ref: https://arxiv.org/pdf/1412.6980.pdf)//static enum ggml_opt_result ggml_opt_adam(@@ -17509,7 +17791,7 @@ static enum ggml_opt_result ggml_opt_adam(int64_t i = 0;for (int p = 0; p < np; ++p) {const int64_t ne = ggml_nelements(ps[p]);- const float p_decay = ((ps[p]->n_dims >= decay_min_ndim) ? decay : 0.0f) * sched;+ const float p_decay = ((ggml_n_dims(ps[p]) >= decay_min_ndim) ? decay : 0.0f) * sched;for (int64_t j = 0; j < ne; ++j) {float x = ggml_get_f32_1d(ps[p], j);float g_ = g[i]*gnorm;@@ -18777,7 +19059,7 @@ struct gguf_context * gguf_init_from_file(struct llamafile * file, struct gguf_ireturn NULL;}- const size_t size_cur = (ne*ggml_type_size(info->type))/ggml_blck_size(info->type);+ const size_t size_cur = ggml_row_size(info->type, ne);ctx->size += GGML_PAD(size_cur, ctx->alignment);}@@ -19102,6 +19384,10 @@ char * gguf_get_tensor_name(const struct gguf_context * ctx, int i) {return ctx->infos[i].name.data;}+enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int i) {+ return ctx->infos[i].type;+}+// returns the indexstatic int gguf_get_or_add_key(struct gguf_context * ctx, const char * key) {const int idx = gguf_find_key(ctx, key);@@ -19252,7 +19538,7 @@ void gguf_set_kv(struct gguf_context * ctx, struct gguf_context * src) {data[j] = ((struct gguf_str *)src->kv[i].value.arr.data)[j].data;}gguf_set_arr_str(ctx, src->kv[i].key.data, data, src->kv[i].value.arr.n);- free(data);+ free((void *)data);} else if (src->kv[i].value.arr.type == GGUF_TYPE_ARRAY) {GGML_ASSERT(false && "nested arrays not supported");} else {@@ -19277,8 +19563,8 @@ void gguf_add_tensor(ctx->infos[idx].ne[i] = 1;}- ctx->infos[idx].n_dims = tensor->n_dims;- for (int i = 0; i < tensor->n_dims; i++) {+ ctx->infos[idx].n_dims = ggml_n_dims(tensor);+ for (uint32_t i = 0; i < ctx->infos[idx].n_dims; i++) {ctx->infos[idx].ne[i] = tensor->ne[i];}@@ -19535,28 +19821,20 @@ int ggml_cpu_has_avx2(void) {return X86_HAVE(AVX2);}+int ggml_cpu_has_avx_vnni(void) {+ return X86_HAVE(AVXVNNI);+}+int ggml_cpu_has_avx512(void) {-#if defined(__AVX512F__)- return 1;-#else- return 0;-#endif+ return X86_HAVE(AVX512F);}int ggml_cpu_has_avx512_vbmi(void) {-#if defined(__AVX512VBMI__)- return 1;-#else- return 0;-#endif+ return X86_HAVE(AVX512VBMI);}int ggml_cpu_has_avx512_vnni(void) {-#if defined(__AVX512VNNI__)- return 1;-#else- return 0;-#endif+ return X86_HAVE(AVX512_VNNI);}int ggml_cpu_has_fma(void) {diff --git a/llama.cpp/ggml.h b/llama.cpp/ggml.hindex 5119dd92..accc942c 100644--- a/llama.cpp/ggml.h+++ b/llama.cpp/ggml.h@@ -219,7 +219,7 @@#define GGML_QNT_VERSION_FACTOR 1000 // do not change this#define GGML_MAX_DIMS 4-#define GGML_MAX_PARAMS 1024+#define GGML_MAX_PARAMS 2048#define GGML_MAX_CONTEXTS 64#define GGML_MAX_SRC 10#define GGML_MAX_NAME 64@@ -258,6 +258,8 @@#define GGML_UNREACHABLE() GGML_ASSERT(!"statement should not be reached")#elif defined(__GNUC__)#define GGML_UNREACHABLE() __builtin_unreachable()+#elif defined(_MSC_VER)+#define GGML_UNREACHABLE() __assume(0)#else#define GGML_UNREACHABLE() ((void) 0)#endif@@ -306,7 +308,7 @@ extern "C" {#if defined(__ARM_NEON) && defined(__CUDACC__)typedef half ggml_fp16_t;-#elif defined(__ARM_NEON)+#elif defined(__ARM_NEON) && !defined(_MSC_VER)typedef __fp16 ggml_fp16_t;#elsetypedef uint16_t ggml_fp16_t;@@ -487,7 +489,8 @@ extern "C" {enum ggml_log_level {GGML_LOG_LEVEL_ERROR = 2,GGML_LOG_LEVEL_WARN = 3,- GGML_LOG_LEVEL_INFO = 4+ GGML_LOG_LEVEL_INFO = 4,+ GGML_LOG_LEVEL_DEBUG = 5};// ggml object@@ -511,7 +514,6 @@ extern "C" {struct ggml_backend_buffer * buffer;- int n_dims;int64_t ne[GGML_MAX_DIMS]; // number of elementssize_t nb[GGML_MAX_DIMS]; // stride in bytes:// nb[0] = ggml_type_size(type)@@ -543,7 +545,7 @@ extern "C" {void * extra; // extra things e.g. for ggml-cuda.cu- char padding[12];+ char padding[8];};static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor);@@ -649,9 +651,13 @@ extern "C" {GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor);GGML_API size_t ggml_nbytes_pad (const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN- GGML_API int ggml_blck_size (enum ggml_type type);- GGML_API size_t ggml_type_size (enum ggml_type type); // size in bytes for all elements in a block- GGML_API float ggml_type_sizef(enum ggml_type type); // ggml_type_size()/ggml_blck_size() as float+ GGML_API int ggml_blck_size(enum ggml_type type);+ GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block+ GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row++ GGML_DEPRECATED(+ GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float+ "use ggml_row_size() instead");GGML_API const char * ggml_type_name(enum ggml_type type);GGML_API const char * ggml_op_name (enum ggml_op op);@@ -670,6 +676,11 @@ extern "C" {GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor);GGML_API bool ggml_is_contiguous(const struct ggml_tensor * tensor);GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor);+ GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor);+ GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalarsGGML_API bool ggml_are_same_shape(const struct ggml_tensor * t0, const struct ggml_tensor * t1);@@ -730,8 +741,8 @@ extern "C" {GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src);// Context tensor enumeration and lookup- GGML_API struct ggml_tensor * ggml_get_first_tensor(struct ggml_context * ctx);- GGML_API struct ggml_tensor * ggml_get_next_tensor (struct ggml_context * ctx, struct ggml_tensor * tensor);+ GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx);+ GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor);GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name);GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor);@@ -1089,13 +1100,13 @@ extern "C" {GGML_API struct ggml_tensor * ggml_scale(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b);+ float s);// in-place, returns view(a)GGML_API struct ggml_tensor * ggml_scale_inplace(struct ggml_context * ctx,struct ggml_tensor * a,- struct ggml_tensor * b);+ float s);// b -> view(a,offset,nb1,nb2,3), return modified aGGML_API struct ggml_tensor * ggml_set(@@ -2130,10 +2141,11 @@ extern "C" {GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int key_id);GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int key_id, int i);- GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);- GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);- GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);- GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);+ GGML_API int gguf_get_n_tensors (const struct gguf_context * ctx);+ GGML_API int gguf_find_tensor (const struct gguf_context * ctx, const char * name);+ GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int i);+ GGML_API char * gguf_get_tensor_name (const struct gguf_context * ctx, int i);+ GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int i);// overrides existing values or adds a new oneGGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val);@@ -2167,7 +2179,7 @@ extern "C" {//// - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data://- // FILE * f = fopen(fname, "wbe");+ // FILE * f = fopen(fname, "wb");// fseek(f, gguf_get_meta_size(ctx), SEEK_SET);// fwrite(f, ...);// void * data = gguf_meta_get_meta_data(ctx);@@ -2189,6 +2201,7 @@ extern "C" {//GGML_API int ggml_cpu_has_avx (void);+ GGML_API int ggml_cpu_has_avx_vnni (void);GGML_API int ggml_cpu_has_avx2 (void);GGML_API int ggml_cpu_has_avx512 (void);GGML_API int ggml_cpu_has_avx512_vbmi(void);@@ -2208,8 +2221,6 @@ extern "C" {GGML_API int ggml_cpu_has_ssse3 (void);GGML_API int ggml_cpu_has_vsx (void);- GGML_API void ggml_interrupt(bool);-//// Internal types and functions exposed for tests and benchmarks//diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex fc328dcf..0ea6af28 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -2,6 +2,7 @@// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi#define LLAMA_API_INTERNAL+//#define LLAMA_GGML_BACKEND_CUDA_TEST // for testing only - enables ggml-cuda through ggml-backend, disables partial offloading#include "llama.h"#include "unicode.h"@@ -13,15 +14,10 @@#include "ggml-metal.h"#include "ggml-cuda.h"-#ifdef GGML_USE_CUBLAS-# include "ggml-cuda.h"-#elif defined(GGML_USE_CLBLAST)+#if defined(GGML_USE_CLBLAST)# include "ggml-opencl.h"#endif-#ifdef GGML_USE_METAL-# include "ggml-metal.h"-#endif#ifdef GGML_USE_MPI# include "ggml-mpi.h"#endif@@ -33,27 +29,6 @@# endif#endif-#ifdef __has_include- #if __has_include(<unistd.h>)- #include <unistd.h>- #if defined(_POSIX_MAPPED_FILES)- #include <sys/mman.h>- #endif- #if defined(_POSIX_MEMLOCK_RANGE)- #include <sys/resource.h>- #endif- #endif-#endif--#if defined(_WIN32)- #define WIN32_LEAN_AND_MEAN- #ifndef NOMINMAX- #define NOMINMAX- #endif- #include <windows.h>- #include <io.h>-#endif-#include <algorithm>#include <array>#include <cassert>@@ -84,20 +59,11 @@#include <thread>#include <type_traits>#include <unordered_map>+#include <unistd.h>+#include <sys/mman.h>+#include <sys/resource.h>-#if defined(_MSC_VER)-#pragma warning(disable: 4244 4267) // possible loss of data-#endif--#ifdef __GNUC__-#ifdef __MINGW32__-#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))-#else#define LLAMA_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))-#endif-#else-#define LLAMA_ATTRIBUTE_FORMAT(...)-#endif#define LLAMA_MAX_NODES 8192#define LLAMA_MAX_EXPERTS 8@@ -120,7 +86,7 @@ static void llama_log_callback_default(ggml_log_level level, const char * text,void ThrowRuntimeError(std::string message) {#ifndef _LIBCPP_NO_EXCEPTIONS- throw std::runtime_error(message);+ ThrowRuntimeError(message);#elsefprintf(stderr, "error: %s\n", message.c_str());exit(1);@@ -217,6 +183,7 @@ enum llm_arch {LLM_ARCH_STABLELM,LLM_ARCH_QWEN,LLM_ARCH_PHI2,+ LLM_ARCH_PLAMO,LLM_ARCH_UNKNOWN,};@@ -235,6 +202,7 @@ static std::map<llm_arch, std::string> LLM_ARCH_NAMES = {{ LLM_ARCH_STABLELM, "stablelm" },{ LLM_ARCH_QWEN, "qwen" },{ LLM_ARCH_PHI2, "phi2" },+ { LLM_ARCH_PLAMO, "plamo" },};enum llm_kv {@@ -262,6 +230,8 @@ enum llm_kv {LLM_KV_ATTENTION_HEAD_COUNT_KV,LLM_KV_ATTENTION_MAX_ALIBI_BIAS,LLM_KV_ATTENTION_CLAMP_KQV,+ LLM_KV_ATTENTION_KEY_LENGTH,+ LLM_KV_ATTENTION_VALUE_LENGTH,LLM_KV_ATTENTION_LAYERNORM_EPS,LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,@@ -314,6 +284,8 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },{ LLM_KV_ATTENTION_MAX_ALIBI_BIAS, "%s.attention.max_alibi_bias" },{ LLM_KV_ATTENTION_CLAMP_KQV, "%s.attention.clamp_kqv" },+ { LLM_KV_ATTENTION_KEY_LENGTH, "%s.attention.key_length" },+ { LLM_KV_ATTENTION_VALUE_LENGTH, "%s.attention.value_length" },{ LLM_KV_ATTENTION_LAYERNORM_EPS, "%s.attention.layer_norm_epsilon" },{ LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, "%s.attention.layer_norm_rms_epsilon" },@@ -371,6 +343,7 @@ enum llm_tensor {LLM_TENSOR_FFN_GATE,LLM_TENSOR_FFN_DOWN,LLM_TENSOR_FFN_UP,+ LLM_TENSOR_FFN_ACT,LLM_TENSOR_FFN_DOWN_EXP,LLM_TENSOR_FFN_GATE_EXP,LLM_TENSOR_FFN_UP_EXP,@@ -439,6 +412,15 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES =LLM_ARCH_GPT2,{{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_POS_EMBD, "position_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_QKV, "blk.%d.attn_qkv" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },},},{@@ -490,6 +472,7 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ { LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" },},},{@@ -586,6 +569,24 @@ static std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES ={ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },},},+ {+ LLM_ARCH_PLAMO,+ {+ { LLM_TENSOR_TOKEN_EMBD, "token_embd" },+ { LLM_TENSOR_OUTPUT_NORM, "output_norm" },+ { LLM_TENSOR_OUTPUT, "output" },+ { LLM_TENSOR_ROPE_FREQS, "rope_freqs" },+ { LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },+ { LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },+ { LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },+ { LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },+ { LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },+ { LLM_TENSOR_ATTN_ROT_EMBD, "blk.%d.attn_rot_embd" },+ { LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },+ { LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },+ { LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },+ },+ },{LLM_ARCH_UNKNOWN,@@ -733,68 +734,10 @@ static void ggml_graph_compute_helper(std::vector<uint8_t> & buf, ggml_cgraph *// llama helpers//-inline void * llama_host_malloc(size_t n) {- switch (llamafile_gpu_supported()) {- case LLAMAFILE_GPU_APPLE:- return ggml_metal_host_malloc(n);- case LLAMAFILE_GPU_NVIDIA:- return ggml_cuda_host_malloc(n);- default:- return malloc(n);- }-#if GGML_USE_CPU_HBM-#error fix me-#endif-}--inline void llama_host_free(void * ptr) {- switch (llamafile_gpu_supported()) {- case LLAMAFILE_GPU_APPLE:- return ggml_metal_host_free(ptr);- case LLAMAFILE_GPU_NVIDIA:- return ggml_cuda_host_free(ptr);- default:- return free(ptr);- }-#if GGML_USE_CPU_HBM-#error fix me-#endif-}--struct llama_buffer {- void * data = NULL;- size_t size = 0;-- // fallback to malloc / free- // useful in cases where CUDA can try to allocate PINNED memory- bool fallback = false;-- void resize(size_t n) {- llama_host_free(data);-- data = llama_host_malloc(n);- if (!data) {- fallback = true;- data = malloc(n);- } else {- fallback = false;- }-- GGML_ASSERT(data);- size = n;- }-- ~llama_buffer() {- if (data) {- if (fallback) { // NOLINT- free(data);- } else {- llama_host_free(data);- }- }-- data = NULL;- }+template <typename T>+struct no_init {+ T value;+ no_init() { /* do nothing */ }};struct llama_file {@@ -811,7 +754,7 @@ struct llama_file {return llamafile_tell(file);}- void seek(size_t offset, int whence) {+ void seek(size_t offset, int whence) const {return llamafile_seek(file, offset, whence);}@@ -824,7 +767,7 @@ struct llama_file {ThrowRuntimeError(format("read error: %s", strerror(errno)));}if (!rc) {- ThrowRuntimeError(std::string("unexpectedly reached end of file"));+ ThrowRuntimeError("unexpectedly reached end of file");}}@@ -860,6 +803,9 @@ struct llama_mmap {static constexpr bool SUPPORTED = true;+ // list of mapped fragments (first_offset, last_offset)+ std::vector<std::pair<size_t, size_t>> mapped_fragments;+llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1 /* -1 = max value */, bool numa = false) {size = llamafile_size(file->file);if (!llamafile_fp(file->file)) {@@ -900,12 +846,80 @@ struct llama_mmap {if (!llamafile_gpu_supported()) {llamafile_schlep(addr, size);}++ // initialize list of mapped_fragments+ mapped_fragments.emplace_back(0, size);+ }++ static void align_range(size_t * first, size_t * last, size_t page_size) {+ // align first to the next page+ size_t offset_in_page = *first & (page_size - 1);+ size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;+ *first += offset_to_page;++ // align last to the previous page+ *last = *last & ~(page_size - 1);++ if (*last <= *first) {+ *last = *first;+ }+ }++ // partially unmap the file in the range [first, last)+ void unmap_fragment(size_t first, size_t last) {+ // note: this function must not be called multiple times with overlapping ranges+ // otherwise, there is a risk of invalidating addresses that have been repurposed for other mappings+ int page_size = sysconf(_SC_PAGESIZE);+ align_range(&first, &last, page_size);+ size_t len = last - first;++ if (len == 0) {+ return;+ }++ GGML_ASSERT(first % page_size == 0);+ GGML_ASSERT(last % page_size == 0);+ GGML_ASSERT(last > first);++ void * next_page_start = (uint8_t *) addr + first;++ // unmap the range+ if (munmap(next_page_start, len)) {+ LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));+ }++ // update the list of mapped fragments to avoid unmapping the same range again in the destructor+ std::vector<std::pair<size_t, size_t>> new_mapped_fragments;+ for (const auto & frag : mapped_fragments) {+ if (frag.first < first && frag.second > last) {+ // the range is in the middle of the fragment, split it+ new_mapped_fragments.emplace_back(frag.first, first);+ new_mapped_fragments.emplace_back(last, frag.second);+ } else if (frag.first < first && frag.second > first) {+ // the range starts in the middle of the fragment+ new_mapped_fragments.emplace_back(frag.first, first);+ } else if (frag.first < last && frag.second > last) {+ // the range ends in the middle of the fragment+ new_mapped_fragments.emplace_back(last, frag.second);+ } else if (frag.first >= first && frag.second <= last) {+ // the range covers the entire fragment+ } else {+ // the range is outside the fragment+ new_mapped_fragments.push_back(frag);+ }+ }+ mapped_fragments = std::move(new_mapped_fragments);}~llama_mmap() {- if (is_owned) {- munmap(addr, size);+#if 0+ // TODO(jart): make this safe+ for (const auto & frag : mapped_fragments) {+ if (munmap((char *) addr + frag.first, frag.second - frag.first)) {+ LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));+ }}+#endif}};@@ -947,7 +961,6 @@ struct llama_mlock {}}-#ifdef _POSIX_MEMLOCK_RANGEstatic constexpr bool SUPPORTED = true;static size_t lock_granularity() {@@ -990,70 +1003,6 @@ struct llama_mlock {fprintf(stderr, "warning: failed to munlock buffer: %s\n", std::strerror(errno));}}-#elif defined(_WIN32)- static constexpr bool SUPPORTED = true;-- static size_t lock_granularity() {- SYSTEM_INFO si;- GetSystemInfo(&si);- return (size_t) si.dwPageSize;- }-- bool raw_lock(void * ptr, size_t len) const {- for (int tries = 1; ; tries++) {- if (VirtualLock(ptr, len)) {- return true;- }- if (tries == 2) {- fprintf(stderr, "warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",- len, size, llama_format_win_err(GetLastError()).c_str());- return false;- }-- // It failed but this was only the first try; increase the working- // set size and try again.- SIZE_T min_ws_size, max_ws_size;- if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {- fprintf(stderr, "warning: GetProcessWorkingSetSize failed: %s\n",- llama_format_win_err(GetLastError()).c_str());- return false;- }- // Per MSDN: "The maximum number of pages that a process can lock- // is equal to the number of pages in its minimum working set minus- // a small overhead."- // Hopefully a megabyte is enough overhead:- size_t increment = len + 1048576;- // The minimum must be <= the maximum, so we need to increase both:- min_ws_size += increment;- max_ws_size += increment;- if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {- fprintf(stderr, "warning: SetProcessWorkingSetSize failed: %s\n",- llama_format_win_err(GetLastError()).c_str());- return false;- }- }- }-- static void raw_unlock(void * ptr, size_t len) {- if (!VirtualUnlock(ptr, len)) {- fprintf(stderr, "warning: failed to VirtualUnlock buffer: %s\n",- llama_format_win_err(GetLastError()).c_str());- }- }-#else- static constexpr bool SUPPORTED = false;-- static size_t lock_granularity() {- return (size_t) 65536;- }-- bool raw_lock(const void * addr, size_t len) const {- fprintf(stderr, "warning: mlock not supported on this system\n");- return false;- }-- static void raw_unlock(const void * addr, size_t len) {}-#endif};typedef void (*offload_func_t)(struct ggml_tensor * tensor);@@ -1077,11 +1026,45 @@ static std::string llama_token_to_piece(const struct llama_context * ctx, llama_return std::string(result.data(), result.size());}+static ggml_backend_buffer_type_t llama_default_buffer_type(int n_gpu_layers) {+ ggml_backend_buffer_type_t buft = nullptr;++ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {+ if (n_gpu_layers > 0) {+ buft = ggml_backend_metal_buffer_type();+ }+ }++ if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ if (n_gpu_layers > 0) {+ buft = ggml_backend_cuda_buffer_type(0);+ }+#else+ buft = ggml_backend_cuda_host_buffer_type();+#endif+ }++ if (buft == nullptr) {+ buft = ggml_backend_cpu_buffer_type();+ }++ return buft;++ GGML_UNUSED(n_gpu_layers);+}+//// globals//struct llama_state {+ llama_state() {+#ifdef GGML_USE_METAL+ ggml_metal_log_set_callback(log_callback, log_callback_user_data);+#endif+ }+// We save the log callback globallyggml_log_callback log_callback = llama_log_callback_default;void * log_callback_user_data = nullptr;@@ -1103,6 +1086,10 @@ enum e_model {MODEL_40B,MODEL_65B,MODEL_70B,+ MODEL_SMALL,+ MODEL_MEDIUM,+ MODEL_LARGE,+ MODEL_XL,};static const size_t kiB = 1024;@@ -1118,6 +1105,8 @@ struct llama_hparams {uint32_t n_head_kv;uint32_t n_layer;uint32_t n_rot;+ uint32_t n_embd_head_k; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads+ uint32_t n_embd_head_v; // dimension of values (d_v) aka n_embd_headuint32_t n_ff;uint32_t n_expert = 0;uint32_t n_expert_used = 0;@@ -1134,6 +1123,7 @@ struct llama_hparams {float f_clamp_kqv;float f_max_alibi_bias;+bool operator!=(const llama_hparams & other) const {if (this->vocab_only != other.vocab_only) return true;if (this->n_vocab != other.n_vocab) return true;@@ -1143,6 +1133,8 @@ struct llama_hparams {if (this->n_head_kv != other.n_head_kv) return true;if (this->n_layer != other.n_layer) return true;if (this->n_rot != other.n_rot) return true;+ if (this->n_embd_head_k != other.n_embd_head_k) return true;+ if (this->n_embd_head_v != other.n_embd_head_v) return true;if (this->n_ff != other.n_ff) return true;if (this->n_expert != other.n_expert) return true;if (this->n_expert_used != other.n_expert_used) return true;@@ -1150,7 +1142,7 @@ struct llama_hparams {if (this->rope_finetuned != other.rope_finetuned) return true;if (this->n_yarn_orig_ctx != other.n_yarn_orig_ctx) return true;- const float EPSILON = 1e-9;+ const float EPSILON = 1e-9f;if (!is_float_close(this->f_norm_eps, other.f_norm_eps, EPSILON)) return true;if (!is_float_close(this->f_norm_rms_eps, other.f_norm_rms_eps, EPSILON)) return true;@@ -1164,12 +1156,12 @@ struct llama_hparams {return n_head/n_head_kv;}- uint32_t n_embd_head() const {- return n_embd/n_head;+ uint32_t n_embd_k_gqa() const { // dimension of key embeddings across all k-v heads+ return n_embd_head_k * n_head_kv;}- uint32_t n_embd_gqa() const {- return n_embd/n_gqa();+ uint32_t n_embd_v_gqa() const { // dimension of value embeddings across all k-v heads+ return n_embd_head_v * n_head_kv;}};@@ -1237,6 +1229,7 @@ struct llama_layer {// ff biasstruct ggml_tensor * ffn_down_b; // b2struct ggml_tensor * ffn_up_b; // b3+ struct ggml_tensor * ffn_act;};struct llama_kv_cell {@@ -1271,19 +1264,22 @@ struct llama_kv_cache {struct ggml_context * ctx = NULL;- llama_buffer buf;+ ggml_backend_buffer_t buf = NULL;~llama_kv_cache() {- if (ctx) {- ggml_free(ctx);- }-+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {for (size_t i = 0; i < k_l.size(); ++i) {ggml_cuda_free_data(k_l[i]);ggml_cuda_free_data(v_l[i]);}}+#endif+ if (ctx) {+ ggml_free(ctx);+ }++ ggml_backend_buffer_free(buf);}};@@ -1323,11 +1319,11 @@ struct llama_vocab {id special_suffix_id = 32008;id special_eot_id = 32010;- int find_bpe_rank(std::string token_left, std::string token_right) const {- GGML_ASSERT(token_left.find(" ") == std::string::npos);- GGML_ASSERT(token_left.find("\n") == std::string::npos);- GGML_ASSERT(token_right.find(" ") == std::string::npos);- GGML_ASSERT(token_right.find("\n") == std::string::npos);+ int find_bpe_rank(const std::string & token_left, const std::string & token_right) const {+ GGML_ASSERT(token_left.find(' ') == std::string::npos);+ GGML_ASSERT(token_left.find('\n') == std::string::npos);+ GGML_ASSERT(token_right.find(' ') == std::string::npos);+ GGML_ASSERT(token_right.find('\n') == std::string::npos);auto it = bpe_ranks.find(std::make_pair(token_left, token_right));if (it == bpe_ranks.end()) {@@ -1369,7 +1365,7 @@ struct llama_model {struct ggml_context * ctx = NULL;// the model memory buffer- llama_buffer buf;+ ggml_backend_buffer_t buf = NULL;// model memory mapped filestd::unique_ptr<llama_mmap> mapping;@@ -1385,38 +1381,40 @@ struct llama_model {int64_t t_start_us = 0;~llama_model() {- if (ctx) {- ggml_free(ctx);- }-+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {for (size_t i = 0; i < tensors_by_name.size(); ++i) {ggml_cuda_free_data(tensors_by_name[i].second);}ggml_cuda_free_scratch();}+#endif#if defined(GGML_USE_CLBLAST)for (size_t i = 0; i < tensors_by_name.size(); ++i) {ggml_cl_free_data(tensors_by_name[i].second);}#endif+ if (ctx) {+ ggml_free(ctx);+ }++ ggml_backend_buffer_free(buf);}};struct llama_context {llama_context(const llama_model & model) : model(model), t_start_us(model.t_start_us), t_load_us(model.t_load_us) {}~llama_context() {- if (ctx_metal) {- ggml_metal_free(ctx_metal);- }- if (alloc) {- ggml_allocr_free(alloc);- }+ ggml_allocr_free(alloc);+ ggml_backend_buffer_free(buf_alloc);+ ggml_backend_free(backend);}llama_cparams cparams;+ ggml_backend_t backend = nullptr;+const llama_model & model;// key + value cache for the self attention@@ -1438,21 +1436,22 @@ struct llama_context {// decode output (2-dimensional array: [n_tokens][n_vocab])std::vector<float> logits;+#ifndef NDEBUG+ // guard against access to unset logits+ std::vector<bool> logits_valid;+#endifbool logits_all = false;// input embedding (1-dimensional array: [n_embd])std::vector<float> embedding;- // reusable buffer for `struct ggml_graph_plan.work_data`- std::vector<uint8_t> work_buffer;-// memory buffers used to evaluate the model- llama_buffer buf_compute;-- llama_buffer buf_alloc;+ std::vector<uint8_t> buf_compute_meta;+ ggml_backend_buffer_t buf_alloc = NULL;ggml_allocr * alloc = NULL;- ggml_metal_context * ctx_metal = NULL;+ // temporary buffer for copying data to/from the backend+ std::vector<no_init<uint8_t>> buf_copy;#ifdef GGML_USE_MPIggml_mpi_context * ctx_mpi = NULL;@@ -1471,11 +1470,9 @@ static bool llama_kv_cache_init(uint32_t n_ctx,int n_gpu_layers,bool offload) {- const uint32_t n_embd = hparams.n_embd_gqa();- const uint32_t n_layer = hparams.n_layer;-- const int64_t n_mem = n_layer*n_ctx;- const int64_t n_elements = n_embd*n_mem;+ const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa();+ const uint32_t n_layer = hparams.n_layer;cache.has_shift = false;@@ -1486,13 +1483,10 @@ static bool llama_kv_cache_init(cache.cells.clear();cache.cells.resize(n_ctx);- cache.buf.resize(n_elements*(ggml_type_sizef(ktype) + ggml_type_sizef(vtype)) + 2u*n_layer*ggml_tensor_overhead());- memset(cache.buf.data, 0, cache.buf.size);-struct ggml_init_params params;- params.mem_size = cache.buf.size;- params.mem_buffer = cache.buf.data;- params.no_alloc = false;+ params.mem_size = 2u*n_layer*ggml_tensor_overhead();+ params.mem_buffer = NULL;+ params.no_alloc = true;cache.ctx = ggml_init(params);@@ -1506,34 +1500,44 @@ static bool llama_kv_cache_init(cache.k_l.reserve(n_layer);cache.v_l.reserve(n_layer);- const int i_gpu_start = (int) n_layer - n_gpu_layers; GGML_UNUSED(i_gpu_start);-- GGML_UNUSED(offload);+ const int i_gpu_start = (int) n_layer - n_gpu_layers;for (int i = 0; i < (int) n_layer; i++) {- ggml_tensor * k = ggml_new_tensor_1d(cache.ctx, ktype, n_embd*n_ctx);- ggml_tensor * v = ggml_new_tensor_1d(cache.ctx, vtype, n_embd*n_ctx);+ ggml_tensor * k = ggml_new_tensor_1d(cache.ctx, ktype, n_embd_k_gqa*n_ctx);+ ggml_tensor * v = ggml_new_tensor_1d(cache.ctx, vtype, n_embd_v_gqa*n_ctx);ggml_format_name(k, "cache_k_l%d", i);ggml_format_name(v, "cache_v_l%d", i);cache.k_l.push_back(k);cache.v_l.push_back(v);- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (i >= i_gpu_start) {if (offload) {ggml_cuda_assign_buffers_no_scratch(k);- vram_kv_cache += ggml_nbytes(k);ggml_cuda_assign_buffers_no_scratch(v);+ vram_kv_cache += ggml_nbytes(k);vram_kv_cache += ggml_nbytes(v);+ // HACK: mark tensor as allocated+ k->data = v->data = (void *)(uintptr_t)1;}}- }+#endif // GGML_USE_CUBLAS+ }++ // allocate tensors+ cache.buf = ggml_backend_alloc_ctx_tensors_from_buft(cache.ctx, llama_default_buffer_type(n_gpu_layers));++ // buf may be NULL with full offload+ if (cache.buf) {+ // initialize the buffer to avoid NaNs in the padding+ ggml_backend_buffer_clear(cache.buf, 0);}if (vram_kv_cache > 0) {LLAMA_LOG_INFO("%s: VRAM kv self = %.2f MB\n", __func__, vram_kv_cache / 1024.0 / 1024.0);}- GGML_UNUSED(n_gpu_layers);+ GGML_UNUSED(i_gpu_start);+ GGML_UNUSED(offload);return true;}@@ -1864,7 +1868,7 @@ namespace GGUFMeta {target = override->bool_value;return true;}- return true;+ return false;}template<typename OT>@@ -1985,17 +1989,16 @@ struct llama_model_loader {enum ggml_type type_max = GGML_TYPE_F32;for (int i = 0; i < n_tensors; i++) {- const char * name = gguf_get_tensor_name(ctx_gguf, i);- struct ggml_tensor * meta = ggml_get_tensor(ctx_meta, name);+ enum ggml_type type = gguf_get_tensor_type(ctx_gguf, i);- n_type[meta->type]++;+ n_type[type]++;- if (n_type_max < n_type[meta->type]) {- n_type_max = n_type[meta->type];- type_max = meta->type;+ if (n_type_max < n_type[type]) {+ n_type_max = n_type[type];+ type_max = type;}- LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, name, ggml_type_name(meta->type), llama_format_tensor_shape(meta).c_str());+ // LLAMA_LOG_INFO("%s: - tensor %4d: %32s %-8s [ %s ]\n", __func__, i, name, ggml_type_name(meta->type), llama_format_tensor_shape(meta).c_str());}switch (type_max) {@@ -2133,34 +2136,19 @@ struct llama_model_loader {return gguf_get_tensor_name(ctx_gguf, i);}- struct ggml_tensor * get_tensor_meta(int i) const {- return ggml_get_tensor(ctx_meta, get_tensor_name(i));+ struct ggml_tensor * get_tensor_meta(const char * name) const {+ return ggml_get_tensor(ctx_meta, name);}- void calc_sizes(size_t & ctx_size_p, size_t & mmapped_size_p) const {- ctx_size_p = 0;- mmapped_size_p = 0;-- for (int i = 0; i < n_tensors; i++) {- struct ggml_tensor * meta = get_tensor_meta(i);- ctx_size_p += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE;- (use_mmap ? mmapped_size_p : ctx_size_p) += ggml_nbytes_pad(meta);- }+ struct ggml_tensor * get_tensor_meta(int i) const {+ return get_tensor_meta(get_tensor_name(i));}struct ggml_tensor * create_tensor_for(struct ggml_context * ctx, struct ggml_tensor * meta, ggml_backend_type backend) {- if (backend != GGML_BACKEND_CPU) {- ggml_set_no_alloc(ctx, true);- }-struct ggml_tensor * tensor = ggml_dup_tensor(ctx, meta);tensor->backend = backend; // TODO: ggml_set_backendggml_set_name(tensor, ggml_get_name(meta));- if (backend != GGML_BACKEND_CPU) {- ggml_set_no_alloc(ctx, use_mmap);- }-n_created++;return tensor;@@ -2218,90 +2206,144 @@ struct llama_model_loader {return gguf_get_data_offset(ctx_gguf) + gguf_get_tensor_offset(ctx_gguf, idx);}- void load_data_for(struct ggml_tensor * cur) {- const size_t offs = file_offset(ggml_get_name(cur));+ void init_mapping(bool prefetch = true) {+ /*+ // prefetch only CPU tensors+ if (use_mmap) {+ size_t size_pref = 0; // prefetch+ for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {+ struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));+ if (cur->backend == GGML_BACKEND_CPU) {+ size_t tensor_end = gguf_get_tensor_offset(ctx_gguf, i) + ggml_nbytes(cur);+ size_pref = std::max(size_pref, tensor_end);+ }+ }+ mapping.reset(new llama_mmap(&file, gguf_get_data_offset(ctx_gguf) + size_pref, ggml_is_numa()));+ }+ */+ // prefetch the whole file - all the data is needed anywayif (use_mmap) {- cur->data = (uint8_t *) mapping->addr + offs;+ mapping.reset(new llama_mmap(&file, prefetch ? -1 : 0, ggml_is_numa()));+ }+ }++ // for backwards compatibility, does not support ggml-backend+ void load_data_for(struct ggml_tensor * cur) const {+ const size_t offs = file_offset(ggml_get_name(cur));++ if (use_mmap && mapping) {+ GGML_ASSERT(cur->data == nullptr);+ cur->data = (uint8_t *)mapping->addr + offs;} else {+ GGML_ASSERT(cur->data != nullptr);file.seek(offs, SEEK_SET);file.read_raw(cur->data, ggml_nbytes(cur));}}- void load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, llama_mlock * lmlock) {+ // Returns false if cancelled by progress_callback+ bool load_all_data(struct ggml_context * ctx, llama_progress_callback progress_callback, void * progress_callback_user_data, ggml_backend_buffer_t buf_mmap, llama_mlock * lmlock) const {size_t size_data = 0;- size_t size_lock = 0;- size_t size_pref = 0; // prefetchfor (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));size_data += ggml_nbytes(cur);- if (cur->backend == GGML_BACKEND_CPU) {- size_pref += ggml_nbytes(cur);- }}- if (use_mmap) {- mapping.reset(new llama_mmap(&file, size_pref, ggml_is_numa()));+ if (use_mmap && buf_mmap) {if (lmlock) {lmlock->init(mapping->addr);}}- size_t done_size = 0;+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ const bool legacy_offload = llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA;+#else+ const bool legacy_offload = false;+#endif++ std::vector<no_init<uint8_t>> read_buf;++ size_t size_done = 0;++ size_t mmap_first = -1;+ size_t mmap_last = 0;+for (int i = 0; i < gguf_get_n_tensors(ctx_gguf); i++) {struct ggml_tensor * cur = ggml_get_tensor(ctx, gguf_get_tensor_name(ctx_gguf, i));GGML_ASSERT(cur); // unused tensors should have been caught by load_data alreadyif (progress_callback) {- progress_callback((float) done_size / size_data, progress_callback_user_data);- }-- // allocate temp buffer if not using mmap- if (!use_mmap && cur->data == NULL) {- GGML_ASSERT(cur->backend != GGML_BACKEND_CPU);- #ifdef GGML_USE_CPU_HBM- cur->data = (uint8_t*)hbw_malloc(ggml_nbytes(cur));- #else- cur->data = (uint8_t*)malloc(ggml_nbytes(cur));- #endif+ if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {+ return false;+ }}- load_data_for(cur);+ const size_t offs = file_offset(ggml_get_name(cur));- switch (cur->backend) {- case GGML_BACKEND_CPU:- if (use_mmap && lmlock) {- size_lock += ggml_nbytes(cur);- lmlock->grow_to(size_lock);- }- break;- case GGML_BACKEND_GPU:- case GGML_BACKEND_GPU_SPLIT:- // old code:- //ggml_cuda_transform_tensor(lt.data, lt.ggml_tensor);-- // TODO: test if this works !!- ggml_cuda_transform_tensor(cur->data, cur);- if (!use_mmap) {- free(cur->data);+ if (!legacy_offload || cur->backend == GGML_BACKEND_CPU) {+ if (use_mmap && mapping) {+ if (buf_mmap) {+ ggml_backend_tensor_alloc(buf_mmap, cur, (uint8_t *) mapping->addr + offs);+ if (lmlock) {+ lmlock->grow_to(offs + ggml_nbytes(cur));+ }+ mmap_first = std::min(mmap_first, offs);+ mmap_last = std::max(mmap_last, offs + ggml_nbytes(cur));+ } else {+ ggml_backend_tensor_set(cur, (uint8_t *) mapping->addr + offs, 0, ggml_nbytes(cur));}- break;-#if defined(GGML_USE_CLBLAST)- case GGML_BACKEND_GPU:- ggml_cl_transform_tensor(cur->data, cur);- if (!use_mmap) {- free(cur->data);+ } else {+ if (ggml_backend_buffer_is_host(cur->buffer)) {+ file.seek(offs, SEEK_SET);+ file.read_raw(cur->data, ggml_nbytes(cur));+ } else {+ read_buf.resize(ggml_nbytes(cur));+ file.seek(offs, SEEK_SET);+ file.read_raw(read_buf.data(), ggml_nbytes(cur));+ ggml_backend_tensor_set(cur, read_buf.data(), 0, ggml_nbytes(cur));}- break;+ }+ } else {+ // HACK: mark tensor as allocated+ cur->data = (void *)(uintptr_t)1;+ void * data;+ if (use_mmap && mapping) {+ data = (uint8_t *) mapping->addr + offs;+ } else {+ read_buf.resize(ggml_nbytes(cur));+ file.seek(offs, SEEK_SET);+ file.read_raw(read_buf.data(), ggml_nbytes(cur));+ data = read_buf.data();+ }++#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ ggml_cuda_transform_tensor(data, cur);+#elif defined(GGML_USE_CLBLAST)+ GGML_ASSERT(cur->backend == GGML_BACKEND_GPU);+ ggml_cl_transform_tensor(data, cur);+#else+ GGML_ASSERT(!"GPU tensor without a GPU backend");+ GGML_UNUSED(data);#endif- default:- continue;}- done_size += ggml_nbytes(cur);+ size_done += ggml_nbytes(cur);+ }++ // unmap offloaded tensors and metadata+ if (use_mmap && mapping) {+ mapping->unmap_fragment(0, mmap_first);+ mapping->unmap_fragment(mmap_last, mapping->size);+ }++ if (progress_callback) {+ // Even though the model is done loading, we still honor+ // cancellation since we need to free allocations.+ return progress_callback(1.0f, progress_callback_user_data);}+ return true;}};@@ -2324,25 +2366,25 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {switch (ftype) {case LLAMA_FTYPE_ALL_F32: return "all F32";- case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16";- case LLAMA_FTYPE_MOSTLY_Q4_0: return "mostly Q4_0";- case LLAMA_FTYPE_MOSTLY_Q4_1: return "mostly Q4_1";+ case LLAMA_FTYPE_MOSTLY_F16: return "F16";+ case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";+ case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";case LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:- return "mostly Q4_1, some F16";- case LLAMA_FTYPE_MOSTLY_Q5_0: return "mostly Q5_0";- case LLAMA_FTYPE_MOSTLY_Q5_1: return "mostly Q5_1";- case LLAMA_FTYPE_MOSTLY_Q8_0: return "mostly Q8_0";+ return "Q4_1, some F16";+ case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";+ case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";+ case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";// K-quants- case LLAMA_FTYPE_MOSTLY_Q2_K: return "mostly Q2_K";- case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "mostly Q3_K - Small";- case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "mostly Q3_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "mostly Q3_K - Large";- case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "mostly Q4_K - Small";- case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "mostly Q4_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "mostly Q5_K - Small";- case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "mostly Q5_K - Medium";- case LLAMA_FTYPE_MOSTLY_Q6_K: return "mostly Q6_K";+ case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K";+ case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";+ case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";+ case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";+ case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";+ case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";+ case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";+ case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";+ case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";default: return "unknown, may not work";}@@ -2350,18 +2392,22 @@ static std::string llama_model_ftype_name(llama_ftype ftype) {static const char * llama_model_type_name(e_model type) {switch (type) {- case MODEL_1B: return "1B";- case MODEL_3B: return "3B";- case MODEL_7B: return "7B";- case MODEL_8B: return "8B";- case MODEL_13B: return "13B";- case MODEL_15B: return "15B";- case MODEL_30B: return "30B";- case MODEL_34B: return "34B";- case MODEL_40B: return "40B";- case MODEL_65B: return "65B";- case MODEL_70B: return "70B";- default: return "?B";+ case MODEL_1B: return "1B";+ case MODEL_3B: return "3B";+ case MODEL_7B: return "7B";+ case MODEL_8B: return "8B";+ case MODEL_13B: return "13B";+ case MODEL_15B: return "15B";+ case MODEL_30B: return "30B";+ case MODEL_34B: return "34B";+ case MODEL_40B: return "40B";+ case MODEL_65B: return "65B";+ case MODEL_70B: return "70B";+ case MODEL_SMALL: return "0.1B";+ case MODEL_MEDIUM: return "0.4B";+ case MODEL_LARGE: return "0.8B";+ case MODEL_XL: return "1.5B";+ default: return "?B";}}@@ -2453,6 +2499,12 @@ static void llm_load_hparams(// gpt-j n_rot = rotary_dim}+ hparams.n_embd_head_k = hparams.n_embd / hparams.n_head;+ ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false);++ hparams.n_embd_head_v = hparams.n_embd / hparams.n_head;+ ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);+// arch-specific KVsswitch (model.arch) {case LLM_ARCH_LLAMA:@@ -2460,6 +2512,7 @@ static void llm_load_hparams(ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);switch (hparams.n_layer) {+ case 22: model.type = e_model::MODEL_1B; break;case 26: model.type = e_model::MODEL_3B; break;case 32: model.type = e_model::MODEL_7B; break;case 40: model.type = e_model::MODEL_13B; break;@@ -2570,6 +2623,26 @@ static void llm_load_hparams(default: model.type = e_model::MODEL_UNKNOWN;}} break;+ case LLM_ARCH_PLAMO:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);++ switch (hparams.n_layer) {+ case 40: model.type = e_model::MODEL_13B; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;+ case LLM_ARCH_GPT2:+ {+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);+ switch (hparams.n_layer) {+ case 12: model.type = e_model::MODEL_SMALL; break;+ case 24: model.type = e_model::MODEL_MEDIUM; break;+ case 36: model.type = e_model::MODEL_LARGE; break;+ case 48: model.type = e_model::MODEL_XL; break;+ default: model.type = e_model::MODEL_UNKNOWN;+ }+ } break;default: (void)0;}@@ -2736,7 +2809,7 @@ static void llm_load_vocab(// The assumption is, since special tokens aren't meant to be exposed to end user, they are designed// to be unmatchable by the tokenizer, therefore tokens from the vocab, which are unmatchable by the tokenizer// are special tokens.- // From testing, this appears to corelate 1:1 with special tokens.+ // From testing, this appears to correlate 1:1 with special tokens.//// Counting special tokens and verifying in only one direction@@ -2842,8 +2915,12 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {LLAMA_LOG_INFO("%s: n_head = %u\n", __func__, hparams.n_head);LLAMA_LOG_INFO("%s: n_head_kv = %u\n", __func__, hparams.n_head_kv);LLAMA_LOG_INFO("%s: n_layer = %u\n", __func__, hparams.n_layer);- LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot); // a.k.a. n_embd_head, n_head_dim+ LLAMA_LOG_INFO("%s: n_rot = %u\n", __func__, hparams.n_rot);+ LLAMA_LOG_INFO("%s: n_embd_head_k = %u\n", __func__, hparams.n_embd_head_k);+ LLAMA_LOG_INFO("%s: n_embd_head_v = %u\n", __func__, hparams.n_embd_head_v);LLAMA_LOG_INFO("%s: n_gqa = %u\n", __func__, hparams.n_gqa());+ LLAMA_LOG_INFO("%s: n_embd_k_gqa = %u\n", __func__, hparams.n_embd_k_gqa());+ LLAMA_LOG_INFO("%s: n_embd_v_gqa = %u\n", __func__, hparams.n_embd_v_gqa());LLAMA_LOG_INFO("%s: f_norm_eps = %.1e\n", __func__, hparams.f_norm_eps);LLAMA_LOG_INFO("%s: f_norm_rms_eps = %.1e\n", __func__, hparams.f_norm_rms_eps);LLAMA_LOG_INFO("%s: f_clamp_kqv = %.1e\n", __func__, hparams.f_clamp_kqv);@@ -2877,7 +2954,8 @@ static void llm_load_print_meta(llama_model_loader & ml, llama_model & model) {if (vocab.linefeed_id != -1) { LLAMA_LOG_INFO( "%s: LF token = %d '%s'\n", __func__, vocab.linefeed_id, vocab.id_to_token[vocab.linefeed_id].text.c_str() ); }}-static void llm_load_tensors(+// Returns false if cancelled by progress_callback+static bool llm_load_tensors(llama_model_loader & ml,llama_model & model,int n_gpu_layers,@@ -2893,25 +2971,16 @@ static void llm_load_tensors(model.n_gpu_layers = n_gpu_layers;- size_t ctx_size;- size_t mmapped_size;+ size_t ctx_size = ggml_tensor_overhead() * ml.n_tensors;- ml.calc_sizes(ctx_size, mmapped_size);-- LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, ctx_size/1024.0/1024.0);+ LLAMA_LOG_INFO("%s: ggml ctx size = %7.2f MiB\n", __func__, ctx_size/1024.0/1024.0);// create the ggml context{- model.buf.resize(ctx_size);- if (use_mlock) {- model.mlock_buf.init (model.buf.data);- model.mlock_buf.grow_to(model.buf.size);- }-struct ggml_init_params params = {- /*.mem_size =*/ model.buf.size,- /*.mem_buffer =*/ model.buf.data,- /*.no_alloc =*/ ml.use_mmap,+ /*.mem_size =*/ ctx_size,+ /*.mem_buffer =*/ NULL,+ /*.no_alloc =*/ true,};model.ctx = ggml_init(params);@@ -2925,26 +2994,27 @@ static void llm_load_tensors(enum ggml_backend_type llama_backend_offload = GGML_BACKEND_CPU;enum ggml_backend_type llama_backend_offload_split = GGML_BACKEND_CPU;+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {LLAMA_LOG_INFO("%s: using " GGML_CUDA_NAME " for GPU acceleration\n", __func__);ggml_cuda_set_main_device(main_gpu);- llama_backend_offload = GGML_BACKEND_GPU;+ llama_backend_offload = GGML_BACKEND_GPU;llama_backend_offload_split = GGML_BACKEND_GPU_SPLIT;}-#if defined(GGML_USE_CLBLAST)+#elif defined(GGML_USE_CLBLAST)LLAMA_LOG_INFO("%s: using OpenCL for GPU acceleration\n", __func__);- llama_backend_offload = GGML_BACKEND_GPU;+ llama_backend_offload = GGML_BACKEND_GPU;llama_backend_offload_split = GGML_BACKEND_GPU;#endif- // prepare memory for the weights- size_t vram_weights = 0;+ // create tensors for the weights{- const int64_t n_embd = hparams.n_embd;- const int64_t n_embd_gqa = hparams.n_embd_gqa();- const int64_t n_layer = hparams.n_layer;- const int64_t n_vocab = hparams.n_vocab;+ const int64_t n_embd = hparams.n_embd;+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();+ const int64_t n_layer = hparams.n_layer;+ const int64_t n_vocab = hparams.n_vocab;const auto tn = LLM_TN(model.arch);switch (model.arch) {@@ -2968,16 +3038,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3024,28 +3090,6 @@ static void llm_load_tensors(layer.ffn_up_exp[x] = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP_EXP, "weight", i, x), {n_embd, n_ff}, backend_split);}}-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) +- ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) +- (layer.bq ? ggml_nbytes(layer.bq) : 0) +- (layer.bk ? ggml_nbytes(layer.bk) : 0) +- (layer.bv ? ggml_nbytes(layer.bv) : 0) +- (layer.bo ? ggml_nbytes(layer.bo) : 0) +- ggml_nbytes(layer.ffn_norm);-- if (layer.ffn_gate_inp == nullptr) {- vram_weights +=- ggml_nbytes(layer.ffn_gate) + ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);- } else {- vram_weights += ggml_nbytes(layer.ffn_gate_inp);- for (uint32_t x = 0; x < hparams.n_expert; ++x) {- vram_weights +=- ggml_nbytes(layer.ffn_gate_exp[x]) + ggml_nbytes(layer.ffn_down_exp[x]) + ggml_nbytes(layer.ffn_up_exp[x]);- }- }- }}} break;case LLM_ARCH_BAICHUAN:@@ -3065,16 +3109,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3098,19 +3138,10 @@ static void llm_load_tensors(layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) +- ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) +- ggml_nbytes(layer.ffn_gate) + ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);- }}} break;case LLM_ARCH_FALCON:{- // TODO: CPU-only for now-model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);// output@@ -3129,17 +3160,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- vram_weights += ggml_nbytes(model.output_norm_b);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3157,11 +3183,6 @@ static void llm_load_tensors(if (gguf_find_tensor(ml.ctx_gguf, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i).c_str()) >= 0) {layer.attn_norm_2 = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "weight", i), {n_embd}, backend);layer.attn_norm_2_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM_2, "bias", i), {n_embd}, backend);-- if (backend == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(layer.attn_norm_2);- vram_weights += ggml_nbytes(layer.attn_norm_2_b);- }}layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);@@ -3169,13 +3190,6 @@ static void llm_load_tensors(layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.attn_norm_b) +- ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.wo) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);- }}} break;case LLM_ARCH_STARCODER:@@ -3199,17 +3213,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- vram_weights += ggml_nbytes(model.output_norm_b);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3238,16 +3247,6 @@ static void llm_load_tensors(layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.attn_norm_b) +- ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.bqkv) +- ggml_nbytes(layer.wo) + ggml_nbytes(layer.bo) +- ggml_nbytes(layer.ffn_norm) + ggml_nbytes(layer.ffn_norm_b) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_down_b) +- ggml_nbytes(layer.ffn_up) + ggml_nbytes(layer.ffn_up_b);- }}} break;case LLM_ARCH_PERSIMMON:@@ -3269,17 +3268,13 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- vram_weights += ggml_nbytes(model.output_norm_b);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);+const int i_gpu_start = n_layer - n_gpu_layers;model.layers.resize(n_layer);for (uint32_t i = 0; i < n_layer; ++i) {@@ -3306,8 +3301,6 @@ static void llm_load_tensors(} break;case LLM_ARCH_BLOOM:{- // TODO: CPU-only for now-model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);model.tok_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight"), {n_embd}, GGML_BACKEND_CPU);model.tok_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias"), {n_embd}, GGML_BACKEND_CPU);@@ -3328,17 +3321,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- vram_weights += ggml_nbytes(model.output_norm_b);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3367,22 +3355,11 @@ static void llm_load_tensors(layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.attn_norm_b) +- ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.bqkv) +- ggml_nbytes(layer.wo) + ggml_nbytes(layer.bo) +- ggml_nbytes(layer.ffn_norm) + ggml_nbytes(layer.ffn_norm_b) +- ggml_nbytes(layer.ffn_up) + ggml_nbytes(layer.ffn_up_b) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_down_b);- }}} break;case LLM_ARCH_MPT:{model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);-// output{ggml_backend_type backend_norm;@@ -3398,16 +3375,12 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3428,15 +3401,8 @@ static void llm_load_tensors(layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) +- ggml_nbytes(layer.wqkv) +- ggml_nbytes(layer.wo) +- ggml_nbytes(layer.ffn_norm) +- ggml_nbytes(layer.ffn_down) +- ggml_nbytes(layer.ffn_up);- }+ // AWQ ScaleActivation layer+ layer.ffn_act = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_ACT, "scales", i), {n_ff}, backend, false);}} break;case LLM_ARCH_STABLELM:@@ -3459,16 +3425,12 @@ static void llm_load_tensors(model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3497,13 +3459,6 @@ static void llm_load_tensors(layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.wq) + ggml_nbytes(layer.wk) +- ggml_nbytes(layer.wv) + ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) +- ggml_nbytes(layer.ffn_gate) + ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);- }}} break;case LLM_ARCH_QWEN:@@ -3523,14 +3478,7 @@ static void llm_load_tensors(model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- }- if (backend_output == GGML_BACKEND_GPU_SPLIT) {- vram_weights += ggml_nbytes(model.output);- }- }+ }const uint32_t n_ff = hparams.n_ff / 2;@@ -3555,13 +3503,6 @@ static void llm_load_tensors(layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);-- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.bqkv) +- ggml_nbytes(layer.wo) + ggml_nbytes(layer.ffn_norm) + ggml_nbytes(layer.ffn_gate) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_up);- }}} break;case LLM_ARCH_PHI2:@@ -3585,16 +3526,12 @@ static void llm_load_tensors(model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);model.output_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}, backend_output);-- if (backend_norm == GGML_BACKEND_GPU) {- vram_weights += ggml_nbytes(model.output_norm);- vram_weights += ggml_nbytes(model.output_norm_b);- vram_weights += ggml_nbytes(model.output);- vram_weights += ggml_nbytes(model.output_b);- }}- const uint32_t n_ff = hparams.n_ff;+ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);const int i_gpu_start = n_layer - n_gpu_layers;@@ -3620,15 +3557,111 @@ static void llm_load_tensors(layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);+ }+ } break;+ case LLM_ARCH_PLAMO:+ {+ model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);++ // output+ {+ ggml_backend_type backend_norm;+ ggml_backend_type backend_output;++ if (n_gpu_layers > int(n_layer)) {+ backend_norm = llama_backend_offload;+ backend_output = llama_backend_offload_split;+ } else {+ backend_norm = GGML_BACKEND_CPU;+ backend_output = GGML_BACKEND_CPU;+ }++ model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);+ model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ }++ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);++ const int i_gpu_start = n_layer - n_gpu_layers;++ model.layers.resize(n_layer);++ for (uint32_t i = 0; i < n_layer; ++i) {+ const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT+ const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);++ layer.wq = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, backend_split);+ layer.wk = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, backend_split);+ layer.wv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, backend_split);+ layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);++ layer.ffn_gate = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, backend_split);+ layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ }+ } break;+ case LLM_ARCH_GPT2:+ {+ model.tok_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);+ model.pos_embd = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU);++ // output+ {+ ggml_backend_type backend_norm;+ ggml_backend_type backend_output;- if (backend == GGML_BACKEND_GPU) {- vram_weights +=- ggml_nbytes(layer.attn_norm) + ggml_nbytes(layer.attn_norm_b) +- ggml_nbytes(layer.wqkv) + ggml_nbytes(layer.bqkv) +- ggml_nbytes(layer.wo) + ggml_nbytes(layer.bo) +- ggml_nbytes(layer.ffn_up) + ggml_nbytes(layer.ffn_up_b) +- ggml_nbytes(layer.ffn_down) + ggml_nbytes(layer.ffn_down_b);+ if (n_gpu_layers > int(n_layer)) {+ backend_norm = llama_backend_offload;+ backend_output = llama_backend_offload_split;+ } else {+ backend_norm = GGML_BACKEND_CPU;+ backend_output = GGML_BACKEND_CPU;}++ model.output_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, backend_norm);+ model.output_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, backend_norm);+ model.output = ml.create_tensor(ctx, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, backend_output);+ }++ const uint32_t n_ff = hparams.n_ff;+ const int64_t n_embd_gqa = n_embd_v_gqa;+ GGML_ASSERT(n_embd_gqa == n_embd / hparams.n_gqa());+ GGML_ASSERT(n_embd_gqa == n_embd_k_gqa);++ const int i_gpu_start = n_layer - n_gpu_layers;++ model.layers.resize(n_layer);++ for (uint32_t i = 0; i < n_layer; ++i) {+ const ggml_backend_type backend = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload; // NOLINT+ const ggml_backend_type backend_split = int(i) < i_gpu_start ? GGML_BACKEND_CPU : llama_backend_offload_split; // NOLINT++ auto & layer = model.layers[i];++ layer.attn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, backend);+ layer.attn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_NORM, "bias", i), {n_embd}, backend);++ layer.wqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, backend_split);+ layer.bqkv = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, backend);++ layer.wo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, backend_split);+ layer.bo = ml.create_tensor(ctx, tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, backend);++ layer.ffn_norm = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, backend);+ layer.ffn_norm_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_NORM, "bias", i), {n_embd}, backend);++ layer.ffn_down = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, backend_split);+ layer.ffn_down_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, backend);++ layer.ffn_up = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, backend_split);+ layer.ffn_up_b = ml.create_tensor(ctx, tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, backend);}} break;default:@@ -3638,15 +3671,81 @@ static void llm_load_tensors(ml.done_getting_tensors();+ ml.init_mapping();++ // allocate tensors+ size_t vram_weights = 0;+ size_t buf_size = 0;++ ggml_backend_buffer_type_t buft = llama_default_buffer_type(n_gpu_layers);++ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {+ // GGML_BACKEND_GPU tensors are for CUDA and OpenCL only, which are handled separately without ggml-backend+ if (t->backend == GGML_BACKEND_CPU) {+ buf_size += GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), ggml_backend_buft_get_alignment(buft));+ } else {+ vram_weights += ggml_nbytes(t);+ }+ }++ // create backend buffer+ ggml_backend_buffer_t buf_mmap = nullptr;++ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {+ if (n_gpu_layers > 0) {+ if (ml.use_mmap) {+ const size_t max_size = ggml_get_max_tensor_size(ctx);+ model.buf = ggml_backend_metal_buffer_from_ptr(ml.mapping->addr, ml.mapping->size, max_size);+ buf_mmap = model.buf;+ } else {+ model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_metal_buffer_type());+ }+ }+ }+#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+ // for testing only+ if (n_gpu_layers > 0) {+ model.buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx, ggml_backend_cuda_buffer_type(0));+ }+ }+#endif++ if (model.buf == nullptr) {+ // CPU backend, and indirectly CUDA and OpenCL+ if (ml.use_mmap) {+ model.buf = ggml_backend_cpu_buffer_from_ptr(ml.mapping->addr, ml.mapping->size);+ buf_mmap = model.buf;+ } else {+ // allocate only CPU tensors+ model.buf = ggml_backend_buft_alloc_buffer(buft, buf_size);+ ggml_tallocr_t alloc = ggml_tallocr_new_from_buffer(model.buf);+ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) {+ if (t->backend == GGML_BACKEND_CPU) {+ ggml_tallocr_alloc(alloc, t);+ }+ }+ ggml_tallocr_free(alloc);+ }+ }++ if (use_mlock && ggml_backend_buffer_is_host(model.buf)) {+ model.mlock_buf.init (ggml_backend_buffer_get_base(model.buf));+ model.mlock_buf.grow_to(ggml_backend_buffer_get_size(model.buf));+ }+// print memory requirements{- // this is the total memory required to run the inference- size_t mem_required =- ctx_size +- mmapped_size - vram_weights; // weights in VRAM not in memory+ size_t sys_mem_required = ctx_size + buf_size;- LLAMA_LOG_INFO("%s: mem required = %7.2f MiB\n", __func__, mem_required / 1024.0 / 1024.0);+ if (sys_mem_required > 0) {+ LLAMA_LOG_INFO("%s: system memory used = %7.2f MiB\n", __func__, sys_mem_required / 1024.0 / 1024.0);+ }+ if (vram_weights > 0) {+ LLAMA_LOG_INFO("%s: VRAM used = %7.2f MiB\n", __func__, vram_weights / 1024.0 / 1024.0);+ }+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {const int n_gpu = std::min(n_gpu_layers, int(hparams.n_layer));@@ -3659,26 +3758,24 @@ static void llm_load_tensors(const int max_offloadable_layers = hparams.n_layer + 1;LLAMA_LOG_INFO("%s: offloaded %d/%d layers to GPU\n", __func__, std::min(n_gpu_layers, max_offloadable_layers), max_backend_supported_layers);- LLAMA_LOG_INFO("%s: VRAM used: %.2f MiB\n", __func__, vram_weights / 1024.0 / 1024.0);}+#endif // defined(GGML_USE_CUBLAS) || defined(GGML_USE_CLBLAST)}- // populate `tensors_by_name`+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ ggml_cuda_set_tensor_split(tensor_split);+#else+ GGML_UNUSED(tensor_split);+#endif // GGML_USE_CUBLAS++ // populate tensors_by_namefor (int i = 0; i < ml.n_tensors; ++i) {struct ggml_tensor * cur = ggml_get_tensor(ctx, ml.get_tensor_name(i));model.tensors_by_name.emplace_back(ggml_get_name(cur), cur);}- (void) tensor_split;- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA)- {- ggml_cuda_set_tensor_split(tensor_split);- }-- ml.load_all_data(ctx, progress_callback, progress_callback_user_data, use_mlock ? &model.mlock_mmap : NULL);-- if (progress_callback) {- progress_callback(1.0f, progress_callback_user_data);+ if (!ml.load_all_data(ctx, progress_callback, progress_callback_user_data, buf_mmap, use_mlock ? &model.mlock_mmap : NULL)) {+ return false;}model.mapping = std::move(ml.mapping);@@ -3686,9 +3783,11 @@ static void llm_load_tensors(// loading time will be recalculate after the first eval, so// we take page faults deferred by mmap() into considerationmodel.t_load_us = ggml_time_us() - model.t_start_us;+ return true;}-static bool llama_model_load(const std::string & fname, llama_model & model, const llama_model_params & params) {+// Returns 0 on success, -1 on error, and -2 on cancellation via llama_progress_callback+static int llama_model_load(const std::string & fname, llama_model & model, const llama_model_params & params) {#ifndef _LIBCPP_NO_EXCEPTIONStry {#endif@@ -3708,21 +3807,23 @@ static bool llama_model_load(const std::string & fname, llama_model & model, conif (params.vocab_only) {LLAMA_LOG_INFO("%s: vocab only - skipping tensors\n", __func__);- return true;+ return 0;}- llm_load_tensors(+ if (!llm_load_tensors(ml, model, params.n_gpu_layers, params.main_gpu, params.tensor_split, params.use_mlock,params.progress_callback, params.progress_callback_user_data- );+ )) {+ return -2;+ }#ifndef _LIBCPP_NO_EXCEPTIONS} catch (const std::exception & err) {LLAMA_LOG_ERROR("error loading model: %s\n", err.what());- return false;+ return -1;}#endif- return true;+ return 0;}//@@ -3780,8 +3881,8 @@ static struct ggml_tensor * llm_build_inp_embd(return inpL;}-// Persimmon: n_rot = n_embd_head/2-// Other: n_rot = n_embd_head+// Persimmon: n_rot = n_embd_head_k/2+// Other: n_rot = n_embd_head_kstatic void llm_build_k_shift(struct ggml_context * ctx,const llama_hparams & hparams,@@ -3794,17 +3895,17 @@ static void llm_build_k_shift(float freq_base,float freq_scale,const llm_build_cb & cb) {- const int64_t n_layer = hparams.n_layer;- const int64_t n_head_kv = hparams.n_head_kv;- const int64_t n_embd_gqa = hparams.n_embd_gqa();- const int64_t n_embd_head = hparams.n_embd_head();- const int32_t n_orig_ctx = cparams.n_yarn_orig_ctx;- const float ext_factor = cparams.yarn_ext_factor;- const float attn_factor = cparams.yarn_attn_factor;- const float beta_fast = cparams.yarn_beta_fast;- const float beta_slow = cparams.yarn_beta_slow;-- GGML_ASSERT(n_embd_head % n_rot == 0);+ const int64_t n_layer = hparams.n_layer;+ const int64_t n_head_kv = hparams.n_head_kv;+ const int64_t n_embd_head_k = hparams.n_embd_head_k;+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int32_t n_orig_ctx = cparams.n_yarn_orig_ctx;+ const float ext_factor = cparams.yarn_ext_factor;+ const float attn_factor = cparams.yarn_attn_factor;+ const float beta_fast = cparams.yarn_beta_fast;+ const float beta_slow = cparams.yarn_beta_slow;++ GGML_ASSERT(n_embd_head_k % n_rot == 0);struct ggml_tensor * K_shift = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_ctx);cb(K_shift, "K_shift", -1);@@ -3822,9 +3923,9 @@ static void llm_build_k_shift(// we rotate only the first n_rot dimensionsggml_rope_custom_inplace(ctx,ggml_view_3d(ctx, kv.k_l[il],- n_embd_head, n_head_kv, n_ctx,- ggml_type_sizef(kv.k_l[il]->type)*n_embd_head,- ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa,+ n_embd_head_k, n_head_kv, n_ctx,+ ggml_row_size(kv.k_l[il]->type, n_embd_head_k),+ ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa),0),K_shift, n_rot, rope_type, 0, n_orig_ctx, freq_base, freq_scale,ext_factor, attn_factor, beta_fast, beta_slow);@@ -3845,18 +3946,19 @@ static void llm_build_kv_store(int32_t kv_head,const llm_build_cb & cb,int64_t il) {- const int64_t n_embd_gqa = hparams.n_embd_gqa();+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa();// compute the transposed [n_tokens, n_embd] V matrix- struct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, n_embd_gqa, n_tokens));+ struct ggml_tensor * v_cur_t = ggml_transpose(ctx, ggml_reshape_2d(ctx, v_cur, n_embd_v_gqa, n_tokens));//struct ggml_tensor * v_cur_t = ggml_transpose(ctx, v_cur); // TODO: reshape above is likely not neededcb(v_cur_t, "v_cur_t", il);- struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_gqa,- (ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa)*kv_head);+ struct ggml_tensor * k_cache_view = ggml_view_1d(ctx, kv.k_l[il], n_tokens*n_embd_k_gqa,+ (ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa))*kv_head);cb(k_cache_view, "k_cache_view", il);- struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_gqa,+ struct ggml_tensor * v_cache_view = ggml_view_2d(ctx, kv.v_l[il], n_tokens, n_embd_v_gqa,( n_ctx)*ggml_element_size(kv.v_l[il]),(kv_head)*ggml_element_size(kv.v_l[il]));cb(v_cache_view, "v_cache_view", il);@@ -3907,6 +4009,7 @@ static struct ggml_tensor * llm_build_ffn(struct ggml_tensor * gate_b,struct ggml_tensor * down,struct ggml_tensor * down_b,+ struct ggml_tensor * act_scales,llm_ffn_op_type type_op,llm_ffn_gate_type type_gate,const llm_build_cb & cb,@@ -3951,6 +4054,10 @@ static struct ggml_tensor * llm_build_ffn({cur = ggml_gelu(ctx, cur);cb(cur, "ffn_gelu", il);+ if (act_scales != NULL) {+ cur = ggml_div(ctx, cur, act_scales);+ cb(cur, "ffn_act", il);+ }} break;case LLM_FFN_RELU:{@@ -3993,29 +4100,28 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * wo,struct ggml_tensor * wo_b,struct ggml_tensor * q_cur,- struct ggml_tensor * kq_scale,struct ggml_tensor * kq_mask,int64_t n_ctx,int32_t n_tokens,int32_t n_kv,float max_alibi_bias,- float scale,+ float kq_scale,const llm_build_cb & cb,int il) {- const int64_t n_embd = hparams.n_embd;- const int64_t n_head = hparams.n_head;- const int64_t n_head_kv = hparams.n_head_kv;- const int64_t n_embd_head = hparams.n_embd_head();- const int64_t n_embd_gqa = hparams.n_embd_gqa();+ const int64_t n_head = hparams.n_head;+ const int64_t n_head_kv = hparams.n_head_kv;+ const int64_t n_embd_head_k = hparams.n_embd_head_k;+ const int64_t n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int64_t n_embd_head_v = hparams.n_embd_head_v;struct ggml_tensor * q = ggml_permute(ctx, q_cur, 0, 2, 1, 3);cb(q, "q", il);struct ggml_tensor * k =ggml_view_3d(ctx, kv.k_l[il],- n_embd_head, n_kv, n_head_kv,- ggml_type_sizef(kv.k_l[il]->type)*n_embd_gqa,- ggml_type_sizef(kv.k_l[il]->type)*n_embd_head,+ n_embd_head_k, n_kv, n_head_kv,+ ggml_row_size(kv.k_l[il]->type, n_embd_k_gqa),+ ggml_row_size(kv.k_l[il]->type, n_embd_head_k),0);cb(k, "k", il);@@ -4047,16 +4153,16 @@ static struct ggml_tensor * llm_build_kqv(kq = ggml_soft_max(ctx, kq);cb(kq, "kq_soft_max", il);} else {- kq = ggml_soft_max_ext(ctx, kq, kq_mask, scale);+ kq = ggml_soft_max_ext(ctx, kq, kq_mask, kq_scale);cb(kq, "kq_soft_max_ext", il);}// split cached v into n_head headsstruct ggml_tensor * v =ggml_view_3d(ctx, kv.v_l[il],- n_kv, n_embd_head, n_head_kv,+ n_kv, n_embd_head_v, n_head_kv,ggml_element_size(kv.v_l[il])*n_ctx,- ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head,+ ggml_element_size(kv.v_l[il])*n_ctx*n_embd_head_v,0);cb(v, "v", il);@@ -4066,7 +4172,7 @@ static struct ggml_tensor * llm_build_kqv(struct ggml_tensor * kqv_merged = ggml_permute(ctx, kqv, 0, 2, 1, 3);cb(kqv_merged, "kqv_merged", il);- struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, n_embd, n_tokens);+ struct ggml_tensor * cur = ggml_cont_2d(ctx, kqv_merged, n_embd_head_k*n_head, n_tokens);cb(cur, "kqv_merged_cont", il);cur = ggml_mul_mat(ctx, wo, cur);@@ -4093,8 +4199,10 @@ struct llm_build_context {const int64_t n_ctx; // user-specified context size (can be different from n_ctx_train)const int64_t n_head;const int64_t n_head_kv;- const int64_t n_embd_head;- const int64_t n_embd_gqa;+ const int64_t n_embd_head_k;+ const int64_t n_embd_k_gqa;+ const int64_t n_embd_head_v;+ const int64_t n_embd_v_gqa;const int64_t n_expert;const int64_t n_expert_used;@@ -4116,7 +4224,7 @@ struct llm_build_context {const llm_build_cb & cb;- llama_buffer & buf_compute;+ std::vector<uint8_t> & buf_compute_meta;struct ggml_context * ctx0 = nullptr;@@ -4126,35 +4234,37 @@ struct llm_build_context {const llama_batch & batch,const llm_build_cb & cb,bool worst_case) :- model (lctx.model),- hparams (model.hparams),- cparams (lctx.cparams),- batch (batch),- kv_self (lctx.kv_self),- n_embd (hparams.n_embd),- n_layer (hparams.n_layer),- n_ctx (cparams.n_ctx),- n_head (hparams.n_head),- n_head_kv (hparams.n_head_kv),- n_embd_head (hparams.n_embd_head()),- n_embd_gqa (hparams.n_embd_gqa()),- n_expert (hparams.n_expert),- n_expert_used (hparams.n_expert_used),- freq_base (cparams.rope_freq_base),- freq_scale (cparams.rope_freq_scale),- ext_factor (cparams.yarn_ext_factor),- attn_factor (cparams.yarn_attn_factor),- beta_fast (cparams.yarn_beta_fast),- beta_slow (cparams.yarn_beta_slow),- norm_eps (hparams.f_norm_eps),- norm_rms_eps (hparams.f_norm_rms_eps),- n_tokens (batch.n_tokens),- n_kv (worst_case ? n_ctx : kv_self.n),- kv_head (worst_case ? n_ctx - n_tokens : kv_self.head),- n_orig_ctx (cparams.n_yarn_orig_ctx),- do_rope_shift (worst_case || kv_self.has_shift),- cb (cb),- buf_compute (lctx.buf_compute) {+ model (lctx.model),+ hparams (model.hparams),+ cparams (lctx.cparams),+ batch (batch),+ kv_self (lctx.kv_self),+ n_embd (hparams.n_embd),+ n_layer (hparams.n_layer),+ n_ctx (cparams.n_ctx),+ n_head (hparams.n_head),+ n_head_kv (hparams.n_head_kv),+ n_embd_head_k (hparams.n_embd_head_k),+ n_embd_k_gqa (hparams.n_embd_k_gqa()),+ n_embd_head_v (hparams.n_embd_head_v),+ n_embd_v_gqa (hparams.n_embd_v_gqa()),+ n_expert (hparams.n_expert),+ n_expert_used (hparams.n_expert_used),+ freq_base (cparams.rope_freq_base),+ freq_scale (cparams.rope_freq_scale),+ ext_factor (cparams.yarn_ext_factor),+ attn_factor (cparams.yarn_attn_factor),+ beta_fast (cparams.yarn_beta_fast),+ beta_slow (cparams.yarn_beta_slow),+ norm_eps (hparams.f_norm_eps),+ norm_rms_eps (hparams.f_norm_rms_eps),+ n_tokens (batch.n_tokens),+ n_kv (worst_case ? n_ctx : kv_self.n),+ kv_head (worst_case ? n_ctx - n_tokens : kv_self.head),+ n_orig_ctx (cparams.n_yarn_orig_ctx),+ do_rope_shift (worst_case || kv_self.has_shift),+ cb (cb),+ buf_compute_meta (lctx.buf_compute_meta) {GGML_ASSERT(!!kv_self.ctx);// all initializations should be done in init()@@ -4162,8 +4272,8 @@ struct llm_build_context {void init() {struct ggml_init_params params = {- /*.mem_size =*/ buf_compute.size,- /*.mem_buffer =*/ buf_compute.data,+ /*.mem_size =*/ buf_compute_meta.size(),+ /*.mem_buffer =*/ buf_compute_meta.data(),/*.no_alloc =*/ true,};@@ -4180,6 +4290,8 @@ struct llm_build_context {struct ggml_cgraph * build_llama() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);GGML_ASSERT(n_embd_head == hparams.n_rot);struct ggml_tensor * cur;@@ -4192,10 +4304,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4256,7 +4364,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4274,6 +4382,7 @@ struct llm_build_context {model.layers[il].ffn_up, NULL,model.layers[il].ffn_gate, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);} else {@@ -4367,6 +4476,9 @@ struct llm_build_context {struct ggml_cgraph * build_baichuan() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4377,10 +4489,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4439,7 +4547,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4457,6 +4565,7 @@ struct llm_build_context {model.layers[il].ffn_up, NULL,model.layers[il].ffn_gate, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);}@@ -4487,6 +4596,11 @@ struct llm_build_context {struct ggml_cgraph * build_falcon() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4497,10 +4611,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4563,7 +4673,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4575,6 +4685,7 @@ struct llm_build_context {model.layers[il].ffn_up, NULL,NULL, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}@@ -4609,6 +4720,11 @@ struct llm_build_context {struct ggml_cgraph * build_starcoder() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;struct ggml_tensor * pos;struct ggml_tensor * inpL;@@ -4620,10 +4736,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4663,7 +4775,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4683,6 +4795,7 @@ struct llm_build_context {model.layers[il].ffn_up, model.layers[il].ffn_up_b,NULL, NULL,model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}@@ -4708,7 +4821,12 @@ struct llm_build_context {struct ggml_cgraph * build_persimmon() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);- const int64_t n_rot = n_embd_head / 2;+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);++ const int64_t n_rot = n_embd_head_k / 2;struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -4720,10 +4838,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4872,7 +4986,7 @@ struct llm_build_context {// TODO: not tested, could be brokencur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, model.layers[il].bo,- Q, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Q, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4891,6 +5005,7 @@ struct llm_build_context {model.layers[il].ffn_up, model.layers[il].ffn_up_b,NULL, NULL,model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,LLM_FFN_RELU_SQR, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}@@ -4920,16 +5035,17 @@ struct llm_build_context {struct ggml_cgraph * build_refact() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;struct ggml_tensor * inpL;inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);cb(inpL, "inp_embd", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -4963,7 +5079,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -4981,6 +5097,7 @@ struct llm_build_context {model.layers[il].ffn_up, NULL,model.layers[il].ffn_gate, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);}@@ -5011,16 +5128,17 @@ struct llm_build_context {struct ggml_cgraph * build_bloom() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;struct ggml_tensor * inpL;inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);cb(inpL, "inp_embd", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -5060,7 +5178,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, 8.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5080,6 +5198,7 @@ struct llm_build_context {model.layers[il].ffn_up, model.layers[il].ffn_up_b,NULL, NULL,model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}@@ -5105,16 +5224,17 @@ struct llm_build_context {struct ggml_cgraph * build_mpt() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;struct ggml_tensor * inpL;inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);cb(inpL, "inp_embd", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -5154,7 +5274,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, hparams.f_max_alibi_bias, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5169,11 +5289,11 @@ struct llm_build_context {NULL,LLM_NORM, cb, il);cb(cur, "ffn_norm", il);-cur = llm_build_ffn(ctx0, cur,model.layers[il].ffn_up, NULL,NULL, NULL,model.layers[il].ffn_down, NULL,+ model.layers[il].ffn_act,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);cb(cur, "ffn_out", il);}@@ -5204,6 +5324,9 @@ struct llm_build_context {struct ggml_cgraph * build_stablelm() {struct ggml_cgraph * gf = ggml_new_graph(ctx0);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -5214,10 +5337,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -5267,7 +5386,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5286,6 +5405,7 @@ struct llm_build_context {model.layers[il].ffn_up, NULL,model.layers[il].ffn_gate, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);}@@ -5317,6 +5437,9 @@ struct llm_build_context {struct ggml_cgraph * build_qwen() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+struct ggml_tensor * cur;struct ggml_tensor * inpL;@@ -5327,10 +5450,6 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);@@ -5384,7 +5503,7 @@ struct llm_build_context {cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, NULL,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}@@ -5399,14 +5518,238 @@ struct llm_build_context {cb(cur, "ffn_norm", il);cur = llm_build_ffn(ctx0, cur,- model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_up, NULL,+ model.layers[il].ffn_gate, NULL,+ model.layers[il].ffn_down, NULL,+ NULL,+ LLM_FFN_SILU, LLM_FFN_PAR, cb, il);+ cb(cur, "ffn_out", il);+ }++ cur = ggml_add(ctx0, cur, ffn_inp);+ cb(cur, "l_out", il);++ // input for next layer+ inpL = cur;+ }++ cur = inpL;++ cur = llm_build_norm(ctx0, cur, hparams,+ model.output_norm, NULL,+ LLM_NORM_RMS, cb, -1);+ cb(cur, "result_norm", -1);++ // lm_head+ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }+ struct ggml_cgraph * build_phi2() {+ struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);++ struct ggml_tensor * cur;+ struct ggml_tensor * attn_norm_output;+ struct ggml_tensor * ffn_output;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {+ attn_norm_output = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm,+ model.layers[il].attn_norm_b,+ LLM_NORM, cb, il);+ cb(attn_norm_output, "attn_norm", il);++ // self-attention+ {+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output);+ cb(cur, "wqkv", il);++ cur = ggml_add(ctx0, cur, model.layers[il].bqkv);+ cb(cur, "bqkv", il);++ struct ggml_tensor * Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd, n_tokens, cur->nb[1], 0*sizeof(float)*(n_embd)));+ struct ggml_tensor * Kcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd)));+ struct ggml_tensor * Vcur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, n_embd_gqa, n_tokens, cur->nb[1], 1*sizeof(float)*(n_embd + n_embd_gqa)));++ cb(Qcur, "Qcur", il);+ cb(Kcur, "Kcur", il);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);++ Qcur = ggml_rope_custom(+ ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Qcur, "Qcur", il);++ // with phi2, we scale the Q to avoid precision issues+ // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66+ Qcur = ggml_scale(ctx0, Qcur, 1.0f/sqrtf(float(n_embd_head)));+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,+ freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow+ );+ cb(Kcur, "Kcur", il);++ llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);++ cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ model.layers[il].wo, model.layers[il].bo,+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f, cb, il);+ cb(cur, "kqv_out", il);+ }++ // FF+ {+ ffn_output = llm_build_ffn(ctx0, attn_norm_output,+ model.layers[il].ffn_up, model.layers[il].ffn_up_b,+ NULL, NULL,+ model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,+ LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);+ cb(ffn_output, "ffn_out", il);+ }++ cur = ggml_add(ctx0, cur, ffn_output);+ cb(cur, "l_out", il);++ cur = ggml_add(ctx0, cur, inpL);+ cb(cur, "l_out", il);++ inpL = cur;+ }++ cur = llm_build_norm(ctx0, inpL, hparams,+ model.output_norm,+ model.output_norm_b,+ LLM_NORM, cb, -1);+ cb(cur, "result_norm", -1);++ cur = ggml_mul_mat(ctx0, model.output, cur);+ cb(cur, "result_output_no_bias", -1);++ cur = ggml_add(ctx0, cur, model.output_b);+ cb(cur, "result_output", -1);++ ggml_build_forward_expand(gf, cur);++ return gf;+ }++ struct ggml_cgraph * build_plamo() {+ struct ggml_cgraph * gf = ggml_new_graph(ctx0);++ const int64_t n_embd_head = hparams.n_embd_head_v;+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);++ struct ggml_tensor * cur;+ struct ggml_tensor * inpL;++ inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);+ cb(inpL, "inp_embd", -1);++ // inp_pos - contains the positions+ struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);+ cb(inp_pos, "inp_pos", -1);++ // KQ_mask (mask for 1 head, it will be broadcasted to all heads)+ struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);+ cb(KQ_mask, "KQ_mask", -1);++ // shift the entire K-cache if needed+ if (do_rope_shift) {+ llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE, n_ctx, n_embd_head, freq_base, freq_scale, cb);+ }++ for (int il = 0; il < n_layer; ++il) {++ // norm+ cur = llm_build_norm(ctx0, inpL, hparams,+ model.layers[il].attn_norm, NULL,+ LLM_NORM_RMS, cb, il);+ cb(cur, "attn_norm", il);++ struct ggml_tensor * attention_norm = cur;++ // self-attention+ {+ // compute Q and K and RoPE them+ struct ggml_tensor * Qcur = ggml_mul_mat(ctx0, model.layers[il].wq, cur);+ cb(Qcur, "Qcur", il);++ struct ggml_tensor * Kcur = ggml_mul_mat(ctx0, model.layers[il].wk, cur);+ cb(Kcur, "Kcur", il);++ struct ggml_tensor * Vcur = ggml_mul_mat(ctx0, model.layers[il].wv, cur);+ cb(Vcur, "Vcur", il);++ Qcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos,+ n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ cb(Qcur, "Qcur", il);++ Kcur = ggml_rope_custom(+ ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos,+ n_embd_head, 2, 0, n_orig_ctx, freq_base, freq_scale,+ ext_factor, attn_factor, beta_fast, beta_slow);+ cb(Kcur, "Kcur", il);++ llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);++ cur = llm_build_kqv(ctx0, model, hparams, kv_self,+ model.layers[il].wo, NULL,+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);+ cb(cur, "kqv_out", il);+ }+ struct ggml_tensor * sa_out = cur;++ cur = attention_norm;++ // feed-forward network+ {+ cur = llm_build_ffn(ctx0, cur,+ model.layers[il].ffn_up, NULL,model.layers[il].ffn_gate, NULL,model.layers[il].ffn_down, NULL,+ NULL,LLM_FFN_SILU, LLM_FFN_PAR, cb, il);cb(cur, "ffn_out", il);}- cur = ggml_add(ctx0, cur, ffn_inp);+ cur = ggml_add(ctx0, cur, sa_out);+ cb(cur, "l_out", il);++ cur = ggml_add(ctx0, cur, inpL);cb(cur, "l_out", il);// input for next layer@@ -5428,12 +5771,17 @@ struct llm_build_context {return gf;}- struct ggml_cgraph * build_phi2() {++ struct ggml_cgraph * build_gpt2() {struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, LLAMA_MAX_NODES, false);+ const int64_t n_embd_head = hparams.n_embd_head_v;+ const int64_t n_embd_gqa = hparams.n_embd_v_gqa();+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);+ GGML_ASSERT(n_embd_gqa == n_embd);+struct ggml_tensor * cur;- struct ggml_tensor * attn_norm_output;- struct ggml_tensor * ffn_output;+ struct ggml_tensor * pos;struct ggml_tensor * inpL;inpL = llm_build_inp_embd(ctx0, hparams, batch, model.tok_embd, cb);@@ -5443,33 +5791,26 @@ struct llm_build_context {struct ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);cb(inp_pos, "inp_pos", -1);- // Q_scale- struct ggml_tensor * Q_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(Q_scale, "Q_scale", -1);-- // KQ_scale- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- cb(KQ_scale, "KQ_scale", -1);-// KQ_mask (mask for 1 head, it will be broadcasted to all heads)struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, n_tokens, 1);cb(KQ_mask, "KQ_mask", -1);- // shift the entire K-cache if needed- if (do_rope_shift) {- llm_build_k_shift(ctx0, hparams, cparams, kv_self, gf, LLM_ROPE_NEOX, n_ctx, n_embd_head, freq_base, freq_scale, cb);- }+ pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);+ cb(pos, "pos_embd", -1);++ inpL = ggml_add(ctx0, inpL, pos);+ cb(inpL, "inpL", -1);for (int il = 0; il < n_layer; ++il) {- attn_norm_output = llm_build_norm(ctx0, inpL, hparams,+ cur = llm_build_norm(ctx0, inpL, hparams,model.layers[il].attn_norm,model.layers[il].attn_norm_b,LLM_NORM, cb, il);- cb(attn_norm_output, "attn_norm", il);+ cb(cur, "attn_norm", il);// self-attention{- cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, attn_norm_output);+ cur = ggml_mul_mat(ctx0, model.layers[il].wqkv, cur);cb(cur, "wqkv", il);cur = ggml_add(ctx0, cur, model.layers[il].bqkv);@@ -5483,49 +5824,39 @@ struct llm_build_context {cb(Kcur, "Kcur", il);cb(Vcur, "Vcur", il);- Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);- Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);-- Qcur = ggml_rope_custom(- ctx0, Qcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow- );- cb(Qcur, "Qcur", il);-- Qcur = ggml_scale(ctx0, Qcur, Q_scale);- cb(Qcur, "Qcur", il);-- Kcur = ggml_rope_custom(- ctx0, Kcur, inp_pos, hparams.n_rot, 2, 0, n_orig_ctx,- freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow- );- cb(Kcur, "Kcur", il);+ Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens);llm_build_kv_store(ctx0, hparams, kv_self, gf, Kcur, Vcur, n_ctx, n_tokens, kv_head, cb, il);cur = llm_build_kqv(ctx0, model, hparams, kv_self,model.layers[il].wo, model.layers[il].bo,- Qcur, KQ_scale, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f, cb, il);+ Qcur, KQ_mask, n_ctx, n_tokens, n_kv, -1.0f, 1.0f/sqrtf(float(n_embd_head)), cb, il);cb(cur, "kqv_out", il);}+ // add the input+ struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpL);+ cb(ffn_inp, "ffn_inp", il);+// FF{- ffn_output = llm_build_ffn(ctx0, attn_norm_output,+ cur = llm_build_norm(ctx0, ffn_inp, hparams,+ model.layers[il].ffn_norm,+ model.layers[il].ffn_norm_b,+ LLM_NORM, cb, il);+ cb(cur, "ffn_norm", il);++ cur = llm_build_ffn(ctx0, cur,model.layers[il].ffn_up, model.layers[il].ffn_up_b,NULL, NULL,model.layers[il].ffn_down, model.layers[il].ffn_down_b,+ NULL,LLM_FFN_GELU, LLM_FFN_SEQ, cb, il);- cb(ffn_output, "ffn_out", il);+ cb(cur, "ffn_out", il);}- cur = ggml_add(ctx0, cur, ffn_output);- cb(cur, "l_out", il);-- cur = ggml_add(ctx0, cur, inpL);- cb(cur, "l_out", il);-- inpL = cur;+ inpL = ggml_add(ctx0, cur, ffn_inp);+ cb(inpL, "l_out", il);}cur = llm_build_norm(ctx0, inpL, hparams,@@ -5535,9 +5866,6 @@ struct llm_build_context {cb(cur, "result_norm", -1);cur = ggml_mul_mat(ctx0, model.output, cur);- cb(cur, "result_output_no_bias", -1);-- cur = ggml_add(ctx0, cur, model.output_b);cb(cur, "result_output", -1);ggml_build_forward_expand(gf, cur);@@ -5642,8 +5970,6 @@ static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map{ "pos_embd", OFFLOAD_FUNC_NR },{ "inp_pos", OFFLOAD_FUNC_FRC }, // this is often used for KQ ops (e.g. rope)- { "Q_scale", OFFLOAD_FUNC_FRC },- { "KQ_scale", OFFLOAD_FUNC_FRC },{ "KQ_mask", OFFLOAD_FUNC_FRC },{ "K_shift", OFFLOAD_FUNC_FRC },@@ -5701,6 +6027,7 @@ static const std::unordered_map<const char *, llm_offload_func_e> k_offload_map{ "ffn_gate", OFFLOAD_FUNC },{ "ffn_gate_b", OFFLOAD_FUNC },{ "ffn_gate_par", OFFLOAD_FUNC },+ { "ffn_act", OFFLOAD_FUNC },{ "ffn_down", OFFLOAD_FUNC },{ "ffn_down_b", OFFLOAD_FUNC },{ "ffn_out", OFFLOAD_FUNC },@@ -5745,12 +6072,14 @@ static struct ggml_cgraph * llama_build_graph(bool alloc_inp_tokens = false;bool alloc_inp_embd = false;bool alloc_inp_pos = false;- bool alloc_inp_Q_scale = false;- bool alloc_inp_KQ_scale = false;bool alloc_inp_KQ_mask = false;bool alloc_inp_K_shift = false;+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)const bool do_offload = true;+#else+ const bool do_offload = true; // TODO: set to false after finishing refactoring+#endifint n_non_view = 0; // number of non-view tensors that have been processed by the callback@@ -5774,7 +6103,7 @@ static struct ggml_cgraph * llama_build_graph(if (!ggml_allocr_is_measure(lctx.alloc) && batch.token) {const int64_t n_tokens = cur->ne[0];- memcpy(cur->data, batch.token, n_tokens*ggml_element_size(cur));+ ggml_backend_tensor_set(cur, batch.token, 0, n_tokens*ggml_element_size(cur));}alloc_inp_tokens = true;@@ -5787,7 +6116,7 @@ static struct ggml_cgraph * llama_build_graph(const int64_t n_embd = cur->ne[0];const int64_t n_tokens = cur->ne[1];- memcpy(cur->data, batch.embd, n_tokens*n_embd*ggml_element_size(cur));+ ggml_backend_tensor_set(cur, batch.embd, 0, n_tokens*n_embd*ggml_element_size(cur));}alloc_inp_embd = true;@@ -5799,44 +6128,13 @@ static struct ggml_cgraph * llama_build_graph(if (!ggml_allocr_is_measure(lctx.alloc) && batch.pos) {const int64_t n_tokens = cur->ne[0];- int32_t * data = (int32_t *) cur->data;-- for (int i = 0; i < n_tokens; ++i) {- data[i] = batch.pos[i];- }+ static_assert(std::is_same<llama_pos, int32_t>::value, "llama_pos must be int32_t");+ ggml_backend_tensor_set(cur, batch.pos, 0, n_tokens*ggml_element_size(cur));}alloc_inp_pos = true;}- if (!alloc_inp_Q_scale && strcmp(name, "Q_scale") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc)) {- const int64_t n_embd_head = model.hparams.n_embd_head();- ggml_set_f32(cur, 1.0f/sqrtf(float(n_embd_head)));- }-- alloc_inp_Q_scale = true;- }-- if (!alloc_inp_KQ_scale && strcmp(name, "KQ_scale") == 0) {- ggml_allocr_alloc(lctx.alloc, cur);-- if (!ggml_allocr_is_measure(lctx.alloc)) {- const int64_t n_embd_head = model.hparams.n_embd_head();- if (model.arch == LLM_ARCH_PHI2) {- // with phi2, we scale the Q to avoid precision issues- // ref: https://github.com/ml-explore/mlx-examples/blob/08e862336ade809bc37d1035f94b359e7d1a5152/phi2/phi2.py#L64-L66- ggml_set_f32(cur, 1.0f);- } else {- ggml_set_f32(cur, 1.0f/sqrtf(float(n_embd_head)));- }- }-- alloc_inp_KQ_scale = true;- }-if (!alloc_inp_KQ_mask && strcmp(name, "KQ_mask") == 0) {ggml_allocr_alloc(lctx.alloc, cur);@@ -5844,8 +6142,13 @@ static struct ggml_cgraph * llama_build_graph(const int64_t n_kv = cur->ne[0];const int64_t n_tokens = cur->ne[1];- float * data = (float *) cur->data;- memset(data, 0, ggml_nbytes(cur));+ float * data;+ if (ggml_backend_buffer_is_host(cur->buffer)) {+ data = (float *) cur->data;+ } else {+ lctx.buf_copy.resize(ggml_nbytes(cur));+ data = (float *) lctx.buf_copy.data();+ }for (int h = 0; h < 1; ++h) {for (int j = 0; j < n_tokens; ++j) {@@ -5853,12 +6156,20 @@ static struct ggml_cgraph * llama_build_graph(const llama_seq_id seq_id = batch.seq_id[j][0];for (int i = 0; i < n_kv; ++i) {+ float f;if (!lctx.kv_self.cells[i].has_seq_id(seq_id) || lctx.kv_self.cells[i].pos > pos) {- data[h*(n_kv*n_tokens) + j*n_kv + i] = -INFINITY;+ f = -INFINITY;+ } else {+ f = 0;}+ data[h*(n_kv*n_tokens) + j*n_kv + i] = f;}}}++ if (data != cur->data) {+ ggml_backend_tensor_set(cur, data, 0, ggml_nbytes(cur));+ }}alloc_inp_KQ_mask = true;@@ -5870,11 +6181,21 @@ static struct ggml_cgraph * llama_build_graph(if (!ggml_allocr_is_measure(lctx.alloc)) {const int64_t n_ctx = cur->ne[0];- int32_t * data = (int32_t *) cur->data;+ int32_t * data;+ if (ggml_backend_buffer_is_host(cur->buffer)) {+ data = (int32_t *) cur->data;+ } else {+ lctx.buf_copy.resize(ggml_nbytes(cur));+ data = (int32_t *) lctx.buf_copy.data();+ }for (int i = 0; i < n_ctx; ++i) {data[i] = lctx.kv_self.cells[i].delta;}++ if (data != cur->data) {+ ggml_backend_tensor_set(cur, data, 0, ggml_nbytes(cur));+ }}alloc_inp_K_shift = true;@@ -5911,11 +6232,19 @@ static struct ggml_cgraph * llama_build_graph(static const std::unordered_map<llm_offload_func_e, std::string, std::hash<int>> k_offload_func_name = {{ OFFLOAD_FUNC_NOP, "CPU" },{ OFFLOAD_FUNC_OUT, "CPU" },+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST){ OFFLOAD_FUNC, "GPU (CUDA)" },{ OFFLOAD_FUNC_FRC, "GPU (CUDA) FRC" },{ OFFLOAD_FUNC_KQV, "GPU (CUDA) KQV" },{ OFFLOAD_FUNC_NR, "GPU (CUDA) NR" },{ OFFLOAD_FUNC_EMB, "GPU (CUDA) EMB" },+#else+ { OFFLOAD_FUNC, "CPU" },+ { OFFLOAD_FUNC_FRC, "CPU" },+ { OFFLOAD_FUNC_KQV, "CPU" },+ { OFFLOAD_FUNC_NR, "CPU" },+ { OFFLOAD_FUNC_EMB, "CPU" },+#endif // GGML_USE_CUBLAS};// check the global map for what offload function to use for this tensor@@ -5976,12 +6305,11 @@ static struct ggml_cgraph * llama_build_graph(offload_func_t func = ggml_offload_nop;// this is needed for compatibility with Metal for example- static offload_func_t ggml_offload_gpu;- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {- ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc;- } else {- ggml_offload_gpu = ggml_offload_nop;- }+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ static offload_func_t ggml_offload_gpu = ggml_cuda_assign_buffers_no_alloc;+#else+ static offload_func_t ggml_offload_gpu = ggml_offload_nop;+#endifswitch (func_e) {case OFFLOAD_FUNC_NOP:@@ -6055,6 +6383,14 @@ static struct ggml_cgraph * llama_build_graph({result = llm.build_phi2();} break;+ case LLM_ARCH_PLAMO:+ {+ result = llm.build_plamo();+ } break;+ case LLM_ARCH_GPT2:+ {+ result = llm.build_gpt2();+ } break;default:GGML_ASSERT(false);}@@ -6132,7 +6468,7 @@ static int llama_decode_internal(const int64_t n_embd = hparams.n_embd;const int64_t n_vocab = hparams.n_vocab;- // helpers for smoother batch API transistion+ // helpers for smoother batch API transition// after deprecating the llama_eval calls, these will be removedstd::vector<llama_pos> pos;@@ -6199,11 +6535,12 @@ static int llama_decode_internal(GGML_ASSERT(strcmp(embeddings->name, "result_norm") == 0);}- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ char * buf_alloc_base = (char *)ggml_backend_buffer_get_base(lctx.buf_alloc);for (int i = 0; i < gf->n_leafs; i++) {ggml_tensor * node = gf->leafs[i];if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) {- ggml_cuda_assign_scratch_offset(node, (char*)node->data - (char *) lctx.buf_alloc.data);+ ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base);ggml_cuda_copy_to_device(node);}}@@ -6211,7 +6548,7 @@ static int llama_decode_internal(for (int i = 0; i < gf->n_nodes; i++) {ggml_tensor * node = gf->nodes[i];if (node->backend == GGML_BACKEND_GPU && node->extra == NULL) {- ggml_cuda_assign_scratch_offset(node, (char*)node->data - (char *) lctx.buf_alloc.data);+ ggml_cuda_assign_scratch_offset(node, (char *)node->data - buf_alloc_base);}}@@ -6220,7 +6557,7 @@ static int llama_decode_internal(embeddings->backend = GGML_BACKEND_CPU;}res->backend = GGML_BACKEND_CPU;- }+#endif// LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);@@ -6238,19 +6575,23 @@ static int llama_decode_internal(n_threads = 1;}-#if GGML_USE_MPI+#ifdef GGML_USE_MPIconst int64_t n_layer = hparams.n_layer;ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);#endif- if (lctx.ctx_metal) {- ggml_metal_set_n_cb (lctx.ctx_metal, n_threads);- ggml_metal_graph_compute(lctx.ctx_metal, gf);- } else {- ggml_graph_compute_helper(lctx.work_buffer, gf, n_threads);+#ifdef GGML_USE_METAL+ if (ggml_backend_is_metal(lctx.backend)) {+ ggml_backend_metal_set_n_cb(lctx.backend, n_threads);+ }+#endif++ if (ggml_backend_is_cpu(lctx.backend)) {+ ggml_backend_cpu_set_n_threads(lctx.backend, n_threads);}+ ggml_backend_graph_compute(lctx.backend, gf);-#if GGML_USE_MPI+#ifdef GGML_USE_MPIggml_mpi_graph_compute_post(lctx.ctx_mpi, gf, n_layer);#endif@@ -6288,20 +6629,37 @@ static int llama_decode_internal({auto & logits_out = lctx.logits;+#ifndef NDEBUG+ auto & logits_valid = lctx.logits_valid;+ logits_valid.clear();+ logits_valid.resize(n_tokens);++ logits_out.clear();+#endif+if (batch.logits) {logits_out.resize(n_vocab * n_tokens);for (uint32_t i = 0; i < n_tokens; i++) {if (batch.logits[i] == 0) {continue;}- memcpy(logits_out.data() + (n_vocab*i), (float *) ggml_get_data(res) + (n_vocab*i), sizeof(float)*n_vocab);+ ggml_backend_tensor_get(res, logits_out.data() + (n_vocab*i), (n_vocab*i)*sizeof(float), n_vocab*sizeof(float));+#ifndef NDEBUG+ logits_valid[i] = true;+#endif}} else if (lctx.logits_all) {logits_out.resize(n_vocab * n_tokens);- memcpy(logits_out.data(), (float *) ggml_get_data(res), sizeof(float)*n_vocab*n_tokens);+ ggml_backend_tensor_get(res, logits_out.data(), 0, n_vocab*n_tokens*sizeof(float));+#ifndef NDEBUG+ std::fill(logits_valid.begin(), logits_valid.end(), true);+#endif} else {logits_out.resize(n_vocab);- memcpy(logits_out.data(), (float *) ggml_get_data(res) + (n_vocab*(n_tokens - 1)), sizeof(float)*n_vocab);+ ggml_backend_tensor_get(res, logits_out.data(), (n_vocab*(n_tokens - 1))*sizeof(float), n_vocab*sizeof(float));+#ifndef NDEBUG+ logits_valid[0] = true;+#endif}}@@ -6310,7 +6668,7 @@ static int llama_decode_internal(auto & embedding_out = lctx.embedding;embedding_out.resize(n_embd);- memcpy(embedding_out.data(), (float *) ggml_get_data(embeddings) + (n_embd*(n_tokens - 1)), sizeof(float)*n_embd);+ ggml_backend_tensor_get(embeddings, embedding_out.data(), (n_embd*(n_tokens - 1))*sizeof(float), n_embd*sizeof(float));}// measure the performance only for the single-token evals@@ -6911,12 +7269,12 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<// loop over the textwhile (true) {- // find the first occurence of a given special token in this fragment+ // find the first occurrence of a given special token in this fragment// passing offset argument only limit the "search area" but match coordinates// are still relative to the source full raw_textauto match = raw_text->find(special_token, raw_text_base_offset);- // no occurences found, stop processing this fragment for a given special token+ // no occurrences found, stop processing this fragment for a given special tokenif (match == std::string::npos) break;// check if match is within bounds of offset <-> length@@ -7499,7 +7857,7 @@ void llama_sample_softmax(struct llama_context * ctx, llama_token_data_array * c}}-void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int k, size_t min_keep) {+void llama_sample_top_k(struct llama_context * ctx, llama_token_data_array * candidates, int32_t k, size_t min_keep) {const int64_t t_start_sample_us = ggml_time_us();k = std::max(k, (int) min_keep);@@ -7859,7 +8217,7 @@ void llama_sample_classifier_free_guidance(}}-llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int m, float * mu) {+llama_token llama_sample_token_mirostat(struct llama_context * ctx, llama_token_data_array * candidates, float tau, float eta, int32_t m, float * mu) {GGML_ASSERT(ctx);auto N = float(llama_n_vocab(llama_get_model(ctx)));@@ -8268,12 +8626,6 @@ void llama_beam_search(llama_context * ctx,// quantization//-template <typename T>-struct no_init {- T value;- no_init() { /* do nothing */ }-};-struct quantize_state_internal {const llama_model & model;const llama_model_quantize_params * params;@@ -8509,12 +8861,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s// mmap consistently increases speed Linux, and also increases speed on Windows with// hot cache. It may cause a slowdown on macOS, possibly related to free memory.- bool use_mmap = !IsXnu();+#if defined(__linux__) || defined(_WIN32)+ constexpr bool use_mmap = true;+#else+ constexpr bool use_mmap = false;+#endifllama_model_loader ml(fname_inp, use_mmap, NULL);- if (ml.use_mmap) {- ml.mapping.reset(new llama_mmap(&ml.file, /* prefetch */ 0, ggml_is_numa()));- }+ ml.init_mapping(false); // no prefetching?llama_model model;llm_load_arch(ml, model);@@ -8605,7 +8959,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const sbool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?// quantize only 2D tensors- quantize &= (tensor->n_dims == 2);+ quantize &= (ggml_n_dims(tensor) == 2);quantize &= params->quantize_output_tensor || name != "output.weight";quantize &= !params->only_copy;@@ -8760,74 +9114,63 @@ static int llama_apply_lora_from_file_internal(const int64_t t_start_lora_us = ggml_time_us();- auto fin = std::ifstream(path_lora, std::ios::binary);- if (!fin) {- LLAMA_LOG_ERROR("%s: failed to open '%s'\n", __func__, path_lora);- return 1;- }+ llama_file fin(path_lora, "rbe");// verify magic and version{- uint32_t magic;- fin.read((char *) &magic, sizeof(magic));- uint32_t format_version;- fin.read((char *) &format_version, sizeof(format_version));+ uint32_t magic = fin.read_u32();+ if (magic != LLAMA_FILE_MAGIC_GGLA) {+ LLAMA_LOG_ERROR("%s: bad file magic\n", __func__);+ return 1;+ }+ uint32_t format_version = fin.read_u32();if (format_version != 1) {LLAMA_LOG_ERROR("%s: unsupported file version\n", __func__ );return 1;}}- int32_t lora_r;- int32_t lora_alpha;- fin.read((char *) &lora_r, sizeof(lora_r));- fin.read((char *) &lora_alpha, sizeof(lora_alpha));+ int32_t lora_r = fin.read_u32();+ int32_t lora_alpha = fin.read_u32();float scaling = scale * (float)lora_alpha / (float)lora_r;LLAMA_LOG_INFO("%s: r = %d, alpha = %d, scaling = %.2f\n", __func__, lora_r, lora_alpha, scaling);+ // create a name -> tensor map of the model to accelerate lookups+ // find the max tensor size to estimate the required temporary buffer size+ size_t max_tensor_size = 0;+ std::unordered_map<std::string, struct ggml_tensor*> model_tensors;+ for (const auto & kv : model.tensors_by_name) {+ model_tensors.insert(kv);+ size_t f32_size = ggml_nelements(kv.second) * sizeof(float);+ max_tensor_size = std::max(max_tensor_size, f32_size);+ }+// create a temporary ggml context to store the lora tensors- // todo: calculate size from biggest possible tensor- std::vector<uint8_t> lora_buf(1024ull * 1024ull * 1024ull);+ // TODO: use ggml-alloc+ size_t lora_ctx_size = max_tensor_size * 3;+ LLAMA_LOG_INFO("%s: allocating %.f MB for lora temporary buffer\n", __func__, lora_ctx_size / 1024.0 / 1024.0);+ std::vector<uint8_t> lora_buf(lora_ctx_size);+struct ggml_init_params params;params.mem_size = lora_buf.size();params.mem_buffer = lora_buf.data();params.no_alloc = false;- ggml_context * lora_ctx = ggml_init(params);- std::unordered_map<std::string, struct ggml_tensor *> lora_tensors;+ using unique_context = std::unique_ptr<ggml_context, decltype(&ggml_free)>;- // create a name -> tensor map of the model to accelerate lookups- std::unordered_map<std::string, struct ggml_tensor*> model_tensors;- for (const auto & kv : model.tensors_by_name) {- model_tensors.insert(kv);- }+ unique_context lora_ctx(nullptr, ggml_free);+ lora_ctx.reset(ggml_init(params));+ std::unordered_map<std::string, struct ggml_tensor *> lora_tensors;// load base modelstd::unique_ptr<llama_model_loader> ml;- ggml_context * base_ctx = NULL;- std::vector<uint8_t> base_buf;- if (path_base_model) {- LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);- ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ NULL));-- size_t ctx_size;- size_t mmapped_size;- ml->calc_sizes(ctx_size, mmapped_size);- base_buf.resize(ctx_size);-- ggml_init_params base_params;- base_params.mem_size = base_buf.size();- base_params.mem_buffer = base_buf.data();- base_params.no_alloc = ml->use_mmap;-- base_ctx = ggml_init(base_params);- // maybe this should in llama_model_loader- if (ml->use_mmap) {- ml->mapping.reset(new llama_mmap(&ml->file, /* prefetch */ 0, ggml_is_numa()));- }+ if (path_base_model) {+ LLAMA_LOG_INFO("%s: loading base model from '%s'\n", __func__, path_base_model);+ ml.reset(new llama_model_loader(path_base_model, /*use_mmap*/ true, /*kv_overrides*/ nullptr));+ ml->init_mapping(false); // no prefetching}// read tensors and apply@@ -8837,27 +9180,35 @@ static int llama_apply_lora_from_file_internal(std::vector<uint8_t> work_buffer;while (true) {+ if (fin.tell() == llamafile_size(fin.file)) {+ // eof+ break;+ }+int32_t n_dims;- int32_t length;+ int32_t name_len;int32_t ftype;- fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));- fin.read(reinterpret_cast<char *>(&length), sizeof(length));- fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));- if (fin.eof()) {- break;+ fin.read_raw(&n_dims, sizeof(n_dims));+ fin.read_raw(&name_len, sizeof(name_len));+ fin.read_raw(&ftype, sizeof(ftype));++ if (n_dims != 1 && n_dims != 2) {+ LLAMA_LOG_ERROR("%s: unsupported tensor dimension %d\n", __func__, n_dims);+ return 1;}int32_t ne[2] = { 1, 1 };for (int i = 0; i < n_dims; ++i) {- fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));+ fin.read_raw(&ne[i], sizeof(ne[i]));}std::string name;{+ GGML_ASSERT(name_len <= 1024);char buf[1024];- fin.read(buf, length);- name = std::string(buf, length);+ fin.read_raw(buf, name_len);+ name = std::string(buf, name_len);}// check for lora suffix and get the type of tensor@@ -8871,7 +9222,7 @@ static int llama_apply_lora_from_file_internal(std::string lora_type = name.substr(pos + lora_suffix.length());std::string base_name = name;base_name.erase(pos);- // LLAMA_LOG_INFO("%s: %s => %s (lora type %s) \n", __func__, name.c_str(),base_name.c_str(), lora_type.c_str());+ // LLAMA_LOG_INFO("%s: %s => %s (lora type %s) \n", __func__, name.c_str(), base_name.c_str(), lora_type.c_str());if (model_tensors.find(base_name) == model_tensors.end()) {LLAMA_LOG_ERROR("%s: unknown tensor '%s' in lora adapter\n", __func__, name.data());@@ -8890,22 +9241,15 @@ static int llama_apply_lora_from_file_internal(return false;}}- ggml_tensor * lora_tensor;- if (n_dims == 2) {- lora_tensor = ggml_new_tensor_2d(lora_ctx, wtype, ne[0], ne[1]);- }- else {- LLAMA_LOG_ERROR("%s: unsupported tensor dimension %d\n", __func__, n_dims);- return 1;- }- ggml_set_name(lora_tensor, "lora_tensor");+ ggml_tensor * lora_tensor = ggml_new_tensor_2d(lora_ctx.get(), wtype, ne[0], ne[1]);+ ggml_set_name(lora_tensor, name.c_str());// load tensor data- size_t offset = fin.tellg();+ size_t offset = fin.tell();size_t tensor_data_size = ggml_nbytes(lora_tensor);offset = (offset + 31) & -32;- fin.seekg(offset);- fin.read((char*)lora_tensor->data, tensor_data_size);+ fin.seek(offset, SEEK_SET);+ fin.read_raw(lora_tensor->data, tensor_data_size);lora_tensors[name] = lora_tensor;@@ -8918,7 +9262,7 @@ static int llama_apply_lora_from_file_internal(offload_func_t offload_func = ggml_offload_nop;offload_func_t offload_func_force_inplace = ggml_offload_nop;- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (dest_t->backend == GGML_BACKEND_GPU || dest_t->backend == GGML_BACKEND_GPU_SPLIT) {if (dest_t->type != GGML_TYPE_F16) {ThrowRuntimeError(format(@@ -8927,7 +9271,7 @@ static int llama_apply_lora_from_file_internal(offload_func = ggml_cuda_assign_buffers;offload_func_force_inplace = ggml_cuda_assign_buffers_force_inplace;}- }+#endif // GGML_USE_CUBLASggml_tensor * base_t;if (ml) {@@ -8935,13 +9279,11 @@ static int llama_apply_lora_from_file_internal(// load from base modelif (gguf_find_tensor(ctx_gguf, base_name.c_str()) < 0) {- // TODO: throwLLAMA_LOG_ERROR("%s: error: tensor '%s' not found in base model\n", __func__, base_name.c_str());return 1;}- // TODO: not tested!! maybe not working!- base_t = ml->create_tensor(base_ctx, base_name, { (uint32_t)dest_t->ne[0], (uint32_t)dest_t->ne[1] }, GGML_BACKEND_CPU);+ base_t = ml->get_tensor_meta(base_name.c_str());ml->load_data_for(base_t);} else {base_t = dest_t;@@ -8970,43 +9312,42 @@ static int llama_apply_lora_from_file_internal(}// w = w + BA*s- ggml_tensor * BA = ggml_mul_mat(lora_ctx, loraA, loraB);+ ggml_tensor * BA = ggml_mul_mat(lora_ctx.get(), loraA, loraB);offload_func(BA);ggml_set_name(BA, "BA");if (scaling != 1.0f) {- ggml_tensor * scale_tensor = ggml_new_f32(lora_ctx, scaling);- ggml_set_name(scale_tensor, "scale_tensor");-- BA = ggml_scale_inplace(lora_ctx, BA, scale_tensor);+ BA = ggml_scale_inplace(lora_ctx.get(), BA, scaling);offload_func(BA);ggml_set_name(BA, "BA_scaled");}ggml_tensor * r;if (base_t == dest_t) {- r = ggml_add_inplace(lora_ctx, dest_t, BA);+ r = ggml_add_inplace(lora_ctx.get(), dest_t, BA);offload_func_force_inplace(r);ggml_set_name(r, "r_add_inplace");}else {- r = ggml_add(lora_ctx, base_t, BA);+ r = ggml_add(lora_ctx.get(), base_t, BA);offload_func(r);ggml_set_name(r, "r_add");- r = ggml_cpy(lora_ctx, r, dest_t);+ r = ggml_cpy(lora_ctx.get(), r, dest_t);offload_func(r);ggml_set_name(r, "r_cpy");}- struct ggml_cgraph * gf = ggml_new_graph(lora_ctx);+ struct ggml_cgraph * gf = ggml_new_graph(lora_ctx.get());ggml_build_forward_expand(gf, r);ggml_graph_compute_helper(work_buffer, gf, n_threads);+ // the tensors in the adapter must be sorted such that loraA and loraB of the same tensor are next to each other+ GGML_ASSERT(lora_tensors.size() == 2);+// we won't need these tensors again, reset the context to save memory- ggml_free(lora_ctx);- lora_ctx = ggml_init(params);+ lora_ctx.reset(ggml_init(params));lora_tensors.clear();n_tensors++;@@ -9016,12 +9357,6 @@ static int llama_apply_lora_from_file_internal(}}- // TODO: this should be in a destructor, it will leak on failure- ggml_free(lora_ctx);- if (base_ctx) {- ggml_free(base_ctx);- }-const int64_t t_lora_us = ggml_time_us() - t_start_lora_us;LLAMA_LOG_INFO(" done (%.2f ms)\n", t_lora_us / 1000.0);@@ -9044,9 +9379,9 @@ struct llama_model_params llama_model_default_params() {/*.use_mlock =*/ false,};- if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {- result.n_gpu_layers = 1;- }+#ifdef GGML_USE_METAL+ result.n_gpu_layers = 1;+#endifreturn result;}@@ -9090,7 +9425,7 @@ struct llama_model_quantize_params llama_model_quantize_default_params() {return result;}-int llama_max_devices(void) {+int32_t llama_max_devices(void) {return LLAMA_MAX_DEVICES;}@@ -9151,11 +9486,18 @@ struct llama_model * llama_load_model_from_file(LLAMA_LOG_INFO("\n");}}+ return true;};}- if (!llama_model_load(path_model, *model, params)) {- LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);+ int status = llama_model_load(path_model, *model, params);+ GGML_ASSERT(status <= 0);+ if (status < 0) {+ if (status == -1) {+ LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);+ } else if (status == -2) {+ LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);+ }delete model;return nullptr;}@@ -9225,12 +9567,46 @@ struct llama_context * llama_new_context_with_model(const ggml_type type_k = params.type_k;const ggml_type type_v = params.type_v;- GGML_ASSERT(hparams.n_embd_head() % ggml_blck_size(type_k) == 0);- GGML_ASSERT(hparams.n_embd_head() % ggml_blck_size(type_v) == 0);+ GGML_ASSERT(hparams.n_embd_head_k % ggml_blck_size(type_k) == 0);+ GGML_ASSERT(hparams.n_embd_head_v % ggml_blck_size(type_v) == 0);// reserve memory for context buffersif (!hparams.vocab_only) {- if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, type_k, type_v, cparams.n_ctx, model->n_gpu_layers, cparams.offload_kqv)) {+ // initialize backend+#ifdef GGML_USE_METAL+ if (model->n_gpu_layers > 0) {+ ctx->backend = ggml_backend_metal_init();+ if (ctx->backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__);+ }+ }+#elif defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+ // for testing only+ if (model->n_gpu_layers > 0) {+ ctx->backend = ggml_backend_cuda_init(0);+ if (ctx->backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CUDA backend\n", __func__);+ }+ }+ }+#endif++ if (ctx->backend == nullptr && ggml_backend_buffer_is_host(model->buf)) {+ ctx->backend = ggml_backend_cpu_init();+ if (ctx->backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize CPU backend\n", __func__);+ }+ }++ if (ctx->backend == nullptr) {+ LLAMA_LOG_ERROR("%s: failed to initialize a backend\n", __func__);+ delete ctx;+ return nullptr;+ }++ if (!llama_kv_cache_init(ctx->model.hparams, ctx->kv_self, type_k, type_v,+ cparams.n_ctx, model->n_gpu_layers, cparams.offload_kqv)) {LLAMA_LOG_ERROR("%s: llama_kv_cache_init() failed for self-attention cache\n", __func__);llama_free(ctx);return nullptr;@@ -9266,12 +9642,11 @@ struct llama_context * llama_new_context_with_model(}{- static const size_t tensor_alignment = 32;// the compute buffer is used to store the tensor and graph structs, while the allocator buffer is used for the tensor data- ctx->buf_compute.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());+ ctx->buf_compute_meta.resize(ggml_tensor_overhead()*LLAMA_MAX_NODES + ggml_graph_overhead());// create measure allocator- ctx->alloc = ggml_allocr_new_measure(tensor_alignment);+ ctx->alloc = ggml_allocr_new_measure_from_backend(ctx->backend);// build worst-case graphint n_tokens = (int)std::min(cparams.n_ctx, cparams.n_batch);@@ -9279,96 +9654,50 @@ struct llama_context * llama_new_context_with_model(llama_token token = llama_token_bos(&ctx->model); // not actually used by llama_build_graph, but required to choose between token and embedding inputs graphggml_cgraph * gf = llama_build_graph(*ctx, llama_batch_get_one(&token, n_tokens, n_past, 0));- if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {- if (model->n_gpu_layers > 0) {- ctx->ctx_metal = ggml_metal_init(1);- if (!ctx->ctx_metal) {- LLAMA_LOG_ERROR("%s: ggml_metal_init() failed\n", __func__);- llama_free(ctx);- return NULL;- }- //ggml_metal_graph_find_concurrency(ctx->ctx_metal, gf, false);- //ggml_allocr_set_parse_seq(ctx->alloc, ggml_metal_get_concur_list(ctx->ctx_metal), ggml_metal_if_optimized(ctx->ctx_metal));- }- }-// measure memory requirements for the graph- size_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf) + tensor_alignment;+ size_t alloc_size = ggml_allocr_alloc_graph(ctx->alloc, gf);- LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MiB\n", __func__, (ctx->buf_compute.size + alloc_size) / 1024.0 / 1024.0);+ LLAMA_LOG_INFO("%s: compute buffer total size = %.2f MiB\n", __func__, (ctx->buf_compute_meta.size() + alloc_size) / 1024.0 / 1024.0);- // recreate allocator with exact memory requirements+ // create allocator again with exact memory requirementsggml_allocr_free(ctx->alloc);- ctx->buf_alloc.resize(alloc_size);- ctx->alloc = ggml_allocr_new(ctx->buf_alloc.data, ctx->buf_alloc.size, tensor_alignment);- if (ctx->ctx_metal) {- //ggml_allocr_set_parse_seq(ctx->alloc, ggml_metal_get_concur_list(ctx->ctx_metal), ggml_metal_if_optimized(ctx->ctx_metal));- }- if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {- ggml_cuda_set_scratch_size(alloc_size);- LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MiB\n", __func__, alloc_size / 1024.0 / 1024.0);-- // calculate total VRAM usage- auto add_tensor = [](const ggml_tensor * t, size_t & size) {- if (t->backend == GGML_BACKEND_GPU || t->backend == GGML_BACKEND_GPU_SPLIT) {- size += ggml_nbytes(t);+ ctx->buf_alloc = ggml_backend_alloc_buffer(ctx->backend, alloc_size);+ ctx->alloc = ggml_allocr_new_from_buffer(ctx->buf_alloc);+#if !defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ if (model->n_gpu_layers > 0) {+ // the CPU buffer adds this padding in case the malloc buffer is not aligned, so we need to do the same for the GPU buffer, since we use the same offsets+ ggml_cuda_set_scratch_size(alloc_size + 64);+ LLAMA_LOG_INFO("%s: VRAM scratch buffer: %.2f MiB\n", __func__, alloc_size / 1024.0 / 1024.0);++ // calculate total VRAM usage+ auto add_tensor = [](const ggml_tensor * t, size_t & size) {+ if (t->backend == GGML_BACKEND_GPU || t->backend == GGML_BACKEND_GPU_SPLIT) {+ size += ggml_nbytes(t);+ }+ };+ size_t model_vram_size = 0;+ for (const auto & kv : model->tensors_by_name) {+ add_tensor(kv.second, model_vram_size);}- };- size_t model_vram_size = 0;- for (const auto & kv : model->tensors_by_name) {- add_tensor(kv.second, model_vram_size);- }-- size_t kv_vram_size = 0;- for (auto & k : ctx->kv_self.k_l) {- add_tensor(k, kv_vram_size);- }- for (auto & v : ctx->kv_self.v_l) {- add_tensor(v, kv_vram_size);- }-- size_t ctx_vram_size = alloc_size + kv_vram_size;- size_t total_vram_size = model_vram_size + ctx_vram_size;-- LLAMA_LOG_INFO("%s: total VRAM used: %.2f MiB (model: %.2f MiB, context: %.2f MiB)\n", __func__,- total_vram_size / 1024.0 / 1024.0,- model_vram_size / 1024.0 / 1024.0,- ctx_vram_size / 1024.0 / 1024.0);- }- }-- if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {- if (model->n_gpu_layers > 0) {- // this allocates all Metal resources and memory buffers-- void * data_ptr = NULL;- size_t data_size = 0;-- if (ctx->model.mapping) {- data_ptr = ctx->model.mapping->addr;- data_size = ctx->model.mapping->size;- } else {- data_ptr = ggml_get_mem_buffer(ctx->model.ctx);- data_size = ggml_get_mem_size (ctx->model.ctx);- }- const size_t max_size = ggml_get_max_tensor_size(ctx->model.ctx);+ size_t kv_vram_size = 0;+ for (auto & k : ctx->kv_self.k_l) {+ add_tensor(k, kv_vram_size);+ }+ for (auto & v : ctx->kv_self.v_l) {+ add_tensor(v, kv_vram_size);+ }- LLAMA_LOG_INFO("%s: max tensor size = %8.2f MiB\n", __func__, max_size/1024.0/1024.0);+ size_t ctx_vram_size = alloc_size + kv_vram_size;+ size_t total_vram_size = model_vram_size + ctx_vram_size;-#define LLAMA_METAL_CHECK_BUF(result) \- if (!(result)) { \- LLAMA_LOG_ERROR("%s: failed to add buffer\n", __func__); \- llama_free(ctx); \- return NULL; \+ LLAMA_LOG_INFO("%s: total VRAM used: %.2f MiB (model: %.2f MiB, context: %.2f MiB)\n", __func__,+ total_vram_size / 1024.0 / 1024.0,+ model_vram_size / 1024.0 / 1024.0,+ ctx_vram_size / 1024.0 / 1024.0);}-- LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "data", data_ptr, data_size, max_size));- LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "kv", ctx->kv_self.buf.data, ctx->kv_self.buf.size, 0));- LLAMA_METAL_CHECK_BUF(ggml_metal_add_buffer(ctx->ctx_metal, "alloc", ctx->buf_alloc.data, ctx->buf_alloc.size, 0));-#undef LLAMA_METAL_CHECK_BUF- }+#endif}}@@ -9397,23 +9726,27 @@ const llama_model * llama_get_model(const struct llama_context * ctx) {return &ctx->model;}-int llama_n_ctx(const struct llama_context * ctx) {+uint32_t llama_n_ctx(const struct llama_context * ctx) {return ctx->cparams.n_ctx;}+uint32_t llama_n_batch(const struct llama_context * ctx) {+ return ctx->cparams.n_batch;+}+enum llama_vocab_type llama_vocab_type(const struct llama_model * model) {return model->vocab.type;}-int llama_n_vocab(const struct llama_model * model) {+int32_t llama_n_vocab(const struct llama_model * model) {return model->vocab.id_to_token.size();}-int llama_n_ctx_train(const struct llama_model * model) {+int32_t llama_n_ctx_train(const struct llama_model * model) {return model->hparams.n_ctx_train;}-int llama_n_embd(const struct llama_model * model) {+int32_t llama_n_embd(const struct llama_model * model) {return model->hparams.n_embd;}@@ -9421,7 +9754,7 @@ float llama_rope_freq_scale_train(const struct llama_model * model) {return model->hparams.rope_freq_scale_train;}-int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size) {+int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size) {const auto & it = model->gguf_kv.find(key);if (it == model->gguf_kv.end()) {if (buf_size > 0) {@@ -9432,11 +9765,11 @@ int llama_model_meta_val_str(const struct llama_model * model, const char * key,return snprintf(buf, buf_size, "%s", it->second.c_str());}-int llama_model_meta_count(const struct llama_model * model) {+int32_t llama_model_meta_count(const struct llama_model * model) {return (int)model->gguf_kv.size();}-int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {+int32_t llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {if (i < 0 || i >= (int)model->gguf_kv.size()) {if (buf_size > 0) {buf[0] = '\0';@@ -9448,7 +9781,7 @@ int llama_model_meta_key_by_index(const struct llama_model * model, int i, charreturn snprintf(buf, buf_size, "%s", it->first.c_str());}-int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size) {+int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size) {if (i < 0 || i >= (int)model->gguf_kv.size()) {if (buf_size > 0) {buf[0] = '\0';@@ -9460,9 +9793,10 @@ int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, creturn snprintf(buf, buf_size, "%s", it->second.c_str());}-int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {- return snprintf(buf, buf_size, "%s %s %s",+int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size) {+ return snprintf(buf, buf_size, "%s %s%s %s",llama_model_arch_name(model->arch).c_str(),+ model->hparams.n_expert > 0 ? (std::to_string(model->hparams.n_expert) + "x").c_str() : "",llama_model_type_name(model->type),llama_model_ftype_name(model->ftype).c_str());}@@ -9487,7 +9821,7 @@ struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const chreturn ggml_get_tensor(model->ctx, name);}-int llama_model_quantize(+uint32_t llama_model_quantize(const char * fname_inp,const char * fname_out,const llama_model_quantize_params * params) {@@ -9504,7 +9838,7 @@ int llama_model_quantize(#endif}-int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, float scale, const char * path_base_model, int n_threads) {+int32_t llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) {#ifndef _LIBCPP_NO_EXCEPTIONStry {#endif@@ -9517,7 +9851,7 @@ int llama_apply_lora_from_file(struct llama_context * ctx, const char * path_lor#endif}-int llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, float scale, const char * path_base_model, int n_threads) {+int32_t llama_model_apply_lora_from_file(const struct llama_model * model, const char * path_lora, float scale, const char * path_base_model, int32_t n_threads) {#ifndef _LIBCPP_NO_EXCEPTIONStry {#endif@@ -9619,7 +9953,7 @@ void llama_kv_cache_view_update(const struct llama_context * ctx, struct llama_k}}-int llama_get_kv_cache_token_count(const struct llama_context * ctx) {+int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx) {int result = 0;for (uint32_t i = 0; i < ctx->kv_self.size; i++) {@@ -9629,7 +9963,7 @@ int llama_get_kv_cache_token_count(const struct llama_context * ctx) {return result;}-int llama_get_kv_cache_used_cells(const struct llama_context * ctx) {+int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx) {return ctx->kv_self.used;}@@ -9669,7 +10003,7 @@ size_t llama_get_state_size(const struct llama_context * ctx) {const size_t s_embedding = ctx->embedding.size() * sizeof(float);const size_t s_kv_size = sizeof(size_t);const size_t s_kv_ntok = sizeof(int);- const size_t s_kv = ctx->kv_self.buf.size;+ const size_t s_kv = ggml_backend_buffer_get_size(ctx->kv_self.buf);const size_t s_total = (+ s_rng_size@@ -9793,11 +10127,12 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datconst auto & hparams = ctx->model.hparams;const auto & cparams = ctx->cparams;- const auto n_layer = hparams.n_layer;- const auto n_embd = hparams.n_embd_gqa();- const auto n_ctx = cparams.n_ctx;+ const auto n_layer = hparams.n_layer;+ const auto n_embd_k_gqa = hparams.n_embd_k_gqa();+ const auto n_embd_v_gqa = hparams.n_embd_v_gqa();+ const auto n_ctx = cparams.n_ctx;- const size_t kv_buf_size = kv_self.buf.size;+ const size_t kv_buf_size = ggml_backend_buffer_get_size(kv_self.buf);const uint32_t kv_head = kv_self.head;const uint32_t kv_size = kv_self.size;const uint32_t kv_used = kv_self.used;@@ -9813,40 +10148,43 @@ static void llama_copy_state_data_internal(struct llama_context * ctx, llama_datggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });ggml_cgraph * gf = ggml_new_graph(cpy_ctx);- std::vector<std::vector<uint8_t>> kout2d_data(n_layer);- std::vector<std::vector<uint8_t>> vout2d_data(n_layer);+ std::vector<struct ggml_tensor *> kout2d(n_layer);+ std::vector<struct ggml_tensor *> vout2d(n_layer);for (int il = 0; il < (int) n_layer; ++il) {- ggml_tensor * kout2d = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd, kv_head);- kout2d_data[il].resize(ggml_nbytes(kout2d));- kout2d->data = kout2d_data[il].data();-- ggml_tensor * vout2d = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd);- vout2d_data[il].resize(ggml_nbytes(vout2d));- vout2d->data = vout2d_data[il].data();+ kout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head);+ vout2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa);ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],- n_embd, kv_head,- elt_size*n_embd, 0);+ n_embd_k_gqa, kv_head,+ elt_size*n_embd_k_gqa, 0);ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],- kv_head, n_embd,+ kv_head, n_embd_v_gqa,elt_size*n_ctx, 0);- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, k2d, kout2d));- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, v2d, vout2d));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, k2d, kout2d[il]));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, v2d, vout2d[il]));}- ggml_graph_compute_helper(ctx->work_buffer, gf, /*n_threads*/ 1);+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend);- ggml_free(cpy_ctx);+ ggml_backend_graph_compute(ctx->backend, gf);++ std::vector<uint8_t> tmp_buf;+ for (int il = 0; il < (int) n_layer; ++il) {+ tmp_buf.resize(ggml_nbytes(kout2d[il]));+ ggml_backend_tensor_get(kout2d[il], tmp_buf.data(), 0, tmp_buf.size());+ data_ctx->write(tmp_buf.data(), tmp_buf.size());- // our data is now in the kout2d_data and vout2d_data buffers- // write them to file- for (uint32_t il = 0; il < n_layer; ++il) {- data_ctx->write(kout2d_data[il].data(), kout2d_data[il].size());- data_ctx->write(vout2d_data[il].data(), vout2d_data[il].size());+ tmp_buf.resize(ggml_nbytes(vout2d[il]));+ ggml_backend_tensor_get(vout2d[il], tmp_buf.data(), 0, tmp_buf.size());+ data_ctx->write(tmp_buf.data(), tmp_buf.size());}++ ggml_free(cpy_ctx);++ ggml_backend_buffer_free(buf);}for (uint32_t i = 0; i < kv_size; ++i) {@@ -9929,9 +10267,10 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {const auto & hparams = ctx->model.hparams;const auto & cparams = ctx->cparams;- const int n_layer = hparams.n_layer;- const int n_embd = hparams.n_embd_gqa();- const int n_ctx = cparams.n_ctx;+ const int n_layer = hparams.n_layer;+ const int n_embd_k_gqa = hparams.n_embd_k_gqa();+ const int n_embd_v_gqa = hparams.n_embd_v_gqa();+ const int n_ctx = cparams.n_ctx;size_t kv_buf_size;uint32_t kv_head;@@ -9944,37 +10283,48 @@ size_t llama_set_state_data(struct llama_context * ctx, uint8_t * src) {memcpy(&kv_used, inp, sizeof(kv_used)); inp += sizeof(kv_used);if (kv_buf_size) {- GGML_ASSERT(kv_self.buf.size == kv_buf_size);+ GGML_ASSERT(ggml_backend_buffer_get_size(kv_self.buf) == kv_buf_size);const size_t elt_size = ggml_element_size(kv_self.k_l[0]);ggml_context * cpy_ctx = ggml_init({ 6*n_layer*ggml_tensor_overhead() + ggml_graph_overhead(), NULL, /* no_alloc */ true });ggml_cgraph * gf = ggml_new_graph(cpy_ctx);- for (int il = 0; il < n_layer; ++il) {- ggml_tensor * kin2d = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd, kv_head);- kin2d->data = (void *) inp;- inp += ggml_nbytes(kin2d);+ std::vector<struct ggml_tensor *> kin2d(n_layer);+ std::vector<struct ggml_tensor *> vin2d(n_layer);- ggml_tensor * vin2d = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd);- vin2d->data = (void *) inp;- inp += ggml_nbytes(vin2d);+ for (int il = 0; il < n_layer; ++il) {+ kin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.k_l[il]->type, n_embd_k_gqa, kv_head);+ vin2d[il] = ggml_new_tensor_2d(cpy_ctx, kv_self.v_l[il]->type, kv_head, n_embd_v_gqa);ggml_tensor * k2d = ggml_view_2d(cpy_ctx, kv_self.k_l[il],- n_embd, kv_head,- elt_size*n_embd, 0);+ n_embd_k_gqa, kv_head,+ elt_size*n_embd_k_gqa, 0);ggml_tensor * v2d = ggml_view_2d(cpy_ctx, kv_self.v_l[il],- kv_head, n_embd,+ kv_head, n_embd_v_gqa,elt_size*n_ctx, 0);- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, kin2d, k2d));- ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, vin2d, v2d));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, kin2d[il], k2d));+ ggml_build_forward_expand(gf, ggml_cpy(cpy_ctx, vin2d[il], v2d));}- ggml_graph_compute_helper(ctx->work_buffer, gf, /*n_threads*/ 1);+ ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(cpy_ctx, ctx->backend);++ // load data into the tensors+ for (int il = 0; il < n_layer; ++il) {+ ggml_backend_tensor_set(kin2d[il], inp, 0, ggml_nbytes(kin2d[il]));+ inp += ggml_nbytes(kin2d[il]);++ ggml_backend_tensor_set(vin2d[il], inp, 0, ggml_nbytes(vin2d[il]));+ inp += ggml_nbytes(vin2d[il]);+ }++ ggml_backend_graph_compute(ctx->backend, gf);ggml_free(cpy_ctx);++ ggml_backend_buffer_free(buf);}ctx->kv_self.head = kv_head;@@ -10077,7 +10427,7 @@ bool llama_load_session_file(struct llama_context * ctx, const char * path_sessi}bool llama_save_session_file(struct llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {- llama_file file(path_session, "wbe");+ llama_file file(path_session, "wb");file.write_u32(LLAMA_SESSION_MAGIC);file.write_u32(LLAMA_SESSION_VERSION);@@ -10099,7 +10449,7 @@ int llama_eval(struct llama_context * ctx,llama_token * tokens,int32_t n_tokens,- int n_past) {+ int32_t n_past) {llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1);const int ret = llama_decode_internal(*ctx, llama_batch_get_one(tokens, n_tokens, n_past, 0));@@ -10114,7 +10464,7 @@ int llama_eval_embd(struct llama_context * ctx,float * embd,int32_t n_tokens,- int n_past) {+ int32_t n_past) {llama_kv_cache_seq_rm(ctx->kv_self, -1, n_past, -1);llama_batch batch = { n_tokens, nullptr, embd, nullptr, nullptr, nullptr, nullptr, n_past, 1, 0, };@@ -10185,7 +10535,7 @@ void llama_batch_free(struct llama_batch batch) {if (batch.logits) free(batch.logits);}-int llama_decode(+int32_t llama_decode(struct llama_context * ctx,struct llama_batch batch) {const int ret = llama_decode_internal(*ctx, batch);@@ -10201,6 +10551,7 @@ float * llama_get_logits(struct llama_context * ctx) {}float * llama_get_logits_ith(struct llama_context * ctx, int32_t i) {+ assert(ctx->logits_valid.at(i));return ctx->logits.data() + i*ctx->model.hparams.n_vocab;}@@ -10232,11 +10583,11 @@ llama_token llama_token_nl(const struct llama_model * model) {return model->vocab.linefeed_id;}-int llama_add_bos_token(const struct llama_model * model) {+int32_t llama_add_bos_token(const struct llama_model * model) {return model->vocab.special_add_bos;}-int llama_add_eos_token(const struct llama_model * model) {+int32_t llama_add_eos_token(const struct llama_model * model) {return model->vocab.special_add_eos;}@@ -10256,12 +10607,12 @@ llama_token llama_token_eot(const struct llama_model * model) {return model->vocab.special_eot_id;}-int llama_tokenize(+int32_t llama_tokenize(const struct llama_model * model,const char * text,- int text_len,+ int32_t text_len,llama_token * tokens,- int n_max_tokens,+ int32_t n_max_tokens,bool add_bos,bool special) {auto res = llama_tokenize_internal(model->vocab, std::string(text, text_len), add_bos, special);@@ -10289,7 +10640,7 @@ static std::string llama_decode_text(const std::string & text) {}// does not write null-terminator to buf-int llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int length) {+int32_t llama_token_to_piece(const struct llama_model * model, llama_token token, char * buf, int32_t length) {if (0 <= token && token < llama_n_vocab(model)) {switch (llama_vocab_get_type(model->vocab)) {case LLAMA_VOCAB_TYPE_SPM: {@@ -10297,7 +10648,7 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, chstd::string result = model->vocab.id_to_token[token].text;llama_unescape_whitespace(result);if (length < (int) result.length()) {- return -result.length();+ return -(int) result.length();}memcpy(buf, result.c_str(), result.length());return result.length();@@ -10327,7 +10678,7 @@ int llama_token_to_piece(const struct llama_model * model, llama_token token, chstd::string result = model->vocab.id_to_token[token].text;result = llama_decode_text(result);if (length < (int) result.length()) {- return -result.length();+ return -(int) result.length();}memcpy(buf, result.c_str(), result.length());return result.length();@@ -10390,6 +10741,7 @@ const char * llama_print_system_info(void) {s = "";s += "AVX = " + std::to_string(ggml_cpu_has_avx()) + " | ";+ s += "AVX_VNNI = " + std::to_string(ggml_cpu_has_avx_vnni()) + " | ";s += "AVX2 = " + std::to_string(ggml_cpu_has_avx2()) + " | ";s += "AVX512 = " + std::to_string(ggml_cpu_has_avx512()) + " | ";s += "AVX512_VBMI = " + std::to_string(ggml_cpu_has_avx512_vbmi()) + " | ";@@ -10446,6 +10798,9 @@ const std::vector<std::pair<std::string, struct ggml_tensor *>> & llama_internalvoid llama_log_set(ggml_log_callback log_callback, void * user_data) {g_state.log_callback = log_callback ? log_callback : llama_log_callback_default;g_state.log_callback_user_data = user_data;+#ifdef GGML_USE_METAL+ ggml_metal_log_set_callback(g_state.log_callback, g_state.log_callback_user_data);+#endif}static void llama_log_internal_v(ggml_log_level level, const char * format, va_list args) {@@ -10473,5 +10828,8 @@ static void llama_log_internal(ggml_log_level level, const char * format, ...) {}static void llama_log_callback_default(ggml_log_level level, const char * text, void * user_data) {- write(2, text, strlen(text));+ (void) level;+ (void) user_data;+ fputs(text, stderr);+ fflush(stderr);}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 405f9dcb..88cdae18 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -1,6 +1,5 @@// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;tab-width:8;coding:utf-8 -*-// vi: set et ft=c++ ts=4 sts=4 sw=4 fenc=utf-8 :vi-#ifndef LLAMA_H#define LLAMA_H@@ -38,6 +37,7 @@#define LLAMA_MAX_RNG_STATE (64*1024)+#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN@@ -123,7 +123,7 @@ extern "C" {bool sorted;} llama_token_data_array;- typedef void (*llama_progress_callback)(float progress, void *ctx);+ typedef bool (*llama_progress_callback)(float progress, void *ctx);// Input data for llama_decode// A llama_batch object can contain input about one or many sequences@@ -176,7 +176,9 @@ extern "C" {int32_t main_gpu; // the GPU that is used for scratch and small tensorsconst float * tensor_split; // how to split layers across multiple GPUs (size: LLAMA_MAX_DEVICES)- // called with a progress value between 0 and 1, pass NULL to disable+ // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.+ // If the provided progress_callback returns true, model loading continues.+ // If it returns false, model loading is immediately aborted.llama_progress_callback progress_callback;// context pointer passed to the progress callback@@ -213,14 +215,14 @@ extern "C" {// Keep the booleans together to avoid misalignment during copy-by-value.bool mul_mat_q; // if true, use experimental mul_mat_q kernels (DEPRECATED - always true)- bool logits_all; // the llama_eval() call computes all logits, not just the last one+ bool logits_all; // the llama_eval() call computes all logits, not just the last one (DEPRECATED - set llama_batch.logits instead)bool embedding; // embedding mode onlybool offload_kqv; // whether to offload the KQV ops (including the KV cache) to GPU};// model quantization parameterstypedef struct llama_model_quantize_params {- int nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()+ int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()enum llama_ftype ftype; // quantize to this llama_ftypebool allow_requantize; // allow quantizing non-f32/f16 tensorsbool quantize_output_tensor; // quantize output.weight@@ -304,19 +306,20 @@ extern "C" {LLAMA_API int64_t llama_time_us(void);- LLAMA_API int llama_max_devices (void);+ LLAMA_API int32_t llama_max_devices(void);LLAMA_API bool llama_mmap_supported (void);LLAMA_API bool llama_mlock_supported(void);LLAMA_API const struct llama_model * llama_get_model(const struct llama_context * ctx);- LLAMA_API int llama_n_ctx (const struct llama_context * ctx);+ LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);+ LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_model * model);- LLAMA_API int llama_n_vocab (const struct llama_model * model);- LLAMA_API int llama_n_ctx_train(const struct llama_model * model);- LLAMA_API int llama_n_embd (const struct llama_model * model);+ LLAMA_API int32_t llama_n_vocab (const struct llama_model * model);+ LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model);+ LLAMA_API int32_t llama_n_embd (const struct llama_model * model);// Get the model's RoPE frequency scaling factorLLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);@@ -327,19 +330,19 @@ extern "C" {// - GGUF array values are not supported by these functions// Get metadata value as a string by key name- LLAMA_API int llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);+ LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);// Get the number of metadata key/value pairs- LLAMA_API int llama_model_meta_count(const struct llama_model * model);+ LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);// Get metadata key name by index- LLAMA_API int llama_model_meta_key_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);+ LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);// Get metadata value as a string by index- LLAMA_API int llama_model_meta_val_str_by_index(const struct llama_model * model, int i, char * buf, size_t buf_size);+ LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);// Get a string describing the model type- LLAMA_API int llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);+ LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);// Returns the total size of all the tensors in the model in bytesLLAMA_API uint64_t llama_model_size(const struct llama_model * model);@@ -351,7 +354,7 @@ extern "C" {LLAMA_API struct ggml_tensor * llama_get_model_tensor(struct llama_model * model, const char * name);// Returns 0 on success- LLAMA_API int llama_model_quantize(+ LLAMA_API uint32_t llama_model_quantize(const char * fname_inp,const char * fname_out,const llama_model_quantize_params * params);@@ -362,20 +365,20 @@ extern "C" {// The model needs to be reloaded before applying a new adapter, otherwise the adapter// will be applied on top of the previous one// Returns 0 on success- LLAMA_API DEPRECATED(int llama_apply_lora_from_file(+ LLAMA_API DEPRECATED(int32_t llama_apply_lora_from_file(struct llama_context * ctx,const char * path_lora,float scale,const char * path_base_model,- int n_threads),+ int32_t n_threads),"use llama_model_apply_lora_from_file instead");- LLAMA_API int llama_model_apply_lora_from_file(+ LLAMA_API int32_t llama_model_apply_lora_from_file(const struct llama_model * model,const char * path_lora,float scale,const char * path_base_model,- int n_threads);+ int32_t n_threads);//// KV cache@@ -431,10 +434,10 @@ extern "C" {// Returns the number of tokens in the KV cache (slow, use only for debug)// If a KV cell has multiple sequences assigned to it, it will be counted multiple times- LLAMA_API int llama_get_kv_cache_token_count(const struct llama_context * ctx);+ LLAMA_API int32_t llama_get_kv_cache_token_count(const struct llama_context * ctx);// Returns the number of used KV cells (i.e. have at least one sequence assigned to them)- LLAMA_API int llama_get_kv_cache_used_cells(const struct llama_context * ctx);+ LLAMA_API int32_t llama_get_kv_cache_used_cells(const struct llama_context * ctx);// Clear the KV cacheLLAMA_API void llama_kv_cache_clear(@@ -525,7 +528,7 @@ extern "C" {struct llama_context * ctx,llama_token * tokens,int32_t n_tokens,- int n_past),+ int32_t n_past),"use llama_decode() instead");// Same as llama_eval, but use float matrix input directly.@@ -534,7 +537,7 @@ extern "C" {struct llama_context * ctx,float * embd,int32_t n_tokens,- int n_past),+ int32_t n_past),"use llama_decode() instead");// Return batch for single sequence of tokens starting at pos_0@@ -566,7 +569,7 @@ extern "C" {// 0 - success// 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)// < 0 - error- LLAMA_API int llama_decode(+ LLAMA_API int32_t llama_decode(struct llama_context * ctx,struct llama_batch batch);@@ -606,10 +609,10 @@ extern "C" {LLAMA_API llama_token llama_token_nl (const struct llama_model * model); // next-line// Returns -1 if unknown, 1 for true or 0 for false.- LLAMA_API int llama_add_bos_token(const struct llama_model * model);+ LLAMA_API int32_t llama_add_bos_token(const struct llama_model * model);// Returns -1 if unknown, 1 for true or 0 for false.- LLAMA_API int llama_add_eos_token(const struct llama_model * model);+ LLAMA_API int32_t llama_add_eos_token(const struct llama_model * model);// codellama infill tokensLLAMA_API llama_token llama_token_prefix(const struct llama_model * model); // Beginning of infill prefix@@ -627,12 +630,12 @@ extern "C" {/// @return Returns a negative number on failure - the number of tokens that would have been returned/// @param special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated as plaintext./// Does not insert a leading space.- LLAMA_API int llama_tokenize(+ LLAMA_API int32_t llama_tokenize(const struct llama_model * model,const char * text,- int text_len,+ int32_t text_len,llama_token * tokens,- int n_max_tokens,+ int32_t n_max_tokens,bool add_bos,bool special);@@ -640,11 +643,11 @@ extern "C" {// Uses the vocabulary in the provided context.// Does not write null terminator to the buffer.// User code is responsible to remove the leading whitespace of the first non-BOS token when decoding multiple tokens.- LLAMA_API int llama_token_to_piece(+ LLAMA_API int32_t llama_token_to_piece(const struct llama_model * model,llama_token token,char * buf,- int length);+ int32_t length);//// Grammar@@ -696,7 +699,7 @@ extern "C" {LLAMA_API void llama_sample_top_k(struct llama_context * ctx,llama_token_data_array * candidates,- int k,+ int32_t k,size_t min_keep);/// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751@@ -755,7 +758,7 @@ extern "C" {llama_token_data_array * candidates,float tau,float eta,- int m,+ int32_t m,float * mu);/// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words.@@ -828,8 +831,8 @@ extern "C" {llama_beam_search_callback_fn_t callback,void * callback_data,size_t n_beams,- int n_past,- int n_predict);+ int32_t n_past,+ int32_t n_predict);// Performance informationLLAMA_API struct llama_timings llama_get_timings(struct llama_context * ctx);diff --git a/llama.cpp/llava/clip.cpp b/llama.cpp/llava/clip.cppindex 7bc674df..d2d831f7 100644--- a/llama.cpp/llava/clip.cpp+++ b/llama.cpp/llava/clip.cpp@@ -18,8 +18,9 @@#include "clip.h"#include "llama.cpp/ggml.h"-#include "llama.cpp/ggml-alloc.h"#include "llama.cpp/runtime.h"+#include "llama.cpp/ggml-alloc.h"+#include "llama.cpp/ggml-backend.h"#define STB_IMAGE_IMPLEMENTATION#include "llama.cpp/stb_image.h"@@ -143,6 +144,27 @@ static std::string get_ftype(int ftype) {}}+//+// image data+//++// RGB uint8 image+struct clip_image_u8 {+ int nx;+ int ny;++ std::vector<uint8_t> buf;+};++// RGB float32 image (NHWC)+// Memory layout: RGBRGBRGB...+struct clip_image_f32 {+ int nx;+ int ny;++ std::vector<float> buf;+};+//// clip layers//@@ -200,39 +222,31 @@ struct clip_vision_model {struct ggml_tensor * mm_2_b;};-// Replacement for std::vector<uint8_t> that doesn't require zero-initialization.-struct clip_buffer {- uint8_t * data = NULL;- size_t size = 0;-- void resize(size_t size) {- delete[] data;- data = new uint8_t[size];- this->size = size;- }-- ~clip_buffer() { delete[] data; }-};-struct clip_ctx {- bool has_text_encoder = false;- bool has_vision_encoder = false;+ bool has_text_encoder = false;+ bool has_vision_encoder = false;bool has_llava_projector = false;+struct clip_vision_model vision_model;+float image_mean[3];float image_std[3];bool use_gelu = false;int32_t ftype = 1;- struct ggml_context * ctx;+struct gguf_context * ctx_gguf;+ struct ggml_context * ctx_data;++ std::vector<uint8_t> buf_compute_meta;// memory buffers to evaluate the model- clip_buffer buf_compute;- clip_buffer buf_alloc;- ggml_allocr * alloc = NULL;+ ggml_backend_buffer_t params_buffer = NULL;+ ggml_backend_buffer_t compute_buffer = NULL;+ ggml_backend_t backend = NULL;+ ggml_allocr * compute_alloc = NULL;};-static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_image_f32_batch * imgs) {+static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch * imgs) {if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");return nullptr;@@ -253,28 +267,24 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima//const int projection_dim = hparams.projection_dim;const float eps = hparams.eps;int batch_size = imgs->size;- if(ctx->has_llava_projector) {+ if (ctx->has_llava_projector) {GGML_ASSERT(batch_size == 1);}- const auto & buf_compute = ctx->buf_compute;-struct ggml_init_params params = {- /*.mem_size =*/ buf_compute.size,- /*.mem_buffer =*/ buf_compute.data,- /*.no_alloc =*/ false,+ /*.mem_size =*/ ctx->buf_compute_meta.size(),+ /*.mem_buffer =*/ ctx->buf_compute_meta.data(),+ /*.no_alloc =*/ true,};- params.no_alloc = true;-struct ggml_context * ctx0 = ggml_init(params);struct ggml_cgraph * gf = ggml_new_graph(ctx0);struct ggml_tensor * inp_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, image_size, image_size, 3, batch_size);- ggml_allocr_alloc(ctx->alloc, inp_raw);+ ggml_allocr_alloc(ctx->compute_alloc, inp_raw);- if (!ggml_allocr_is_measure(ctx->alloc)) {- float * data = (float *)ggml_get_data(inp_raw);+ if (!ggml_allocr_is_measure(ctx->compute_alloc)) {+ float * data = (float *)malloc(ggml_nbytes(inp_raw));for (size_t i = 0; i < imgs->size; i++) {const int nx = imgs->data[i].nx;@@ -287,12 +297,14 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_imafor (int k = 0; k < 3; k++) {for (int y = 0; y < ny; y++) {for (int x = 0; x < nx; x++) {- data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].data[3 * (y * nx + x) + k];+ data[(b * 3 * n) + k * n + y * nx + x] = imgs->data[b].buf[3 * (y * nx + x) + k];}}}}}+ ggml_backend_tensor_set(inp_raw, data, 0, ggml_nbytes(inp_raw));+ free(data);}struct ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings, inp_raw, patch_size, patch_size, 0, 0, 1, 1);@@ -302,42 +314,39 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima// concat class_embeddings and patch_embeddingsstruct ggml_tensor * embeddings = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, num_positions, batch_size);- ggml_allocr_alloc(ctx->alloc, embeddings);- if (!ggml_allocr_is_measure(ctx->alloc)) {- ggml_set_zero(embeddings);+ ggml_allocr_alloc(ctx->compute_alloc, embeddings);+ if (!ggml_allocr_is_measure(ctx->compute_alloc)) {+ void* zero_mem = malloc(ggml_nbytes(embeddings));+ memset(zero_mem, 0, ggml_nbytes(embeddings));+ ggml_backend_tensor_set(embeddings, zero_mem, 0, ggml_nbytes(embeddings));+ free(zero_mem);}- struct ggml_tensor * temp = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, hidden_size, 1, batch_size);- ggml_allocr_alloc(ctx->alloc, temp);+ embeddings = ggml_acc(ctx0, embeddings, model.class_embedding,+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], 0);- embeddings = ggml_acc(ctx0, embeddings, ggml_repeat(ctx0, model.class_embedding, temp), embeddings->nb[1],- embeddings->nb[2], embeddings->nb[3], 0);- embeddings =- ggml_acc(ctx0, embeddings, inp, embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);+ embeddings = ggml_acc(ctx0, embeddings, inp,+ embeddings->nb[1], embeddings->nb[2], embeddings->nb[3], model.class_embedding->nb[1]);struct ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_positions);- ggml_allocr_alloc(ctx->alloc, positions);- if (!ggml_allocr_is_measure(ctx->alloc)) {+ ggml_allocr_alloc(ctx->compute_alloc, positions);+ if (!ggml_allocr_is_measure(ctx->compute_alloc)) {+ int* positions_data = (int*)malloc(ggml_nbytes(positions));for (int i = 0; i < num_positions; i++) {- ggml_set_i32_1d(positions, i, i);+ positions_data[i] = i;}+ ggml_backend_tensor_set(positions, positions_data, 0, ggml_nbytes(positions));+ free(positions_data);}embeddings =- ggml_add(ctx0, embeddings, ggml_repeat(ctx0, ggml_get_rows(ctx0, model.position_embeddings, positions), embeddings));+ ggml_add(ctx0, embeddings, ggml_get_rows(ctx0, model.position_embeddings, positions));// pre-layernorm{embeddings = ggml_norm(ctx0, embeddings, eps);- embeddings = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.pre_ln_w, embeddings), embeddings),- ggml_repeat(ctx0, model.pre_ln_b, embeddings));- }-- struct ggml_tensor * KQ_scale = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);- ggml_allocr_alloc(ctx->alloc, KQ_scale);- if (!ggml_allocr_is_measure(ctx->alloc)) {- ggml_set_f32(KQ_scale, 1.0f / sqrt((float)d_head));+ embeddings = ggml_add(ctx0, ggml_mul(ctx0, embeddings, model.pre_ln_w), model.pre_ln_b);}// loop over layers@@ -350,30 +359,30 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima{cur = ggml_norm(ctx0, cur, eps);- cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_1_w, cur), cur),- ggml_repeat(ctx0, model.layers[il].ln_1_b, cur));+ cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_1_w),+ model.layers[il].ln_1_b);}// self-attention{struct ggml_tensor * Q =- ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].q_b, cur), ggml_mul_mat(ctx0, model.layers[il].q_w, cur));+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].q_w, cur), model.layers[il].q_b);- Q = ggml_scale_inplace(ctx0, Q, KQ_scale);+ Q = ggml_scale_inplace(ctx0, Q, 1.0f / sqrt((float)d_head));Q = ggml_reshape_4d(ctx0, Q, d_head, n_head, num_positions, batch_size);Q = ggml_cont(ctx0, ggml_permute(ctx0, Q, 0, 2, 1, 3));Q = ggml_reshape_3d(ctx0, Q, d_head, num_positions, n_head * batch_size);struct ggml_tensor * K =- ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].k_b, cur), ggml_mul_mat(ctx0, model.layers[il].k_w, cur));+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].k_w, cur), model.layers[il].k_b);K = ggml_reshape_4d(ctx0, K, d_head, n_head, num_positions, batch_size);K = ggml_cont(ctx0, ggml_permute(ctx0, K, 0, 2, 1, 3));K = ggml_reshape_3d(ctx0, K, d_head, num_positions, n_head * batch_size);struct ggml_tensor * V =- ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].v_b, cur), ggml_mul_mat(ctx0, model.layers[il].v_w, cur));+ ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].v_w, cur), model.layers[il].v_b);V = ggml_reshape_4d(ctx0, V, d_head, n_head, num_positions, batch_size);V = ggml_cont(ctx0, ggml_permute(ctx0, V, 1, 2, 0, 3));@@ -389,7 +398,7 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima}// attention output- cur = ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].o_b, cur), ggml_mul_mat(ctx0, model.layers[il].o_w, cur));+ cur = ggml_add(ctx0, ggml_mul_mat(ctx0, model.layers[il].o_w, cur), model.layers[il].o_b);// re-add the layer input, e.g., residualcur = ggml_add(ctx0, cur, embeddings);@@ -400,12 +409,11 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima{cur = ggml_norm(ctx0, cur, eps);- cur = ggml_add(ctx0, ggml_mul(ctx0, ggml_repeat(ctx0, model.layers[il].ln_2_w, cur), cur),- ggml_repeat(ctx0, model.layers[il].ln_2_b, cur));+ cur = ggml_add(ctx0, ggml_mul(ctx0, cur, model.layers[il].ln_2_w), model.layers[il].ln_2_b);}cur = ggml_mul_mat(ctx0, model.layers[il].ff_i_w, cur);- cur = ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].ff_i_b, cur), cur);+ cur = ggml_add(ctx0, cur, model.layers[il].ff_i_b);if (ctx->use_gelu) {cur = ggml_gelu_inplace(ctx0, cur);@@ -414,7 +422,7 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima}cur = ggml_mul_mat(ctx0, model.layers[il].ff_o_w, cur);- cur = ggml_add(ctx0, ggml_repeat(ctx0, model.layers[il].ff_o_b, cur), cur);+ cur = ggml_add(ctx0, cur, model.layers[il].ff_o_b);// residual 2cur = ggml_add(ctx0, embeddings, cur);@@ -427,23 +435,26 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_imaembeddings = ggml_reshape_2d(ctx0, embeddings, embeddings->ne[0], embeddings->ne[1]);struct ggml_tensor * patches = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_patches);- ggml_allocr_alloc(ctx->alloc, patches);- if (!ggml_allocr_is_measure(ctx->alloc)) {- for (int i = 0; i < num_patches; ++i) {- ggml_set_i32_1d(patches, i, i+1);+ ggml_allocr_alloc(ctx->compute_alloc, patches);+ if (!ggml_allocr_is_measure(ctx->compute_alloc)) {+ int* patches_data = (int*)malloc(ggml_nbytes(patches));+ for (int i = 0; i < num_patches; i++) {+ patches_data[i] = i + 1;}+ ggml_backend_tensor_set(patches, patches_data, 0, ggml_nbytes(patches));+ free(patches_data);}embeddings = ggml_get_rows(ctx0, embeddings, patches);// mm projection 0embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);- embeddings = ggml_add(ctx0, ggml_repeat(ctx0, model.mm_0_b, embeddings), embeddings);+ embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);embeddings = ggml_gelu(ctx0, embeddings);embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);- embeddings = ggml_add(ctx0, ggml_repeat(ctx0, model.mm_2_b, embeddings), embeddings);+ embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);}// build the graph@@ -456,7 +467,6 @@ static ggml_cgraph * clip_image_build_graph(const clip_ctx * ctx, const clip_ima// read and create ggml_context containing the tensors and their datastruct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {-struct ggml_context * meta = NULL;struct gguf_init_params params = {@@ -494,7 +504,7 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {fprintf(stderr, "%s: ftype: %s\n", __func__, ftype_str.c_str());fprintf(stderr, "\n");}-+ const int n_tensors = gguf_get_n_tensors(ctx);// kvif (verbosity >= 3) {const int n_kv = gguf_get_n_kv(ctx);@@ -508,28 +518,42 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {}// data- size_t ctx_size = 0;+ size_t buffer_size = 0;{- const int n_tensors = gguf_get_n_tensors(ctx);-for (int i = 0; i < n_tensors; ++i) {const char * name = gguf_get_tensor_name(ctx, i);const size_t offset = gguf_get_tensor_offset(ctx, i);-struct ggml_tensor * cur = ggml_get_tensor(meta, name);- ctx_size += sizeof(struct ggml_tensor) + GGML_OBJECT_SIZE;size_t tensor_size = ggml_nbytes(cur);- size_t padded_size = ggml_nbytes_pad(cur);- ctx_size += padded_size;+ buffer_size += tensor_size;if (verbosity >= 3) {- fprintf(stderr, "%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, padded_size=%zu, offset=%zu\n", __func__, i,- cur->n_dims, cur->name, tensor_size, padded_size, offset);+ fprintf(stderr, "%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu\n", __func__, i,+ ggml_n_dims(cur), cur->name, tensor_size, offset);}}}+ buffer_size += n_tensors * 128 /* CLIP PADDING */;+clip_ctx * new_clip = new clip_ctx;+#if 0+ // TODO(jart): destroy this backend framework+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {+ new_clip->backend = ggml_backend_cuda_init(0);+ fprintf(stderr, "%s: CLIP using CUDA backend\n", __func__);+ }+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {+ new_clip->backend = ggml_backend_metal_init();+ fprintf(stderr, "%s: CLIP using Metal backend\n", __func__);+ }+#endif++ if (!new_clip->backend) {+ new_clip->backend = ggml_backend_cpu_init();+ fprintf(stderr, "%s: CLIP using CPU backend\n", __func__);+ }+// model size and capabilities{int idx = get_key_idx(ctx, KEY_HAS_TEXT_ENC);@@ -554,21 +578,24 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {fprintf(stderr, "%s: text_encoder: %d\n", __func__, new_clip->has_text_encoder);fprintf(stderr, "%s: vision_encoder: %d\n", __func__, new_clip->has_vision_encoder);fprintf(stderr, "%s: llava_projector: %d\n", __func__, new_clip->has_llava_projector);- fprintf(stderr, "%s: model size: %.2f MB\n", __func__, (ctx_size / 1024.0 / 1024.0));+ fprintf(stderr, "%s: model size: %.2f MB\n", __func__, buffer_size / 1024.0 / 1024.0);fprintf(stderr, "%s: metadata size: %.2f MB\n", __func__, ggml_get_mem_size(meta) / 1024.0 / 1024.0);}}+ fprintf(stderr, "%s: params backend buffer size = % 6.2f MB (%i tensors)\n", __func__, buffer_size / (1024.0 * 1024.0), n_tensors);+// load tensors{+ std::vector<uint8_t> read_buf;struct ggml_init_params params = {- /*.mem_size =*/ ctx_size,+ /*.mem_size =*/ (n_tensors + 1) * ggml_tensor_overhead(),/*.mem_buffer =*/ NULL,- /*.no_alloc =*/ false,+ /*.no_alloc =*/ true,};- new_clip->ctx = ggml_init(params);- if (!new_clip->ctx) {+ new_clip->ctx_data = ggml_init(params);+ if (!new_clip->ctx_data) {fprintf(stderr, "%s: ggml_init() failed\n", __func__);clip_free(new_clip);return nullptr;@@ -581,13 +608,21 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {return nullptr;}- const int n_tensors = gguf_get_n_tensors(ctx);+ // add tensors to contextfor (int i = 0; i < n_tensors; ++i) {const char * name = gguf_get_tensor_name(ctx, i);struct ggml_tensor * t = ggml_get_tensor(meta, name);- struct ggml_tensor * cur = ggml_dup_tensor(new_clip->ctx, t);+ struct ggml_tensor * cur = ggml_dup_tensor(new_clip->ctx_data, t);ggml_set_name(cur, name);+ }+ // alloc memory and offload data+ new_clip->params_buffer = ggml_backend_alloc_buffer(new_clip->backend, buffer_size);+ ggml_allocr* alloc = ggml_allocr_new_from_buffer(new_clip->params_buffer);+ for (int i = 0; i < n_tensors; ++i) {+ const char * name = gguf_get_tensor_name(ctx, i);+ struct ggml_tensor * cur = ggml_get_tensor(new_clip->ctx_data, name);+ ggml_allocr_alloc(alloc, cur);const size_t offset = gguf_get_data_offset(ctx) + gguf_get_tensor_offset(ctx, i);llamafile_seek(fin, offset, SEEK_SET);if (!fin) {@@ -595,10 +630,18 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {clip_free(new_clip);return nullptr;}-- llamafile_read(fin, reinterpret_cast<void *>(cur->data), ggml_nbytes(t));+ int num_bytes = ggml_nbytes(cur);+ if (ggml_backend_buffer_is_host(new_clip->params_buffer)) {+ // for the CPU and Metal backend, we can read directly into the tensor+ llamafile_read(fin, reinterpret_cast<void *>(cur->data), num_bytes);+ } else {+ // read into a temporary buffer first, then copy to device memory+ read_buf.resize(num_bytes);+ llamafile_read(fin, reinterpret_cast<void *>(read_buf.data()), num_bytes);+ ggml_backend_tensor_set(cur, read_buf.data(), 0, num_bytes);+ }}-+ ggml_allocr_free(alloc);llamafile_close(fin);}@@ -607,20 +650,20 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {// load vision modelauto & vision_model = new_clip->vision_model;auto & hparams = vision_model.hparams;- hparams.hidden_size = get_u32(ctx, format(KEY_N_EMBD, "vision"));- hparams.n_head = get_u32(ctx, format(KEY_N_HEAD, "vision"));+ hparams.hidden_size = get_u32(ctx, format(KEY_N_EMBD, "vision"));+ hparams.n_head = get_u32(ctx, format(KEY_N_HEAD, "vision"));hparams.n_intermediate = get_u32(ctx, format(KEY_N_FF, "vision"));- hparams.n_layer = get_u32(ctx, format(KEY_N_BLOCK, "vision"));- hparams.image_size = get_u32(ctx, KEY_IMAGE_SIZE);- hparams.patch_size = get_u32(ctx, KEY_PATCH_SIZE);+ hparams.n_layer = get_u32(ctx, format(KEY_N_BLOCK, "vision"));+ hparams.image_size = get_u32(ctx, KEY_IMAGE_SIZE);+ hparams.patch_size = get_u32(ctx, KEY_PATCH_SIZE);hparams.projection_dim = get_u32(ctx, format(KEY_PROJ_DIM, "vision"));- hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision"));+ hparams.eps = get_f32(ctx, format(KEY_LAYER_NORM_EPS, "vision"));int idx_mean = get_key_idx(ctx, KEY_IMAGE_MEAN);- int idx_std = get_key_idx(ctx, KEY_IMAGE_STD);+ int idx_std = get_key_idx(ctx, KEY_IMAGE_STD);for (int i = 0; i < 3; ++i) {new_clip->image_mean[i] = *((const float *)gguf_get_arr_data(ctx, idx_mean));- new_clip->image_std[i] = *((const float *)gguf_get_arr_data(ctx, idx_std));+ new_clip->image_std[i] = *((const float *)gguf_get_arr_data(ctx, idx_std));}if (verbosity >= 2) {@@ -634,35 +677,35 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {fprintf(stderr, "v_n_layer %d\n", hparams.n_layer);}- vision_model.patch_embeddings = get_tensor(new_clip->ctx, TN_PATCH_EMBD);- vision_model.class_embedding = get_tensor(new_clip->ctx, TN_CLASS_EMBD);- vision_model.position_embeddings = get_tensor(new_clip->ctx, format(TN_POS_EMBD, "v"));- vision_model.pre_ln_w = get_tensor(new_clip->ctx, format(TN_LN_PRE, "v", "weight"));- vision_model.pre_ln_b = get_tensor(new_clip->ctx, format(TN_LN_PRE, "v", "bias"));- vision_model.mm_0_w = get_tensor(new_clip->ctx, format(TN_LLAVA_PROJ, 0, "weight"));- vision_model.mm_0_b = get_tensor(new_clip->ctx, format(TN_LLAVA_PROJ, 0, "bias"));- vision_model.mm_2_w = get_tensor(new_clip->ctx, format(TN_LLAVA_PROJ, 2, "weight"));- vision_model.mm_2_b = get_tensor(new_clip->ctx, format(TN_LLAVA_PROJ, 2, "bias"));+ vision_model.patch_embeddings = get_tensor(new_clip->ctx_data, TN_PATCH_EMBD);+ vision_model.class_embedding = get_tensor(new_clip->ctx_data, TN_CLASS_EMBD);+ vision_model.position_embeddings = get_tensor(new_clip->ctx_data, format(TN_POS_EMBD, "v"));+ vision_model.pre_ln_w = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "weight"));+ vision_model.pre_ln_b = get_tensor(new_clip->ctx_data, format(TN_LN_PRE, "v", "bias"));+ vision_model.mm_0_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "weight"));+ vision_model.mm_0_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 0, "bias"));+ vision_model.mm_2_w = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "weight"));+ vision_model.mm_2_b = get_tensor(new_clip->ctx_data, format(TN_LLAVA_PROJ, 2, "bias"));vision_model.layers.resize(hparams.n_layer);for (int il = 0; il < hparams.n_layer; ++il) {auto & layer = vision_model.layers[il];- layer.k_w = get_tensor(new_clip->ctx, format(TN_ATTN_K, "v", il, "weight"));- layer.q_w = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "v", il, "weight"));- layer.v_w = get_tensor(new_clip->ctx, format(TN_ATTN_V, "v", il, "weight"));- layer.o_w = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "v", il, "weight"));- layer.ln_1_w = get_tensor(new_clip->ctx, format(TN_LN_1, "v", il, "weight"));- layer.ln_2_w = get_tensor(new_clip->ctx, format(TN_LN_2, "v", il, "weight"));- layer.ff_i_w = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "v", il, "weight"));- layer.ff_o_w = get_tensor(new_clip->ctx, format(TN_FFN_UP, "v", il, "weight"));- layer.k_b = get_tensor(new_clip->ctx, format(TN_ATTN_K, "v", il, "bias"));- layer.q_b = get_tensor(new_clip->ctx, format(TN_ATTN_Q, "v", il, "bias"));- layer.v_b = get_tensor(new_clip->ctx, format(TN_ATTN_V, "v", il, "bias"));- layer.o_b = get_tensor(new_clip->ctx, format(TN_ATTN_OUTPUT, "v", il, "bias"));- layer.ln_1_b = get_tensor(new_clip->ctx, format(TN_LN_1, "v", il, "bias"));- layer.ln_2_b = get_tensor(new_clip->ctx, format(TN_LN_2, "v", il, "bias"));- layer.ff_i_b = get_tensor(new_clip->ctx, format(TN_FFN_DOWN, "v", il, "bias"));- layer.ff_o_b = get_tensor(new_clip->ctx, format(TN_FFN_UP, "v", il, "bias"));+ layer.k_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "weight"));+ layer.q_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_Q, "v", il, "weight"));+ layer.v_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_V, "v", il, "weight"));+ layer.o_w = get_tensor(new_clip->ctx_data, format(TN_ATTN_OUTPUT, "v", il, "weight"));+ layer.ln_1_w = get_tensor(new_clip->ctx_data, format(TN_LN_1, "v", il, "weight"));+ layer.ln_2_w = get_tensor(new_clip->ctx_data, format(TN_LN_2, "v", il, "weight"));+ layer.ff_i_w = get_tensor(new_clip->ctx_data, format(TN_FFN_DOWN, "v", il, "weight"));+ layer.ff_o_w = get_tensor(new_clip->ctx_data, format(TN_FFN_UP, "v", il, "weight"));+ layer.k_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_K, "v", il, "bias"));+ layer.q_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_Q, "v", il, "bias"));+ layer.v_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_V, "v", il, "bias"));+ layer.o_b = get_tensor(new_clip->ctx_data, format(TN_ATTN_OUTPUT, "v", il, "bias"));+ layer.ln_1_b = get_tensor(new_clip->ctx_data, format(TN_LN_1, "v", il, "bias"));+ layer.ln_2_b = get_tensor(new_clip->ctx_data, format(TN_LN_2, "v", il, "bias"));+ layer.ff_i_b = get_tensor(new_clip->ctx_data, format(TN_FFN_DOWN, "v", il, "bias"));+ layer.ff_o_b = get_tensor(new_clip->ctx_data, format(TN_FFN_UP, "v", il, "bias"));}}@@ -670,45 +713,45 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {new_clip->ctx_gguf = ctx;-// measure mem requirement and allocate+ // measure mem requirement and allocate{- static const size_t tensor_alignment = 32;- new_clip->buf_compute.resize(ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead());- new_clip->alloc = ggml_allocr_new_measure(tensor_alignment);+ new_clip->buf_compute_meta.resize(GGML_DEFAULT_GRAPH_SIZE * ggml_tensor_overhead() + ggml_graph_overhead());+ new_clip->compute_alloc = ggml_allocr_new_measure_from_backend(new_clip->backend);clip_image_f32_batch batch;batch.size = 1;ggml_cgraph * gf = clip_image_build_graph(new_clip, &batch);- size_t alloc_size = ggml_allocr_alloc_graph(new_clip->alloc, gf) + tensor_alignment;- ggml_allocr_free(new_clip->alloc);- new_clip->buf_alloc.resize(alloc_size);- new_clip->alloc = ggml_allocr_new(new_clip->buf_alloc.data, new_clip->buf_alloc.size, tensor_alignment);+ size_t compute_memory_buffer_size = ggml_allocr_alloc_graph(new_clip->compute_alloc, gf);+ ggml_allocr_free(new_clip->compute_alloc);+ new_clip->compute_buffer = ggml_backend_alloc_buffer(new_clip->backend, compute_memory_buffer_size);+ new_clip->compute_alloc = ggml_allocr_new_from_buffer(new_clip->compute_buffer);- fprintf(stderr, "%s: total allocated memory: %.2f MB\n", __func__, (new_clip->buf_compute.size + alloc_size)/1024.0/1024.0);+ fprintf(stderr, "%s: compute allocated memory: %.2f MB\n", __func__, compute_memory_buffer_size /1024.0/1024.0);}return new_clip;}-clip_image_u8 * make_clip_image_u8() {- auto img = new clip_image_u8();- return img;+struct clip_image_u8 * clip_image_u8_init() {+ return new clip_image_u8();+}++struct clip_image_f32 * clip_image_f32_init() {+ return new clip_image_f32();}-clip_image_f32 * make_clip_image_f32() { return new clip_image_f32(); }-void clip_image_u8_free(clip_image_u8 * img) { if (img->data) { delete[] img->data; } delete img; }-void clip_image_f32_free(clip_image_f32 * img) { if (img->data) { delete[] img->data; } delete img; }+void clip_image_u8_free (struct clip_image_u8 * img) { delete img; }+void clip_image_f32_free(struct clip_image_f32 * img) { delete img; }static void build_clip_img_from_data(const stbi_uc * data, int nx, int ny, clip_image_u8 * img) {img->nx = nx;img->ny = ny;- img->size = nx * ny * 3;- img->data = new uint8_t[img->size]();- memcpy(img->data, data, img->size);+ img->buf.resize(3 * nx * ny);+ memcpy(img->buf.data(), data, img->buf.size());}bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {int nx, ny, nc;- auto data = stbi_load(fname, &nx, &ny, &nc, 3);+ auto * data = stbi_load(fname, &nx, &ny, &nc, 3);if (!data) {fprintf(stderr, "%s: failed to load image '%s'\n", __func__, fname);return false;@@ -720,7 +763,7 @@ bool clip_image_load_from_file(const char * fname, clip_image_u8 * img) {bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img) {int nx, ny, nc;- auto data = stbi_load_from_memory(bytes, bytes_length, &nx, &ny, &nc, 3);+ auto * data = stbi_load_from_memory(bytes, bytes_length, &nx, &ny, &nc, 3);if (!data) {fprintf(stderr, "%s: failed to decode image bytes\n", __func__);return false;@@ -732,7 +775,7 @@ bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length// normalize: x = (x - mean) / std// TODO: implement bicubic interpolation instead of linear.-bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res, const bool pad2square) {+bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32 * res, const bool pad2square) {if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");return false;@@ -741,18 +784,17 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip// the logic below is to pad the shorter side to the longer side with a background color: rgb(122, 116, 104)// see https://github.com/haotian-liu/LLaVA/blob/e854a2bf85118c504f6f16bf5c3c7c92f8fa8c6b/llava/conversation.py#L113-L156- clip_image_u8 * temp = make_clip_image_u8(); // we will keep the input image data here temporarily+ clip_image_u8 * temp = clip_image_u8_init(); // we will keep the input image data here temporarilyif (pad2square && img->nx != img->ny) {int longer_side = std::max(img->nx, img->ny);temp->nx = longer_side;temp->ny = longer_side;- temp->size = 3 * longer_side * longer_side;- temp->data = new uint8_t[temp->size]();- uint8_t bc[3] = {122, 116, 104}; // bakground color in RGB from LLaVA+ temp->buf.resize(3 * longer_side * longer_side);+ const uint8_t bc[3] = {122, 116, 104}; // background color in RGB from LLaVA// fill with background color- for (size_t i = 0; i < temp->size; i++) {- temp->data[i] = bc[i % 3];+ for (size_t i = 0; i < temp->buf.size(); i++) {+ temp->buf[i] = bc[i % 3];}// copy from the input image@@ -760,17 +802,16 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clipfor (int x = 0; x < img->nx; x++) {const int i = 3 * (y * img->nx + x);const int j = 3 * (y * temp->nx + x);- temp->data[j] = img->data[i];- temp->data[j+1] = img->data[i+1];- temp->data[j+2] = img->data[i+2];+ temp->buf[j] = img->buf[i];+ temp->buf[j+1] = img->buf[i+1];+ temp->buf[j+2] = img->buf[i+2];}}} else {- temp->nx = img->nx;- temp->ny = img->ny;- temp->size = img->size;- temp->data = new uint8_t[temp->size]();- memcpy(&temp->data[0], &img->data[0], temp->size); // copy+ temp->nx = img->nx;+ temp->ny = img->ny;+ temp->buf.resize(img->buf.size());+ memcpy(temp->buf.data(), img->buf.data(), temp->buf.size());}const int nx = temp->nx;@@ -781,8 +822,7 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clipres->nx = nx2;res->ny = ny2;- res->size = 3 * nx2 * ny2;- res->data = new float[res->size]();+ res->buf.resize(3 * nx2 * ny2);const float scale = std::max(nx, ny) / (float)ctx->vision_model.hparams.image_size;@@ -813,10 +853,10 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clipconst int j10 = 3 * (y1 * nx + x0) + c;const int j11 = 3 * (y1 * nx + x1) + c;- const float v00 = temp->data[j00];- const float v01 = temp->data[j01];- const float v10 = temp->data[j10];- const float v11 = temp->data[j11];+ const float v00 = temp->buf[j00];+ const float v01 = temp->buf[j01];+ const float v10 = temp->buf[j10];+ const float v11 = temp->buf[j11];const float v0 = v00 * (1.0f - dx) + v01 * dx;const float v1 = v10 * (1.0f - dx) + v11 * dx;@@ -827,7 +867,7 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clipconst int i = 3 * (y * nx3 + x) + c;- res->data[i] = ((float(v2) / 255.0f) - m3[c]) / s3[c];+ res->buf[i] = ((float(v2) / 255.0f) - m3[c]) / s3[c];}}}@@ -837,12 +877,13 @@ bool clip_image_preprocess(const clip_ctx * ctx, const clip_image_u8 * img, clip}void clip_free(clip_ctx * ctx) {- ggml_free(ctx->ctx);+ ggml_free(ctx->ctx_data);gguf_free(ctx->ctx_gguf);+delete ctx;}-bool clip_image_encode(const clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {+bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, float * vec) {if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");return false;@@ -854,8 +895,7 @@ bool clip_image_encode(const clip_ctx * ctx, const int n_threads, clip_image_f32return clip_image_batch_encode(ctx, n_threads, &imgs, vec);}-bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs, float * vec) {-+bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs, float * vec) {if (!ctx->has_vision_encoder) {fprintf(stderr, "This gguf file seems to have no vision encoder\n");return false;@@ -867,29 +907,32 @@ bool clip_image_batch_encode(const clip_ctx * ctx, const int n_threads, const cl}// reset alloc buffer to clean the memory from previous invocations- ggml_allocr_reset(ctx->alloc);+ ggml_allocr_reset(ctx->compute_alloc);// build the inference graphggml_cgraph * gf = clip_image_build_graph(ctx, imgs);- ggml_allocr_alloc_graph(ctx->alloc, gf);+ ggml_allocr_alloc_graph(ctx->compute_alloc, gf);++ if (ggml_backend_is_cpu(ctx->backend)) {+ ggml_backend_cpu_set_n_threads(ctx->backend, n_threads);+ }- struct ggml_cplan plan = ggml_graph_plan(gf, n_threads);- if (plan.work_size > 0) {- plan.work_data = (uint8_t *)malloc(plan.work_size);+#if 0+ // TODO(jart): destroy this backend framework+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {+ if (ggml_backend_is_metal(ctx->backend)) {+ ggml_backend_metal_set_n_cb(ctx->backend, n_threads);+ }}+#endif- ggml_graph_compute(gf, &plan);+ ggml_backend_graph_compute(ctx->backend, gf);// the last node is the embedding tensor-struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 1];+ struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 1];// copy the embeddings to the location passed by the user- memcpy(vec, ggml_get_data_f32(embeddings), ggml_nbytes(embeddings));-- if (plan.work_size > 0) {- free(plan.work_data);- }-+ ggml_backend_tensor_get(embeddings, vec, 0, ggml_nbytes(embeddings));return true;}@@ -898,31 +941,32 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const iggml_type type = GGML_TYPE_Q4_1;switch (itype) {- case 2:- type = GGML_TYPE_Q4_0;- break;- case 3:- type = GGML_TYPE_Q4_1;- break;- case 6:- type = GGML_TYPE_Q5_0;- break;- case 7:- type = GGML_TYPE_Q5_1;- break;- case 8:- type = GGML_TYPE_Q8_0;- break;- default:- fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype);- return false;+ case 2:+ type = GGML_TYPE_Q4_0;+ break;+ case 3:+ type = GGML_TYPE_Q4_1;+ break;+ case 6:+ type = GGML_TYPE_Q5_0;+ break;+ case 7:+ type = GGML_TYPE_Q5_1;+ break;+ case 8:+ type = GGML_TYPE_Q8_0;+ break;+ default:+ fprintf(stderr, "%s: invalid quantization type %d\n", __func__, itype);+ return false;};- auto ctx_clip = clip_model_load(fname_inp, 2);+ auto * ctx_clip = clip_model_load(fname_inp, 2);+const auto & ctx_src = ctx_clip->ctx_gguf;- const auto & ctx_data = ctx_clip->ctx;+ const auto & ctx_data = ctx_clip->ctx_data;- auto ctx_out = gguf_init_empty();+ auto * ctx_out = gguf_init_empty();gguf_set_kv(ctx_out, ctx_src);gguf_set_val_u32(ctx_out, "general.quantization_version", GGML_QNT_VERSION);gguf_set_val_u32(ctx_out, "general.file_type", itype);@@ -971,7 +1015,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const i}// quantize only 2D tensors- quantize &= (cur->n_dims == 2);+ quantize &= (ggml_n_dims(cur) == 2);if (quantize) {new_type = type;@@ -1044,7 +1088,7 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const ifout.put(0);}- fprintf(stderr, "%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), cur->n_dims, quantize,+ fprintf(stderr, "%s: n_dims = %d | quantize=%d | size = %f MB -> %f MB\n", name.c_str(), ggml_n_dims(cur), quantize,orig_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0);}@@ -1060,8 +1104,8 @@ bool clip_model_quantize(const char * fname_inp, const char * fname_out, const igguf_free(ctx_out);{- fprintf(stderr, "%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);- fprintf(stderr, "%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);+ fprintf(stderr, "%s: original size = %8.2f MB\n", __func__, total_size_org / 1024.0 / 1024.0);+ fprintf(stderr, "%s: quantized size = %8.2f MB\n", __func__, total_size_new / 1024.0 / 1024.0);int64_t sum_all = 0;for (size_t i = 0; i < hist_all.size(); ++i) {diff --git a/llama.cpp/llava/clip.h b/llama.cpp/llava/clip.hindex f11df85d..458a256a 100644--- a/llama.cpp/llava/clip.h+++ b/llama.cpp/llava/clip.h@@ -35,31 +35,14 @@ struct clip_vision_hparams {float eps;};-/** load mmproj model */-CLIP_API struct clip_ctx * clip_model_load(const char * fname, const int verbosity);-/** free mmproj model */+CLIP_API struct clip_ctx * clip_model_load(const char * fname, int verbosity);+CLIP_API void clip_free(struct clip_ctx * ctx);-size_t clip_embd_nbytes(const struct clip_ctx * ctx);-int clip_n_patches(const struct clip_ctx * ctx);-int clip_n_mmproj_embd(const struct clip_ctx * ctx);+CLIP_API size_t clip_embd_nbytes(const struct clip_ctx * ctx);-// RGB uint8 image-struct clip_image_u8 {- int nx;- int ny;- uint8_t * data = NULL;- size_t size;-};--// RGB float32 image (NHWC)-// Memory layout: RGBRGBRGB...-struct clip_image_f32 {- int nx;- int ny;- float * data = NULL;- size_t size;-};+CLIP_API int clip_n_patches (const struct clip_ctx * ctx);+CLIP_API int clip_n_mmproj_embd(const struct clip_ctx * ctx);struct clip_image_u8_batch {struct clip_image_u8 * data;@@ -71,21 +54,22 @@ struct clip_image_f32_batch {size_t size;};-struct clip_image_u8 * make_clip_image_u8();-struct clip_image_f32 * make_clip_image_f32();-CLIP_API void clip_image_u8_free(clip_image_u8 * img);-CLIP_API void clip_image_f32_free(clip_image_f32 * img);+CLIP_API struct clip_image_u8 * clip_image_u8_init ();+CLIP_API struct clip_image_f32 * clip_image_f32_init();++CLIP_API void clip_image_u8_free (struct clip_image_u8 * img);+CLIP_API void clip_image_f32_free(struct clip_image_f32 * img);+CLIP_API bool clip_image_load_from_file(const char * fname, struct clip_image_u8 * img);+/** interpret bytes as an image file with length bytes_length, and use the result to populate img */CLIP_API bool clip_image_load_from_bytes(const unsigned char * bytes, size_t bytes_length, struct clip_image_u8 * img);-bool clip_image_preprocess(const struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res, const bool pad2square);-bool clip_image_encode(const struct clip_ctx * ctx, const int n_threads, struct clip_image_f32 * img, float * vec);--bool clip_image_batch_encode(const struct clip_ctx * ctx, const int n_threads, const struct clip_image_f32_batch * imgs,- float * vec);+CLIP_API bool clip_image_preprocess (struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32 * res, bool pad2square);+CLIP_API bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);+CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);-bool clip_model_quantize(const char * fname_inp, const char * fname_out, const int itype);+CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype);#ifdef __cplusplus}diff --git a/llama.cpp/llava/llava.cpp b/llama.cpp/llava/llava.cppindex 1e40db11..8f6c0e6b 100644--- a/llama.cpp/llava/llava.cpp+++ b/llama.cpp/llava/llava.cpp@@ -13,7 +13,7 @@#include "llama.cpp/base64.h"static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {- clip_image_f32 * img_res = make_clip_image_f32();+ clip_image_f32 * img_res = clip_image_f32_init();if (!clip_image_preprocess(ctx_clip, img, img_res, /*pad2square =*/ true)) {fprintf(stderr, "%s: unable to preprocess image\n", __func__);clip_image_f32_free(img_res);@@ -34,7 +34,7 @@ static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const cliconst int64_t t_img_enc_end_us = ggml_time_us();float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;- fprintf(stderr, "%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);+ printf("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);return true;}@@ -44,7 +44,7 @@ bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx *int n_llama_embd = llama_n_embd(llama_get_model(ctx_llama));auto n_image_embd = clip_n_mmproj_embd(ctx_clip);if (n_image_embd != n_llama_embd) {- fprintf(stderr, "%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);+ printf("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);return false;}return true;@@ -89,7 +89,7 @@ bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_}LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {- clip_image_u8 * img = make_clip_image_u8();+ clip_image_u8 * img = clip_image_u8_init();if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {clip_image_u8_free(img);fprintf(stderr, "%s: can't load image from bytes, is it a valid image?", __func__);@@ -113,7 +113,7 @@ LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct cl}static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {- auto file = fopen(path, "rbe");+ auto file = fopen(path, "rb");if (file == NULL) {fprintf(stderr, "%s: can't read file %s\n", __func__, path);return false;@@ -130,7 +130,14 @@ static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, longfclose(file);return false;}- fread(buffer, 1, fileSize, file); // Read the file into the buffer+ errno = 0;+ size_t ret = fread(buffer, 1, fileSize, file); // Read the file into the buffer+ if (ferror(file)) {+ die_fmt("read error: %s", strerror(errno));+ }+ if (ret != (size_t) fileSize) {+ die("unexpectedly reached end of file");+ }fclose(file); // Close the file*bytesOut = buffer;diff --git a/llama.cpp/perplexity/perplexity.1.asc b/llama.cpp/perplexity/perplexity.1.ascindex be203355..ba693d9b 100644--- a/llama.cpp/perplexity/perplexity.1.asc+++ b/llama.cpp/perplexity/perplexity.1.asc@@ -11,7 +11,7 @@ DDEESSCCRRIIPPTTIIOONNmodels. The llllaammaaffiillee--ppeerrpplleexxiittyy program can be used to gauge the qualityof an LLM implementation. It is defined as the exponentiated averagenegative log-likelihood of a sequence, calculated with exponent base e.- Lower perplexity is better.+ Lower perplexity scores are better.OOPPTTIIOONNSSThe following options are available:diff --git a/llama.cpp/sampling.cpp b/llama.cpp/sampling.cppindex 5a6cef06..006333dc 100644--- a/llama.cpp/sampling.cpp+++ b/llama.cpp/sampling.cpp@@ -152,11 +152,12 @@ static void sampler_queue(}}-llama_token llama_sampling_sample(+static llama_token llama_sampling_sample_impl(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,struct llama_context * ctx_cfg,- const int idx) {+ const int idx,+ bool is_resampling) { // Add a parameter to indicate if we are resamplingconst llama_sampling_params & params = ctx_sampling->params;const int n_vocab = llama_n_vocab(llama_get_model(ctx_main));@@ -176,8 +177,17 @@ llama_token llama_sampling_sample(llama_token id = 0;+ // Get a pointer to the logitsfloat * logits = llama_get_logits_ith(ctx_main, idx);+ // Declare original_logits at the beginning of the function scope+ std::vector<float> original_logits;++ if (!is_resampling) {+ // Only make a copy of the original logits if we are not in the resampling phase, not sure if I actually have to do this.+ original_logits = std::vector<float>(logits, logits + llama_n_vocab(llama_get_model(ctx_main)));+ }+// apply params.logit_bias mapfor (auto it = params.logit_bias.begin(); it != params.logit_bias.end(); it++) {logits[it->first] += it->second;@@ -196,12 +206,14 @@ llama_token llama_sampling_sample(}// apply penalties- if (!prev.empty()) {+ const auto& penalty_tokens = params.use_penalty_prompt_tokens ? params.penalty_prompt_tokens : prev;+ const int penalty_tokens_used_size = std::min((int)penalty_tokens.size(), penalty_last_n);+ if (penalty_tokens_used_size) {const float nl_logit = logits[llama_token_nl(llama_get_model(ctx_main))];llama_sample_repetition_penalties(ctx_main, &cur_p,- prev.data() + prev.size() - penalty_last_n,- penalty_last_n, penalty_repeat, penalty_freq, penalty_present);+ penalty_tokens.data() + penalty_tokens.size() - penalty_tokens_used_size,+ penalty_tokens_used_size, penalty_repeat, penalty_freq, penalty_present);if (!penalize_nl) {for (size_t idx = 0; idx < cur_p.size; idx++) {@@ -213,7 +225,8 @@ llama_token llama_sampling_sample(}}- if (ctx_sampling->grammar != NULL) {+ // If we are in the resampling phase, apply grammar checks before sampling logic+ if (is_resampling && ctx_sampling->grammar != NULL) {llama_sample_grammar(ctx_main, &cur_p, ctx_sampling->grammar);}@@ -255,9 +268,40 @@ llama_token llama_sampling_sample(}}+ if (ctx_sampling->grammar != NULL && !is_resampling) {+ // Create an array with a single token data element for the sampled id+ llama_token_data single_token_data = {id, logits[id], 0.0f};+ llama_token_data_array single_token_data_array = { &single_token_data, 1, false };++ // Apply grammar constraints to the single token+ llama_sample_grammar(ctx_main, &single_token_data_array, ctx_sampling->grammar);++ // Check if the token is valid according to the grammar by seeing if its logit has been set to -INFINITY+ bool is_valid = single_token_data_array.data[0].logit != -INFINITY;++ // If the token is not valid according to the grammar, perform resampling+ if (!is_valid) {+ LOG("Resampling because token %d: '%s' does not meet grammar rules\n", id, llama_token_to_piece(ctx_main, id).c_str());++ // Restore logits from the copy+ std::copy(original_logits.begin(), original_logits.end(), logits);++ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, true); // Pass true for is_resampling+ }+ }+return id;}+llama_token llama_sampling_sample(+ struct llama_sampling_context * ctx_sampling,+ struct llama_context * ctx_main,+ struct llama_context * ctx_cfg,+ const int idx) {+ // Call the implementation function with is_resampling set to false by default+ return llama_sampling_sample_impl(ctx_sampling, ctx_main, ctx_cfg, idx, false);+}+void llama_sampling_accept(struct llama_sampling_context * ctx_sampling,struct llama_context * ctx_main,diff --git a/llama.cpp/sampling.h b/llama.cpp/sampling.hindex fdfa9eed..f16ef97e 100644--- a/llama.cpp/sampling.h+++ b/llama.cpp/sampling.h@@ -36,6 +36,9 @@ typedef struct llama_sampling_params {float cfg_scale = 1.f; // how strong is guidancestd::unordered_map<llama_token, float> logit_bias; // logit bias for specific tokens++ std::vector<llama_token> penalty_prompt_tokens;+ bool use_penalty_prompt_tokens = false;} llama_sampling_params;// general sampler contextdiff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 0f5ff322..c54b055d 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -48,6 +48,7 @@ using json = nlohmann::json;struct server_params{std::string hostname = "127.0.0.1";+ std::string api_key;std::string public_path = "/zip/llama.cpp/server/public";int32_t port = 8080;int32_t read_timeout = 600;@@ -93,7 +94,7 @@ static inline bool is_base64(uint8_t c)return (isalnum(c) || (c == '+') || (c == '/'));}-static std::vector<uint8_t> base64_decode(std::string const &encoded_string)+static std::vector<uint8_t> base64_decode(const std::string & encoded_string){int i = 0;int j = 0;@@ -220,10 +221,10 @@ struct slot_imageint32_t id;bool request_encode_image = false;- float* image_embedding = nullptr;+ float * image_embedding = nullptr;int32_t image_tokens = 0;- clip_image_u8 img_data;+ clip_image_u8 * img_data;std::string prefix_prompt; // before of this image};@@ -390,7 +391,6 @@ struct llama_client_slotint32_t num_prompt_tokens = 0;int32_t num_prompt_tokens_processed = 0;- int32_t multibyte_pending = 0;json prompt;std::string generated_text;@@ -439,7 +439,6 @@ struct llama_client_slotstopped_word = false;stopped_limit = false;stopping_word = "";- multibyte_pending = 0;n_past = 0;sent_count = 0;sent_token_probs_index = 0;@@ -447,10 +446,12 @@ struct llama_client_slotgenerated_token_probs.clear();- for (slot_image &img : images)+ for (slot_image & img : images){free(img.image_embedding);- delete[] img.img_data.data;+ if (img.img_data) {+ clip_image_u8_free(img.img_data);+ }img.prefix_prompt = "";}@@ -776,6 +777,42 @@ struct llama_server_contextslot->prompt = "";}+ slot->sparams.penalty_prompt_tokens.clear();+ slot->sparams.use_penalty_prompt_tokens = false;+ const auto &penalty_prompt = data.find("penalty_prompt");+ if (penalty_prompt != data.end())+ {+ if (penalty_prompt->is_string())+ {+ const auto penalty_prompt_string = penalty_prompt->get<std::string>();+ auto penalty_tokens = llama_tokenize(model, penalty_prompt_string, false);+ slot->sparams.penalty_prompt_tokens.swap(penalty_tokens);+ if (slot->params.n_predict > 0)+ {+ slot->sparams.penalty_prompt_tokens.reserve(slot->sparams.penalty_prompt_tokens.size() + slot->params.n_predict);+ }+ slot->sparams.use_penalty_prompt_tokens = true;+ }+ else if (penalty_prompt->is_array())+ {+ const auto n_tokens = penalty_prompt->size();+ slot->sparams.penalty_prompt_tokens.reserve(n_tokens + std::max(0, slot->params.n_predict));+ const int n_vocab = llama_n_vocab(model);+ for (const auto &penalty_token : *penalty_prompt)+ {+ if (penalty_token.is_number_integer())+ {+ const auto tok = penalty_token.get<llama_token>();+ if (tok >= 0 && tok < n_vocab)+ {+ slot->sparams.penalty_prompt_tokens.push_back(tok);+ }+ }+ }+ slot->sparams.use_penalty_prompt_tokens = true;+ }+ }+slot->sparams.logit_bias.clear();if (json_value(data, "ignore_eos", false))@@ -828,24 +865,17 @@ struct llama_server_context{for (const auto &img : *images_data){- std::string data_b64 = img["data"].get<std::string>();+ const std::vector<uint8_t> image_buffer = base64_decode(img["data"].get<std::string>());+slot_image img_sl;img_sl.id = img.count("id") != 0 ? img["id"].get<int>() : slot->images.size();- int width, height, channels;- std::vector<uint8_t> image_buffer = base64_decode(data_b64);- data_b64.clear();- auto data = stbi_load_from_memory(image_buffer.data(), image_buffer.size(), &width, &height, &channels, 3);- if (!data) {+ img_sl.img_data = clip_image_u8_init();+ if (!clip_image_load_from_bytes(image_buffer.data(), image_buffer.size(), img_sl.img_data))+ {LOG_TEE("slot %i - failed to load image [id: %i]\n", slot->id, img_sl.id);return false;}- LOG_TEE("slot %i - image loaded [id: %i] resolution (%i x %i)\n", slot->id, img_sl.id, width, height);- img_sl.img_data.nx = width;- img_sl.img_data.ny = height;- img_sl.img_data.size = width * height * 3;- img_sl.img_data.data = new uint8_t[width * height * 3]();- memcpy(img_sl.img_data.data, data, width * height * 3);- stbi_image_free(data);+ LOG_TEE("slot %i - loaded image\n", slot->id);img_sl.request_encode_image = true;slot->images.push_back(img_sl);}@@ -1012,35 +1042,42 @@ struct llama_server_contextslot.generated_text += token_str;slot.has_next_token = true;- if (slot.multibyte_pending > 0)+ if (slot.ctx_sampling->params.use_penalty_prompt_tokens && result.tok != -1){- slot.multibyte_pending -= token_str.size();+ // we can change penalty_prompt_tokens because it is always created from scratch each request+ slot.ctx_sampling->params.penalty_prompt_tokens.push_back(result.tok);}- else if (token_str.size() == 1)++ // check if there is incomplete UTF-8 character at the end+ bool incomplete = false;+ for (unsigned i = 1; i < 5 && i <= slot.generated_text.size(); ++i){- const char c = token_str[0];- // 2-byte characters: 110xxxxx 10xxxxxx+ unsigned char c = slot.generated_text[slot.generated_text.size() - i];+ if ((c & 0xC0) == 0x80)+ {+ // continuation byte: 10xxxxxx+ continue;+ }if ((c & 0xE0) == 0xC0){- slot.multibyte_pending = 1;- // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx+ // 2-byte character: 110xxxxx ...+ incomplete = i < 2;}else if ((c & 0xF0) == 0xE0){- slot.multibyte_pending = 2;- // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx+ // 3-byte character: 1110xxxx ...+ incomplete = i < 3;}else if ((c & 0xF8) == 0xF0){- slot.multibyte_pending = 3;- }- else- {- slot.multibyte_pending = 0;+ // 4-byte character: 11110xxx ...+ incomplete = i < 4;}+ // else 1-byte character or invalid byte+ break;}- if (slot.multibyte_pending == 0)+ if (!incomplete){size_t pos = std::min(slot.sent_count, slot.generated_text.size());const std::string str_test = slot.generated_text.substr(pos);@@ -1075,7 +1112,7 @@ struct llama_server_context}}- if (slot.multibyte_pending > 0 && !slot.has_next_token)+ if (incomplete){slot.has_next_token = true;}@@ -1117,8 +1154,8 @@ struct llama_server_context{continue;}- clip_image_f32 img_res;- if (!clip_image_preprocess(clp_ctx, &img.img_data, &img_res, /*pad2square =*/ true))+ clip_image_f32 * img_res = clip_image_f32_init();+ if (!clip_image_preprocess(clp_ctx, img.img_data, img_res, /*pad2square =*/ true)){LOG_TEE("Error processing the given image");clip_free(clp_ctx);@@ -1133,11 +1170,12 @@ struct llama_server_contextreturn false;}LOG_TEE("slot %i - encoding image [id: %i]\n", slot.id, img.id);- if (!clip_image_encode(clp_ctx, params.n_threads, &img_res, img.image_embedding))+ if (!clip_image_encode(clp_ctx, params.n_threads, img_res, img.image_embedding)){LOG_TEE("Unable to encode image\n");return false;}+ clip_image_f32_free(img_res);img.request_encode_image = false;}@@ -1205,6 +1243,8 @@ struct llama_server_context{"repeat_penalty", slot.sparams.penalty_repeat},{"presence_penalty", slot.sparams.penalty_present},{"frequency_penalty", slot.sparams.penalty_freq},+ {"penalty_prompt_tokens", slot.sparams.penalty_prompt_tokens},+ {"use_penalty_prompt_tokens", slot.sparams.use_penalty_prompt_tokens},{"mirostat", slot.sparams.mirostat},{"mirostat_tau", slot.sparams.mirostat_tau},{"mirostat_eta", slot.sparams.mirostat_eta},@@ -1241,7 +1281,7 @@ struct llama_server_context{std::vector<completion_token_output> probs_output = {};const std::vector<llama_token> to_send_toks = llama_tokenize(ctx, tkn.text_to_send, false);- size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());+ size_t probs_pos = std::min(slot.sent_token_probs_index, slot.generated_token_probs.size());size_t probs_stop_pos = std::min(slot.sent_token_probs_index + to_send_toks.size(), slot.generated_token_probs.size());if (probs_pos < probs_stop_pos){@@ -1301,7 +1341,7 @@ struct llama_server_context{probs = std::vector<completion_token_output>(slot.generated_token_probs.begin(),- slot.generated_token_probs.begin() + slot.sent_token_probs_index);+ slot.generated_token_probs.end());}res.result_json["completion_probabilities"] = probs_vector_to_json(ctx, probs);}@@ -1980,6 +2020,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());+ printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);printf(" --embedding enable embedding vector output (default: %s)\n", params.embedding ? "enabled" : "disabled");printf(" -np N, --parallel N number of slots for process requests (default: %d)\n", params.n_parallel);@@ -1989,6 +2030,10 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --mmproj MMPROJ_FILE path to a multimodal projector file for LLaVA.\n");printf(" --log-disable disables logging to a file.\n");printf("\n");+ printf(" --override-kv KEY=TYPE:VALUE\n");+ printf(" advanced option to override model metadata by key. may be specified multiple times.\n");+ printf(" types: int, float, bool. example: --override-kv tokenizer.ggml.add_bos_token=bool:false\n");+ printf("\n");}static void server_params_parse(int argc, char **argv, server_params &sparams,@@ -2029,6 +2074,15 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}sparams.public_path = argv[i];}+ else if (arg == "--api-key")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ sparams.api_key = argv[i];+ }else if (arg == "--timeout" || arg == "-to"){if (++i >= argc)@@ -2254,6 +2308,39 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,server_verbose = true;#endif}+ else if (arg == "--nobrowser")+ {+ sparams.nobrowser = true;+ }+ else if (arg == "--unsecure")+ {+ sparams.unsecure = true;+ }+ else if (arg == "--server")+ {+ }+ else if (arg == "--nocompile")+ {+ FLAG_nocompile = true;+ }+ else if (arg == "--recompile")+ {+ FLAG_recompile = true;+ }+ else if (arg == "--gpu")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ FLAG_gpu = llamafile_gpu_parse(argv[i]);+ if (FLAG_gpu == -1)+ {+ fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);+ exit(1);+ }+ }else if (arg == "--mlock"){params.use_mlock = true;@@ -2322,42 +2409,51 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}else if (arg == "--log-disable"){- FLAG_log_disable = true;log_set_target(stdout);LOG_INFO("logging to file is disabled.", {});}- else if (arg == "--nobrowser")- {- sparams.nobrowser = true;- }- else if (arg == "--unsecure")- {- sparams.unsecure = true;- }- else if (arg == "--server")- {- }- else if (arg == "--nocompile")- {- FLAG_nocompile = true;- }- else if (arg == "--recompile")+ else if (arg == "--override-kv"){- FLAG_recompile = true;- }- else if (arg == "--gpu")- {- if (++i >= argc)- {+ if (++i >= argc) {invalid_param = true;break;}- FLAG_gpu = llamafile_gpu_parse(argv[i]);- if (FLAG_gpu == -1)- {- fprintf(stderr, "error: invalid --gpu flag value: %s\n", argv[i]);- exit(1);+ char * sep = strchr(argv[i], '=');+ if (sep == nullptr || sep - argv[i] >= 128) {+ fprintf(stderr, "error: Malformed KV override: %s\n", argv[i]);+ invalid_param = true;+ break;}+ struct llama_model_kv_override kvo;+ std::strncpy(kvo.key, argv[i], sep - argv[i]);+ kvo.key[sep - argv[i]] = 0;+ sep++;+ if (strncmp(sep, "int:", 4) == 0) {+ sep += 4;+ kvo.tag = LLAMA_KV_OVERRIDE_INT;+ kvo.int_value = std::atol(sep);+ } else if (strncmp(sep, "float:", 6) == 0) {+ sep += 6;+ kvo.tag = LLAMA_KV_OVERRIDE_FLOAT;+ kvo.float_value = std::atof(sep);+ } else if (strncmp(sep, "bool:", 5) == 0) {+ sep += 5;+ kvo.tag = LLAMA_KV_OVERRIDE_BOOL;+ if (std::strcmp(sep, "true") == 0) {+ kvo.bool_value = true;+ } else if (std::strcmp(sep, "false") == 0) {+ kvo.bool_value = false;+ } else {+ fprintf(stderr, "error: Invalid boolean value for KV override: %s\n", argv[i]);+ invalid_param = true;+ break;+ }+ } else {+ fprintf(stderr, "error: Invalid type for KV override: %s\n", argv[i]);+ invalid_param = true;+ break;+ }+ params.kv_overrides.push_back(kvo);}else{@@ -2366,6 +2462,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,exit(1);}}+ if (!params.kv_overrides.empty()) {+ params.kv_overrides.emplace_back(llama_model_kv_override());+ params.kv_overrides.back().key[0] = 0;+ }if (invalid_param){@@ -2452,7 +2552,7 @@ json oaicompat_completion_params_parse(llama_params["ignore_eos"] = json_value(body, "ignore_eos", false);llama_params["tfs_z"] = json_value(body, "tfs_z", default_sparams.tfs_z);- if (llama_params.count("grammar") != 0) {+ if (body.count("grammar") != 0) {llama_params["grammar"] = json_value(body, "grammar", json::object());}@@ -2681,8 +2781,11 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con}}-int server_cli(int argc, char ** argv) {-+int server_cli(int argc, char **argv)+{+#if SERVER_VERBOSE != 1+ log_disable();+#endif// own arguments required by this examplegpt_params params;server_params sparams;@@ -2719,6 +2822,32 @@ int server_cli(int argc, char ** argv) {httplib::Server svr;+ // Middleware for API key validation+ auto validate_api_key = [&sparams](const httplib::Request &req, httplib::Response &res) -> bool {+ // If API key is not set, skip validation+ if (sparams.api_key.empty()) {+ return true;+ }++ // Check for API key in the header+ auto auth_header = req.get_header_value("Authorization");+ std::string prefix = "Bearer ";+ if (auth_header.substr(0, prefix.size()) == prefix) {+ std::string received_api_key = auth_header.substr(prefix.size());+ if (received_api_key == sparams.api_key) {+ return true; // API key is valid+ }+ }++ // API key is invalid or not provided+ res.set_content("Unauthorized: Invalid API Key", "text/plain; charset=utf-8");+ res.status = 401; // Unauthorized++ LOG_WARNING("Unauthorized: Invalid API Key", {});++ return false;+ };+svr.set_default_headers({{"Server", "llama.cpp"},{"Access-Control-Allow-Origin", "*"},{"Access-Control-Allow-Headers", "content-type"}});@@ -2730,23 +2859,26 @@ int server_cli(int argc, char ** argv) {{ "user_name", llama.name_user.c_str() },{ "assistant_name", llama.name_assistant.c_str() }};- res.set_content(data.dump(), "application/json");+ res.set_content(data.dump(), "application/json; charset=utf-8");});- svr.Post("/completion", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ if (!validate_api_key(req, res)) {+ return;+ }json data = json::parse(req.body);const int task_id = llama.request_completion(data, false, false, -1);if (!json_value(data, "stream", false)) {std::string completion_text;task_result result = llama.next_result(task_id);if (!result.error && result.stop) {- res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json");+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");}else{res.status = 404;- res.set_content(result.result_json["content"], "text/plain");+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");return;}} else {@@ -2817,12 +2949,15 @@ int server_cli(int argc, char ** argv) {}}};- res.set_content(models.dump(), "application/json");+ res.set_content(models.dump(), "application/json; charset=utf-8");});// TODO: add mount point without "/v1" prefix -- how?- svr.Post("/v1/chat/completions", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post("/v1/chat/completions", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ if (!validate_api_key(req, res)) {+ return;+ }json data = oaicompat_completion_params_parse(json::parse(req.body));const int task_id = llama.request_completion(data, false, false, -1);@@ -2836,10 +2971,10 @@ int server_cli(int argc, char ** argv) {res.set_content(oaicompat_result.dump(-1, ' ', false,json::error_handler_t::replace),- "application/json");+ "application/json; charset=utf-8");} else {res.status = 500;- res.set_content(result.result_json["content"], "text/plain");+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");return;}} else {@@ -2891,8 +3026,11 @@ int server_cli(int argc, char ** argv) {}});- svr.Post("/infill", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){+ if (!validate_api_key(req, res)) {+ return;+ }json data = json::parse(req.body);const int task_id = llama.request_completion(data, true, false, -1);if (!json_value(data, "stream", false)) {@@ -2900,12 +3038,12 @@ int server_cli(int argc, char ** argv) {task_result result = llama.next_result(task_id);if (!result.error && result.stop){- res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json");+ res.set_content(result.result_json.dump(-1, ' ', false, json::error_handler_t::replace), "application/json; charset=utf-8");}else{res.status = 404;- res.set_content(result.result_json["content"], "text/plain");+ res.set_content(result.result_json["content"], "text/plain; charset=utf-8");return;}} else {@@ -2954,11 +3092,11 @@ int server_cli(int argc, char ** argv) {svr.Get("/model.json", [&llama](const httplib::Request &, httplib::Response &res){const json data = llama.get_model_props();- return res.set_content(data.dump(), "application/json");+ return res.set_content(data.dump(), "application/json; charset=utf-8");});svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res)- { return res.set_content("", "application/json"); });+ { return res.set_content("", "application/json; charset=utf-8"); });svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res){@@ -2969,7 +3107,7 @@ int server_cli(int argc, char ** argv) {tokens = llama.tokenize(body["content"], false);}const json data = format_tokenizer_response(tokens);- return res.set_content(data.dump(), "application/json");+ return res.set_content(data.dump(), "application/json; charset=utf-8");});svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)@@ -2983,7 +3121,7 @@ int server_cli(int argc, char ** argv) {}const json data = format_detokenized_response(content);- return res.set_content(data.dump(), "application/json");+ return res.set_content(data.dump(), "application/json; charset=utf-8");});svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)@@ -2998,9 +3136,19 @@ int server_cli(int argc, char ** argv) {{prompt = "";}- const int task_id = llama.request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);++ json image_data;+ if (body.count("image_data") != 0) {+ image_data = body["image_data"];+ }+ else+ {+ image_data = "";+ }++ const int task_id = llama.request_completion({ {"prompt", prompt}, { "n_predict", 0}, {"image_data", image_data} }, false, true, -1);task_result result = llama.next_result(task_id);- return res.set_content(result.result_json.dump(), "application/json");+ return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");});svr.set_logger(log_server_request);@@ -3022,20 +3170,24 @@ int server_cli(int argc, char ** argv) {{snprintf(buf, sizeof(buf), fmt, "Unknown Exception");}- res.set_content(buf, "text/plain");+ res.set_content(buf, "text/plain; charset=utf-8");res.status = 500;});#endifsvr.set_error_handler([](const httplib::Request &, httplib::Response &res){+ if (res.status == 401)+ {+ res.set_content("Unauthorized", "text/plain; charset=utf-8");+ }if (res.status == 400){- res.set_content("Invalid request", "text/plain");+ res.set_content("Invalid request", "text/plain; charset=utf-8");}- else if (res.status != 500)+ else if (res.status == 404){- res.set_content("File Not Found", "text/plain");+ res.set_content("File Not Found", "text/plain; charset=utf-8");res.status = 404;}});@@ -3058,6 +3210,7 @@ int server_cli(int argc, char ** argv) {// Set the base directory for serving static filessvr.set_base_dir(sparams.public_path);+ // to make it ctrl+clickable:LOG_TEE("\nllama server listening at http://%s:%d\n\n", sparams.hostname.c_str(), sparams.port);if (!sparams.nobrowser) {@@ -3096,12 +3249,15 @@ int server_cli(int argc, char ** argv) {}}- tinylog("loading weights...\n", NULL);+ std::unordered_map<std::string, std::string> log_data;+ log_data["hostname"] = sparams.hostname;+ log_data["port"] = std::to_string(sparams.port);++ if (!sparams.api_key.empty()) {+ log_data["api_key"] = "api_key: ****" + sparams.api_key.substr(sparams.api_key.length() - 4);+ }- LOG_INFO("HTTP server listening", {- {"hostname", sparams.hostname},- {"port", sparams.port},- });+ LOG_INFO("HTTP server listening", log_data);// run the HTTP server in a thread - see comment belowstd::thread t([&]()diff --git a/llama.cpp/train.cpp b/llama.cpp/train.cppdeleted file mode 100644index 56d09d85..00000000--- a/llama.cpp/train.cpp+++ /dev/null@@ -1,1498 +0,0 @@-#include "train.h"-#include "common.h"--#include <random>-#include <sstream>-#include <functional>--struct random_normal_distribution {- std::mt19937 gen;- std::normal_distribution<float> rd;- float min;- float max;-};--struct random_uniform_distribution {- std::mt19937 gen;- std::uniform_real_distribution<float> rd;-};--struct train_state * init_train_state() {- struct train_state * state = new struct train_state;- state->train_its = 0;- state->train_samples = 0;- state->train_tokens = 0;- state->train_epochs = 0;- state->shuffle_samples_hash = 0;- state->shuffle_sample_count = 0;- state->shuffle_next_sample = 0;- state->shuffle_rng_state_current = "";- state->shuffle_rng_state_next = "";-- state->opt = new struct ggml_opt_context;- state->opt->ctx = NULL;- state->opt->params = ggml_opt_default_params(GGML_OPT_ADAM);- state->opt->loss_after = 0.0f;-- return state;-}--void free_train_state(struct train_state * state) {- delete state->opt;- delete state;-}--struct random_normal_distribution * init_random_normal_distribution(- int seed, float mean, float std, float min, float max-) {- struct random_normal_distribution * rnd = (struct random_normal_distribution *) malloc(sizeof(struct random_normal_distribution));- rnd->gen = std::mt19937(seed);- rnd->rd = std::normal_distribution<float>{mean, std};- rnd->min = min;- rnd->max = max;- return rnd;-}--struct random_uniform_distribution * init_random_uniform_distribution(int seed, float min, float max) {- struct random_uniform_distribution * rnd = (struct random_uniform_distribution *) malloc(sizeof(struct random_uniform_distribution));- rnd->gen = std::mt19937(seed);- rnd->rd = std::uniform_real_distribution<float>{min, max};- return rnd;-}--void free_random_normal_distribution (struct random_normal_distribution * rnd) {- free(rnd);-}--void free_random_uniform_distribution(struct random_uniform_distribution * rnd) {- free(rnd);-}--struct ggml_tensor * randomize_tensor_normal(struct ggml_tensor * tensor, struct random_normal_distribution * rnd) {- float scale = 1.0f; // xavier- switch (tensor->n_dims) {- case 1:- scale /= sqrtf((float) tensor->ne[0]);- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0]);- *dst = scale * frand_normal(rnd);- }- break;- case 2:- scale /= sqrtf((float) tensor->ne[0]+tensor->ne[1]);- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]);- *dst = scale * frand_normal(rnd);- }- }- break;- case 3:- scale /= sqrtf((float) tensor->ne[0]+tensor->ne[1]);- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2]);- *dst = scale * frand_normal(rnd);- }- }- }- break;- case 4:- scale /= sqrtf((float) tensor->ne[0]+tensor->ne[1]);- for (int i3 = 0; i3 < tensor->ne[3]; i3++) {- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]);- *dst = scale * frand_normal(rnd);- }- }- }- }- break;- default:- die("Unsupported tensor->n_dims");- };- return tensor;-}--struct ggml_tensor * randomize_tensor_uniform(struct ggml_tensor * tensor, struct random_uniform_distribution * rnd) {- switch (tensor->n_dims) {- case 1:- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0]);- *dst = frand_uniform(rnd);- }- break;- case 2:- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1]);- *dst = frand_uniform(rnd);- }- }- break;- case 3:- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2]);- *dst = frand_uniform(rnd);- }- }- }- break;- case 4:- for (int i3 = 0; i3 < tensor->ne[3]; i3++) {- for (int i2 = 0; i2 < tensor->ne[2]; i2++) {- for (int i1 = 0; i1 < tensor->ne[1]; i1++) {- for (int i0 = 0; i0 < tensor->ne[0]; i0++) {- float * dst = (float *) ((char *) tensor->data + i0*tensor->nb[0] + i1*tensor->nb[1] + i2*tensor->nb[2] + i3*tensor->nb[3]);- *dst = frand_uniform(rnd);- }- }- }- }- break;- default:- die("Unsupported tensor->n_dims");- };- return tensor;-}--float frand() {- return (float)rand()/((float)(RAND_MAX) + 1.0f);-}--float frand_normal(struct random_normal_distribution * rnd) {- return fclamp(rnd->rd(rnd->gen), rnd->min, rnd->max);-}--float frand_uniform(struct random_uniform_distribution * rnd) {- return rnd->rd(rnd->gen);-}--int clamp(const int v, const int min, const int max) {- return ((v < min) ? (min) : (v > max) ? (max) : v);-}--float fclamp(const float v, const float min, const float max) {- return ((v < min) ? (min) : (v > max) ? (max) : v);-}--void assert_shape_1d(struct ggml_tensor * tensor, int64_t ne0) {- GGML_ASSERT(tensor->n_dims == 1);- GGML_ASSERT(tensor->ne[0] == ne0);-}--void assert_shape_2d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1) {- GGML_ASSERT(tensor->n_dims == 2);- GGML_ASSERT(tensor->ne[0] == ne0);- GGML_ASSERT(tensor->ne[1] == ne1);-}--void assert_shape_3d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1, int64_t ne2) {- GGML_ASSERT(tensor->n_dims == 3);- GGML_ASSERT(tensor->ne[0] == ne0);- GGML_ASSERT(tensor->ne[1] == ne1);- GGML_ASSERT(tensor->ne[2] == ne2);-}--void assert_shape_4d(struct ggml_tensor * tensor, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) {- GGML_ASSERT(tensor->n_dims == 4);- GGML_ASSERT(tensor->ne[0] == ne0);- GGML_ASSERT(tensor->ne[1] == ne1);- GGML_ASSERT(tensor->ne[2] == ne2);- GGML_ASSERT(tensor->ne[3] == ne3);-}--int64_t get_example_targets_batch(- struct llama_context * lctx,- struct ggml_tensor * tokens_input,- struct ggml_tensor * target_probs,- int64_t example_id,- const size_t * samples_offs,- const size_t * samples_begin,- const size_t * samples_size,- size_t samples_count,- const llama_token * train_data,- size_t n_train_data,- bool separate_with_eos,- bool separate_with_bos,- bool fill_with_next_samples,- bool sample_random_offsets-) {- GGML_ASSERT(samples_count > 0);- GGML_ASSERT(tokens_input->n_dims == 2);- GGML_ASSERT(target_probs->n_dims == 3);- int64_t n_vocab = target_probs->ne[0];- int64_t n_tokens = tokens_input->ne[0];- int64_t n_batch = tokens_input->ne[1];- GGML_ASSERT(n_vocab == target_probs->ne[0]);- GGML_ASSERT(n_tokens == target_probs->ne[1]);- GGML_ASSERT(n_batch == target_probs->ne[2]);-- int64_t used_samples = 0;-- ggml_set_f32(target_probs, 0.0f);- llama_token bos = llama_token_bos(llama_get_model(lctx));- llama_token eos = llama_token_eos(llama_get_model(lctx));- // printf("%s: example_id=%d n_batch=%d n_train_samples=%zu\n", __func__, example_id, n_batch, n_train_samples);- for (int k=0; k<n_batch; ++k) {- // printf("%s: batch %d\n", __func__, k);- size_t sample_idx = (example_id + used_samples) % samples_count;- size_t sample_offs = sample_random_offsets ? samples_offs[sample_idx] : 0;- size_t sample_begin = samples_begin[sample_idx];- size_t sample_size = samples_size[sample_idx];- ++used_samples;-- // printf("%s: sample_idx=%zu sample=%zu\n", __func__, sample_idx, sample);- GGML_ASSERT(sample_begin+sample_size-1 < n_train_data);-- ggml_set_i32_nd(tokens_input, 0, k, 0, 0, bos);- bool sample_separation_eos = !separate_with_eos;- bool sample_separation_bos = !separate_with_bos;- for (int64_t i=0; i<n_tokens; ++i) {- llama_token token = eos;- if (sample_offs >= sample_size && fill_with_next_samples) {- if (!sample_separation_eos) {- // insert eos token to separate samples- sample_separation_eos = true;- } else if (!sample_separation_bos) {- // insert bos token to separate samples- sample_separation_bos = true;- token = bos;- } else {- // sample separation is done, continue with next sample- sample_separation_eos = !separate_with_eos;- sample_separation_bos = !separate_with_bos;- sample_offs = 0;- sample_idx = (example_id + used_samples) % samples_count;- sample_begin = samples_begin[sample_idx];- sample_size = samples_size[sample_idx];- ++used_samples;- }- }- // note: no else-if here- if (sample_offs < sample_size) {- token = clamp(train_data[sample_begin+sample_offs], 0, (llama_token) (n_vocab - 1));- ++sample_offs;- }- ggml_set_f32_nd(target_probs, token, (int) i, (int) k, 0, +1.0f);- if (i+1<n_tokens) {- ggml_set_i32_nd(tokens_input, (int) (i + 1), (int) k, 0, 0, token);- }- }- }-- return used_samples;-}--void mt19937_set_state(std::mt19937& rng, const std::string& rng_state) {- std::stringstream s_rng_state;- s_rng_state.imbue(std::locale::classic());- s_rng_state.exceptions(std::stringstream::failbit);- s_rng_state.str(rng_state);- s_rng_state >> rng;-}--std::string mt19937_get_state(const std::mt19937& rng) {- std::stringstream s_rng_state;- s_rng_state.imbue(std::locale::classic());- s_rng_state << rng;- return s_rng_state.str();-}--std::string mt19937_seed_to_state(unsigned seed) {- std::mt19937 rng(seed);- return mt19937_get_state(rng);-}--std::string shuffle_samples(- const std::string & rng_state,- size_t * shuffled_offs,- size_t * shuffled_begins,- size_t * shuffled_sizes,- const size_t * begins,- const size_t * sizes,- size_t count) {- if (count == 0) return rng_state;-- std::mt19937 rng;- mt19937_set_state(rng, rng_state);-- // sort indices by random value for each index- std::vector<size_t> idcs;- {- std::vector<unsigned> rnd;- idcs.resize(count);- rnd.resize(count);- for (unsigned i=0; i<count; ++i) {- idcs[i] = i;- rnd[i] = rng();- }-- std::sort(idcs.begin(), idcs.end(), [&rnd](size_t a, size_t b){- // stable sort for reproducibility- return (rnd[a] == rnd[b]) ? (a < b) : (rnd[a] < rnd[b]);- });- }-- // create random offsets- for (unsigned i=0; i<count; ++i) {- shuffled_offs[i] = (size_t) ((sizes[idcs[i]] - 1) * ((double) rng() / (double) (rng.max()-1)));- }-- // reorder begins and sizes by sorted indices- for (unsigned i=0; i<count; ++i) {- shuffled_begins[i] = begins[idcs[i]];- }-- for (unsigned i=0; i<count; ++i) {- shuffled_sizes[i] = sizes[idcs[i]];- }-- return mt19937_get_state(rng);-}--size_t hash_combine(size_t h1, size_t h2) {- return h1 ^ (h2 << 1);-}--size_t compute_samples_hash(const char* fn, const size_t* samples_begin, const size_t* samples_size, size_t sample_count) {- std::hash<std::string> h_string;- std::hash<unsigned long long> h_ull;- size_t h = h_string(std::string(fn));- h = hash_combine(h, h_ull((unsigned long long) sample_count));- for (size_t i=0; i< sample_count; ++i) {- h = hash_combine(h, h_ull((unsigned long long) samples_begin[i]));- h = hash_combine(h, h_ull((unsigned long long) samples_size[i]));- }- return h;-}--std::string replace_str(const char * s, const char * needle, const char * replacement) {- std::string str = s;- size_t pos = str.find(needle);- if (pos != std::string::npos) {- str.replace(pos, strlen(needle), replacement);- }- return str;-}--void print_duration(double fmillis) {- if (fmillis < 1000.0f) {- printf("%.1fms", (float) fmillis);- return;- }- const int64_t one_sec = 1000;- const int64_t one_min = one_sec * 60;- const int64_t one_hour = one_min * 60;- const int64_t one_day = one_hour * 24;-- int64_t millis = (int64_t) fmillis;- int64_t days = millis/one_day;- int64_t hours = (millis - days*one_day)/one_hour;- int64_t minutes = (millis - days*one_day - hours*one_hour)/one_min;- int64_t seconds = (millis - days*one_day - hours*one_hour - minutes*one_min)/one_sec;-- // to print int64_t either cast to (long long int) or use macro PRId64 from <inttypes.h>- if (days > 0) {- printf("%lldd ", (long long int) days);- }- printf("%02lld:%02lld:%02lld", (long long int) hours, (long long int) minutes, (long long int) seconds);-}--float cosine_decay(int64_t step, int64_t decay_steps, float minimum) {- if (step > decay_steps) {- step = decay_steps;- }- const float cosine_decay = 0.50f*(1.0f + cosf(3.14159265359f*step/decay_steps));- const float decay = (1 - minimum)*cosine_decay + minimum;- return decay;-}--float cosine_decay_restart(int64_t step, int64_t decay_steps, float minimum, float restart_step_mult) {- while (step > decay_steps) {- step -= decay_steps;- decay_steps = (int64_t) (restart_step_mult * decay_steps);- }- return cosine_decay(step, decay_steps, minimum);-}--float learning_schedule(- int64_t step,- int64_t warmup_steps,- int64_t cos_decay_steps,- float learning_rate,- float overall_minimum,- float cos_decay_minimum,- float cos_decay_restart_step_mult,- bool enable_restart) {-- float result =- (step < warmup_steps)- ? (float) step / (float) warmup_steps- : enable_restart- ? cosine_decay_restart(- step - warmup_steps,- cos_decay_steps,- cos_decay_minimum,- cos_decay_restart_step_mult)- : cosine_decay(- step,- cos_decay_steps,- cos_decay_minimum);-- float min = overall_minimum / learning_rate;- result = min + result * (1.0f - min);- return result;-}--static bool are_same_layout(struct ggml_tensor * a, struct ggml_tensor * b) {- GGML_ASSERT(a != NULL);- GGML_ASSERT(b != NULL);- GGML_ASSERT(a->type == b->type);- GGML_ASSERT(ggml_are_same_shape(a, b));- GGML_ASSERT(ggml_is_contiguous(a) && ggml_is_contiguous(b));-- return true;-}--void copy_tensor_by_name(struct ggml_tensor * dst, struct ggml_context * ctx, const char * name) {- if (dst == NULL) {- return;- }- struct ggml_tensor * t = ggml_get_tensor(ctx, name);- GGML_ASSERT(are_same_layout(dst, t));- memcpy(dst->data, t->data, ggml_nbytes(t));-- if (strlen(ggml_get_name(dst)) == 0) {- ggml_set_name(dst, name);- }-}--// gguf constants-static const char * LLM_KV_OPTIMIZER_TYPE = "optimizer.type";-static const char * LLM_KV_OPTIMIZER_TYPE_ADAM = "adam";-static const char * LLM_KV_OPTIMIZER_TYPE_LBFGS = "lbfgs";-static const char * LLM_KV_OPTIMIZER_FILE_VERSION = "optimizer.file_version";-static const char * LLM_KV_OPTIMIZER_CONVERGENCE_PAST_COUNT = "optimizer.convergence_past_count";-static const char * LLM_KV_OPTIMIZER_PARAMETER_COUNT = "optimizer.parameter_count";-static const char * LLM_KV_OPTIMIZER_ITERATION_COUNT = "optimizer.iteration_count";-static const char * LLM_KV_OPTIMIZER_JUST_INITIALIZED = "optimizer.just_initialized";-static const char * LLM_KV_OPTIMIZER_ADAM_BEST_LOSS = "optimizer.adam.best_loss";-static const char * LLM_KV_OPTIMIZER_ADAM_PREVIOUS_LOSS = "optimizer.adam.previous_loss";-static const char * LLM_KV_OPTIMIZER_ADAM_NO_IMPROVEMENT_COUNT = "optimizer.adam.no_improvement_count";-static const char * LLM_KV_OPTIMIZER_LBFGS_APPROX_HESSIAN_COUNT = "optimizer.lbfgs.approx_hessian_count";-static const char * LLM_KV_OPTIMIZER_LBFGS_BEST_LOSS = "optimizer.lbfgs.best_loss";-static const char * LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_STEP = "optimizer.lbfgs.line_search_step";-static const char * LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_J = "optimizer.lbfgs.line_search_j";-static const char * LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_K = "optimizer.lbfgs.line_search_k";-static const char * LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_END = "optimizer.lbfgs.line_search_end";-static const char * LLM_KV_OPTIMIZER_LBFGS_NO_IMPROVEMENT_COUNT = "optimizer.lbfgs.no_improvement_count";--static const char * LLM_TENSOR_OPTIMIZER_ADAM_FIRST_MOMENTS = "optimizer.adam.first_moments";-static const char * LLM_TENSOR_OPTIMIZER_ADAM_SECOND_MOMENTS = "optimizer.adam.second_moments";-static const char * LLM_TENSOR_OPTIMIZER_ADAM_PAST_LOSS_VALUES = "optimizer.adam.past_loss_values";--static const char * LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_PARAMETERS = "optimizer.lbfgs.current_parameters";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_PARAMETERS = "optimizer.lbfgs.previous_parameters";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_GRADIENTS = "optimizer.lbfgs.current_gradients";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_GRADIENTS = "optimizer.lbfgs.previous_gradients";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_SEARCH_DIRECTION = "optimizer.lbfgs.search_direction";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_PAST_LOSS_VALUES = "optimizer.lbfgs.past_loss_values";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_ALPHA = "optimizer.lbfgs.memory_alpha";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_YS = "optimizer.lbfgs.memory_ys";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_S = "optimizer.lbfgs.memory_s";-static const char * LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_Y = "optimizer.lbfgs.memory_y";--static const char * LLM_KV_TRAINING_FILE_VERSION = "training.file_version";-static const char * LLM_KV_TRAINING_ITERATION_COUNT = "training.iteration_count";-static const char * LLM_KV_TRAINING_SAMPLE_COUNT = "training.sample_count";-static const char * LLM_KV_TRAINING_TOKEN_COUNT = "training.token_count";-static const char * LLM_KV_TRAINING_EPOCH_COUNT = "training.epoch_count";-static const char * LLM_KV_TRAINING_SHUFFLE_SAMPLES_HASH = "training.shuffle.samples_hash";-static const char * LLM_KV_TRAINING_SHUFFLE_RNG_STATE = "training.shuffle.rng_state";-static const char * LLM_KV_TRAINING_SHUFFLE_SAMPLE_COUNT = "training.shuffle.sample_count";-static const char * LLM_KV_TRAINING_SHUFFLE_NEXT_SAMPLE = "training.shuffle.next_sample";--#define GGUF_GET_KEY(ctx, dst, func, type, req, key) \-{ \- const std::string skey(key); \- const int kid = gguf_find_key(ctx, skey.c_str()); \- if (kid >= 0) { \- enum gguf_type ktype = gguf_get_kv_type(ctx, kid); \- if (ktype != (type)) { \- die_fmt("key %s has wrong type: %s", skey.c_str(), gguf_type_name(ktype)); \- } \- (dst) = func(ctx, kid); \- } else if (req) { \- die_fmt("key not found in model: %s", skey.c_str()); \- } \-}--void load_opt_context_gguf(struct gguf_context * fctx, struct ggml_context * f_ggml_ctx, struct ggml_opt_context * opt) {- // NOTE: gguf_context must be initialized with f_ggml_ctx and no_alloc=false, otherwise tensor data can not be read-- uint32_t file_version;- GGUF_GET_KEY(fctx, file_version, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_FILE_VERSION);- GGML_ASSERT(file_version == 0);-- GGUF_GET_KEY(fctx, opt->params.past, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_CONVERGENCE_PAST_COUNT);- GGUF_GET_KEY(fctx, opt->iter, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_ITERATION_COUNT);- GGUF_GET_KEY(fctx, opt->just_initialized, gguf_get_val_bool, GGUF_TYPE_BOOL, true, LLM_KV_OPTIMIZER_JUST_INITIALIZED);-- uint64_t nx;- GGUF_GET_KEY(fctx, nx, gguf_get_val_u64, GGUF_TYPE_UINT64, true, LLM_KV_OPTIMIZER_PARAMETER_COUNT);- opt->nx = (size_t) nx;-- // don't call ggml_opt_init until optimizer type and optimizer specific parameters are know-- std::string opt_type;- GGUF_GET_KEY(fctx, opt_type, gguf_get_val_str, GGUF_TYPE_STRING, true, LLM_KV_OPTIMIZER_TYPE);- if (opt_type == LLM_KV_OPTIMIZER_TYPE_ADAM) {- opt->params.type = GGML_OPT_ADAM;-- GGUF_GET_KEY(fctx, opt->adam.fx_best, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_ADAM_BEST_LOSS);- GGUF_GET_KEY(fctx, opt->adam.fx_prev, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_ADAM_PREVIOUS_LOSS);- GGUF_GET_KEY(fctx, opt->adam.n_no_improvement, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_ADAM_NO_IMPROVEMENT_COUNT);-- ggml_opt_init(opt->ctx, opt, opt->params, opt->nx);-- copy_tensor_by_name(opt->adam.m, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_ADAM_FIRST_MOMENTS);- copy_tensor_by_name(opt->adam.v, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_ADAM_SECOND_MOMENTS);- copy_tensor_by_name(opt->adam.pf, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_ADAM_PAST_LOSS_VALUES);- } else if (opt_type == LLM_KV_OPTIMIZER_TYPE_LBFGS) {- opt->params.type = GGML_OPT_LBFGS;-- GGUF_GET_KEY(fctx, opt->params.lbfgs.m, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_LBFGS_APPROX_HESSIAN_COUNT);- GGUF_GET_KEY(fctx, opt->lbfgs.fx_best, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_LBFGS_BEST_LOSS);- GGUF_GET_KEY(fctx, opt->lbfgs.step, gguf_get_val_f32, GGUF_TYPE_FLOAT32, true, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_STEP);- GGUF_GET_KEY(fctx, opt->lbfgs.j, gguf_get_val_i32, GGUF_TYPE_INT32, true, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_J);- GGUF_GET_KEY(fctx, opt->lbfgs.k, gguf_get_val_i32, GGUF_TYPE_INT32, true, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_K);- GGUF_GET_KEY(fctx, opt->lbfgs.end, gguf_get_val_i32, GGUF_TYPE_INT32, true, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_END);- GGUF_GET_KEY(fctx, opt->lbfgs.n_no_improvement, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_OPTIMIZER_LBFGS_NO_IMPROVEMENT_COUNT);-- ggml_opt_init(opt->ctx, opt, opt->params, opt->nx);-- copy_tensor_by_name(opt->lbfgs.x, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_PARAMETERS);- copy_tensor_by_name(opt->lbfgs.xp, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_PARAMETERS);- copy_tensor_by_name(opt->lbfgs.g, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_GRADIENTS);- copy_tensor_by_name(opt->lbfgs.gp, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_GRADIENTS);- copy_tensor_by_name(opt->lbfgs.d, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_SEARCH_DIRECTION);- copy_tensor_by_name(opt->lbfgs.pf, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_PAST_LOSS_VALUES);- copy_tensor_by_name(opt->lbfgs.lmal, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_ALPHA);- copy_tensor_by_name(opt->lbfgs.lmys, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_YS);- copy_tensor_by_name(opt->lbfgs.lms, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_S);- copy_tensor_by_name(opt->lbfgs.lmy, f_ggml_ctx, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_Y);- } else {- die("unknown optimizer type\n");- }-}--void save_opt_context_gguf(struct gguf_context * fctx, struct ggml_opt_context * opt) {- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_FILE_VERSION, 0);- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_CONVERGENCE_PAST_COUNT, opt->params.past);- gguf_set_val_u64(fctx, LLM_KV_OPTIMIZER_PARAMETER_COUNT, (uint64_t) opt->nx);- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_ITERATION_COUNT, opt->iter);- gguf_set_val_bool(fctx, LLM_KV_OPTIMIZER_JUST_INITIALIZED, opt->just_initialized);-- switch (opt->params.type) {- case GGML_OPT_ADAM:- {- gguf_set_val_str(fctx, LLM_KV_OPTIMIZER_TYPE, LLM_KV_OPTIMIZER_TYPE_ADAM);- gguf_set_val_f32(fctx, LLM_KV_OPTIMIZER_ADAM_BEST_LOSS, opt->adam.fx_best);- gguf_set_val_f32(fctx, LLM_KV_OPTIMIZER_ADAM_PREVIOUS_LOSS, opt->adam.fx_prev);- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_ADAM_NO_IMPROVEMENT_COUNT, opt->adam.n_no_improvement);-- ggml_set_name(opt->adam.m, LLM_TENSOR_OPTIMIZER_ADAM_FIRST_MOMENTS);- ggml_set_name(opt->adam.v, LLM_TENSOR_OPTIMIZER_ADAM_SECOND_MOMENTS);- if (opt->adam.pf) {- ggml_set_name(opt->adam.pf, LLM_TENSOR_OPTIMIZER_ADAM_PAST_LOSS_VALUES);- }-- gguf_add_tensor(fctx, opt->adam.m);- gguf_add_tensor(fctx, opt->adam.v);- if (opt->adam.pf) {- gguf_add_tensor(fctx, opt->adam.pf);- }- } break;- case GGML_OPT_LBFGS:- {- gguf_set_val_str(fctx, LLM_KV_OPTIMIZER_TYPE, LLM_KV_OPTIMIZER_TYPE_LBFGS);- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_LBFGS_APPROX_HESSIAN_COUNT, opt->params.lbfgs.m);- gguf_set_val_f32(fctx, LLM_KV_OPTIMIZER_LBFGS_BEST_LOSS, opt->lbfgs.fx_best);- gguf_set_val_f32(fctx, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_STEP, opt->lbfgs.step);- gguf_set_val_i32(fctx, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_J, opt->lbfgs.j);- gguf_set_val_i32(fctx, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_K, opt->lbfgs.k);- gguf_set_val_i32(fctx, LLM_KV_OPTIMIZER_LBFGS_LINE_SEARCH_END, opt->lbfgs.end);- gguf_set_val_u32(fctx, LLM_KV_OPTIMIZER_LBFGS_NO_IMPROVEMENT_COUNT, opt->lbfgs.n_no_improvement);-- ggml_set_name(opt->lbfgs.x, LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_PARAMETERS);- ggml_set_name(opt->lbfgs.xp, LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_PARAMETERS);- ggml_set_name(opt->lbfgs.g, LLM_TENSOR_OPTIMIZER_LBFGS_CURRENT_GRADIENTS);- ggml_set_name(opt->lbfgs.gp, LLM_TENSOR_OPTIMIZER_LBFGS_PREVIOUS_GRADIENTS);- ggml_set_name(opt->lbfgs.d, LLM_TENSOR_OPTIMIZER_LBFGS_SEARCH_DIRECTION);- if (opt->lbfgs.pf) {- ggml_set_name(opt->lbfgs.pf, LLM_TENSOR_OPTIMIZER_LBFGS_PAST_LOSS_VALUES);- }- ggml_set_name(opt->lbfgs.lmal, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_ALPHA);- ggml_set_name(opt->lbfgs.lmys, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_YS);- ggml_set_name(opt->lbfgs.lms, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_S);- ggml_set_name(opt->lbfgs.lmy, LLM_TENSOR_OPTIMIZER_LBFGS_MEMORY_Y);-- gguf_add_tensor(fctx, opt->lbfgs.x);- gguf_add_tensor(fctx, opt->lbfgs.xp);- gguf_add_tensor(fctx, opt->lbfgs.g);- gguf_add_tensor(fctx, opt->lbfgs.gp);- gguf_add_tensor(fctx, opt->lbfgs.d);- if (opt->lbfgs.pf) {- gguf_add_tensor(fctx, opt->lbfgs.pf);- }- gguf_add_tensor(fctx, opt->lbfgs.lmal);- gguf_add_tensor(fctx, opt->lbfgs.lmys);- gguf_add_tensor(fctx, opt->lbfgs.lms);- gguf_add_tensor(fctx, opt->lbfgs.lmy);- } break;- }-}--bool load_train_state_gguf(struct gguf_context * fctx, struct ggml_context * f_ggml_ctx, struct train_state * train) {- if (gguf_find_key(fctx, LLM_KV_TRAINING_FILE_VERSION) < 0) {- return false;- }-- uint32_t file_version;- GGUF_GET_KEY(fctx, file_version, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_FILE_VERSION);- GGML_ASSERT(file_version <= 1);-- if (file_version == 0) {-- GGUF_GET_KEY(fctx, train->train_its, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_ITERATION_COUNT);- GGUF_GET_KEY(fctx, train->train_samples, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_SAMPLE_COUNT);- GGUF_GET_KEY(fctx, train->train_tokens, gguf_get_val_u32, GGUF_TYPE_UINT32, true, LLM_KV_TRAINING_TOKEN_COUNT);-- } else if (file_version == 1) {-- GGUF_GET_KEY(fctx, train->train_its, gguf_get_val_u64, GGUF_TYPE_UINT64, true, LLM_KV_TRAINING_ITERATION_COUNT);- GGUF_GET_KEY(fctx, train->train_samples, gguf_get_val_u64, GGUF_TYPE_UINT64, true, LLM_KV_TRAINING_SAMPLE_COUNT);- GGUF_GET_KEY(fctx, train->train_tokens, gguf_get_val_u64, GGUF_TYPE_UINT64, true, LLM_KV_TRAINING_TOKEN_COUNT);- GGUF_GET_KEY(fctx, train->train_epochs, gguf_get_val_u64, GGUF_TYPE_UINT64, true, LLM_KV_TRAINING_EPOCH_COUNT);-- GGUF_GET_KEY(fctx, train->shuffle_samples_hash, gguf_get_val_u64, GGUF_TYPE_UINT64, false, LLM_KV_TRAINING_SHUFFLE_SAMPLES_HASH);- GGUF_GET_KEY(fctx, train->shuffle_rng_state_current, gguf_get_val_str, GGUF_TYPE_STRING, false, LLM_KV_TRAINING_SHUFFLE_RNG_STATE);- GGUF_GET_KEY(fctx, train->shuffle_sample_count, gguf_get_val_u64, GGUF_TYPE_UINT64, false, LLM_KV_TRAINING_SHUFFLE_SAMPLE_COUNT);- GGUF_GET_KEY(fctx, train->shuffle_next_sample, gguf_get_val_u64, GGUF_TYPE_UINT64, false, LLM_KV_TRAINING_SHUFFLE_NEXT_SAMPLE);- }-- load_opt_context_gguf(fctx, f_ggml_ctx, train->opt);- return true;-}--void save_train_state_gguf(struct gguf_context * fctx, struct train_state * train) {- gguf_set_val_u32(fctx, LLM_KV_TRAINING_FILE_VERSION, 1);- gguf_set_val_u64(fctx, LLM_KV_TRAINING_ITERATION_COUNT, train->train_its);- gguf_set_val_u64(fctx, LLM_KV_TRAINING_SAMPLE_COUNT, train->train_samples);- gguf_set_val_u64(fctx, LLM_KV_TRAINING_TOKEN_COUNT, train->train_tokens);- gguf_set_val_u64(fctx, LLM_KV_TRAINING_EPOCH_COUNT, train->train_epochs);-- gguf_set_val_u64(fctx, LLM_KV_TRAINING_SHUFFLE_SAMPLES_HASH, (uint64_t) train->shuffle_samples_hash);- gguf_set_val_str(fctx, LLM_KV_TRAINING_SHUFFLE_RNG_STATE, train->shuffle_rng_state_current.c_str());- gguf_set_val_u64(fctx, LLM_KV_TRAINING_SHUFFLE_SAMPLE_COUNT, (uint64_t) train->shuffle_sample_count);- gguf_set_val_u64(fctx, LLM_KV_TRAINING_SHUFFLE_NEXT_SAMPLE, (uint64_t) train->shuffle_next_sample);-- save_opt_context_gguf(fctx, train->opt);-}---struct llama_file {- // use FILE * so we don't have to re-open the file to mmap- FILE * fp;- size_t size;-- llama_file(const char * fname, const char * mode) {- fp = std::fopen(fname, mode);- if (fp == NULL) {- size = 0;- } else {- seek(0, SEEK_END);- size = tell();- seek(0, SEEK_SET);- }- }-- size_t tell() const {-#ifdef _WIN32- __int64 ret = _ftelli64(fp);-#else- long ret = std::ftell(fp);-#endif- GGML_ASSERT(ret != -1); // this really shouldn't fail- return (size_t) ret;- }-- void seek(size_t offset, int whence) {-#ifdef _WIN32- int ret = _fseeki64(fp, (__int64) offset, whence);-#else- int ret = std::fseek(fp, (long) offset, whence);-#endif- GGML_ASSERT(ret == 0); // same- }-- void read_raw(void * ptr, size_t size) {- if (size == 0) {- return;- }- errno = 0;- std::size_t ret = std::fread(ptr, size, 1, fp);- if (ferror(fp)) {- die_fmt("read error: %s", strerror(errno));- }- if (ret != 1) {- die("unexpectedly reached end of file");- }- }-- std::uint32_t read_u32() {- std::uint32_t ret;- read_raw(&ret, sizeof(ret));- return ret;- }-- std::string read_string(std::uint32_t len) {- std::vector<char> chars(len);- read_raw(chars.data(), len);- return std::string(chars.data(), len);- }-- void write_raw(const void * ptr, size_t size) {- if (size == 0) {- return;- }- errno = 0;- size_t ret = std::fwrite(ptr, size, 1, fp);- if (ret != 1) {- die_fmt("write error: %s", strerror(errno));- }- }-- void write_u32(std::uint32_t val) {- write_raw(&val, sizeof(val));- }-- ~llama_file() {- if (fp) {- std::fclose(fp);- }- }-};--static size_t utf8_len(char src) {- const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };- uint8_t highbits = static_cast<uint8_t>(src) >> 4;- return lookup[highbits];-}--// mark each byte with its utf8 unit number.-// returns the number of utf8 characters.-// e.g. when bytes == '\x61\xD0\xB0\x62',-// then utf8_units will become [0,0,1,0]-// utf8_nunits will become [1,2,2,1] and 3 is returned.-// bytes where utf8_units is zero, are the begin of an utf8 character.-static size_t mark_utf8_units(const char* bytes, int * utf8_units, int * utf8_nunits, size_t count) {- size_t offs = 0;- size_t count_utf8 = 0;- while(offs < count) {- int len = (int) utf8_len(bytes[offs]);- for (int i=0; i<len; ++i) {- utf8_units[offs+i] = i;- utf8_nunits[offs+i] = len;- }- offs += len;- ++count_utf8;- }- return count_utf8;-}--size_t tokenize_file(- struct llama_context * lctx,- const char * filename,- const std::string & sample_start,- bool include_sample_start,- bool overlapping_samples,- unsigned context_length,- std::vector<llama_token> & out_tokens,- std::vector<size_t> & out_samples_begin,- std::vector<size_t> & out_samples_size) {- struct llama_file f(filename, "rbe");-- if (f.size == 0) {- out_tokens.clear();- out_samples_begin.clear();- out_samples_size.clear();- printf("%s: warning: empty or not existing training data file '%s'\n",- __func__, filename);- return out_tokens.size();- }-- // account for possible leading whitespace that will be added by tokenizer- // e.g. '\t' will be tokenized by llama spm tokenizer to [29871, 12]- const int n_max_tokens_overhead = 1;-- std::vector<char> buf;- buf.resize(f.size);-- f.read_raw(buf.data(), f.size);-- std::vector<int> utf8_units;- std::vector<int> utf8_nunits;- utf8_units.resize(buf.size());- utf8_nunits.resize(buf.size());- mark_utf8_units(buf.data(), utf8_units.data(), utf8_nunits.data(), buf.size());-- if (sample_start.size() == 0) {- // tokenize all data at once- out_tokens.resize(buf.size() + n_max_tokens_overhead);-- int n_tokens = llama_tokenize(- llama_get_model(lctx),- buf.data(),- (int) buf.size(),- out_tokens.data(),- (int) out_tokens.size(),- false, false);- if (n_tokens < 0) {- out_tokens.resize(-n_tokens);- n_tokens = llama_tokenize(- llama_get_model(lctx),- buf.data(),- (int) buf.size(),- out_tokens.data(),- (int) out_tokens.size(),- false, false);- }- if (n_tokens >= 0) {- out_tokens.resize(n_tokens);- }-- // generate sample starts at all token positions- out_samples_begin.clear();- out_samples_begin.push_back(0);- out_samples_size.push_back(std::min((size_t) context_length, out_tokens.size()));- size_t end = (out_tokens.size() >= context_length) ? (out_tokens.size() - context_length) : 0;- for (size_t sample_begin = 1; sample_begin < end; ++sample_begin) {- out_samples_begin.push_back(sample_begin);- out_samples_size.push_back(context_length);- }- } else {- // split data into samples and tokenize each sample- std::string data_str(buf.data(), buf.size());- out_samples_begin.clear();- out_samples_size.clear();- out_tokens.clear();-- // find all positions of pattern sample_start- size_t sample_begin = data_str.find(sample_start, 0);- while (sample_begin != std::string::npos) {- out_samples_begin.push_back(sample_begin);- const size_t search_start = sample_begin + sample_start.size();- sample_begin = data_str.find(sample_start, search_start);- }- if (out_samples_begin.size() == 0) {- printf("%s: warning: sample start pattern '%s' not found. inserting single sample at data begin\n",- __func__, sample_start.c_str());- out_samples_begin.push_back(0);- }-- out_samples_size.resize(out_samples_begin.size(), 0);-- std::vector<char> buf_sample;- std::vector<llama_token> tok_sample;-- const size_t sample_begin_offset = (include_sample_start ? 0 : sample_start.size());- size_t found_too_big_sample = 0;- size_t found_too_small_sample = 0;- size_t found_empty_sample = 0;- size_t found_min_sample_size = SIZE_MAX;- size_t found_max_sample_size = 0;-- size_t max_token_text_size = 0;- int n_vocab = llama_n_vocab(llama_get_model(lctx));- for (llama_token token=0; token < n_vocab; ++token) {- max_token_text_size = std::max(- max_token_text_size,- strlen(llama_token_get_text(llama_get_model(lctx), token)));- }-- // upper bound of context byte length.- // strings with this byte length should always tokenize to at least context_length tokens.- size_t context_byte_len = max_token_text_size*context_length;-- for (unsigned i=0; i<out_samples_begin.size(); ++i) {- // determine sample begin and end from pattern positions- size_t sample_begin = out_samples_begin[i] + sample_begin_offset;- size_t sample_end = overlapping_samples- ? std::min(- data_str.size(),- sample_begin + context_byte_len)- : (i+1 < out_samples_begin.size()- ? out_samples_begin[i+1]- : data_str.size());- if (sample_end < utf8_units.size() && utf8_units[sample_end] > 0) {- // sample end is in the middle of an utf8 character.- // advance sample_end to the begin of the next utf8 character.- sample_end += utf8_nunits[sample_end] - utf8_units[sample_end];- }- size_t sample_size = sample_end - sample_begin;- if (sample_size == 0) {- ++found_empty_sample;- }-- if (sample_size > 0) {- // llama_tokenize expects zero terminated string,- // copy sample into buffer and zero terminate it.- buf_sample.resize(sample_size);- memcpy(buf_sample.data(), data_str.data() + sample_begin, sample_size);-- // printf("sample: '%s'\n", buf_sample.data());-- // tokenize the sample- tok_sample.resize(buf_sample.size() + n_max_tokens_overhead);- int n_tokens = llama_tokenize(llama_get_model(lctx),- buf_sample.data(),- (int) buf_sample.size(),- tok_sample.data(),- (int) tok_sample.size(),- false, false);- if (n_tokens < 0) {- tok_sample.resize(-n_tokens);- n_tokens = llama_tokenize(llama_get_model(lctx),- buf_sample.data(),- (int) buf_sample.size(),- tok_sample.data(),- (int) tok_sample.size(),- false, false);- GGML_ASSERT(n_tokens >= 0);- }- GGML_ASSERT(n_tokens <= (int) tok_sample.size());-- if ((size_t) n_tokens > context_length) {- ++found_too_big_sample;- } else if ((size_t) n_tokens < context_length) {- ++found_too_small_sample;- }- found_max_sample_size = std::max(found_max_sample_size, (size_t) n_tokens);- found_min_sample_size = std::min(found_min_sample_size, (size_t) n_tokens);-- // write out tokens, start and size of sample- // overwrite the string start position with the token start position- out_samples_begin[i] = out_tokens.size();- out_samples_size[i] = (size_t) n_tokens;- out_tokens.insert(out_tokens.end(), tok_sample.begin(), tok_sample.begin() + n_tokens);- } else {- out_samples_begin[i] = out_tokens.size();- out_samples_size[i] = 0;- }-- }- if (found_too_big_sample > 0) {- printf("%s: warning: found %zu samples (max length %zu) that exceed context length of %u. samples will be cut off.\n",- __func__, found_too_big_sample, found_max_sample_size, context_length);- }-- if (found_too_small_sample > 0) {- printf("%s: warning: found %zu samples (min length %zu) that are shorter than context length of %u.\n",- __func__, found_too_small_sample, found_min_sample_size, context_length);- }-- if (found_empty_sample) {- printf("%s: warning: found %zu empty samples.\n",- __func__, found_empty_sample);- }- }- printf("%s: total number of samples: %zu\n",- __func__, out_samples_begin.size());-- GGML_ASSERT(out_samples_begin.size() == out_samples_size.size());-- return out_tokens.size();-}--std::string get_train_filename(const char * filename, const char * pattern_it, const char * latest, int64_t iteration) {- std::string sit = (iteration >= 0) ? std::to_string(iteration) : std::string(latest);- return replace_str(filename, pattern_it, sit.c_str());-}--struct train_params_common get_default_train_params_common() {- struct train_params_common params;- params.fn_train_data = "shakespeare.txt";- params.fn_checkpoint_in = "checkpoint.gguf";- params.fn_checkpoint_out = "checkpoint-ITERATION.gguf";- params.pattern_fn_it = "ITERATION";- params.fn_latest = "LATEST";-- params.print_usage = false;-- params.save_every = 10;-- params.seed = -1;-- params.n_ctx = 128;- params.n_threads = 6;- params.n_batch = 8;- params.n_gradient_accumulation = 1;- params.n_epochs = -1;- params.n_gpu_layers = 0;-- params.custom_n_ctx = false;-- params.use_flash = true;- params.use_checkpointing = true;-- params.sample_start = "";- params.include_sample_start = false;- params.escape = false;- params.overlapping_samples = false;- params.fill_with_next_samples = false;- params.separate_with_eos = false;- params.separate_with_bos = true;- params.sample_random_offsets = false;- params.force_reshuffle = false;-- params.opt_past = 0;- params.opt_delta = 1e-5f;- params.opt_max_no_improvement = 0;-- params.warmup = 100;- params.cos_decay_steps = 1000;- params.cos_decay_restart = 1.1f;- params.cos_decay_min = 0.1f;- params.enable_restart = false;-- params.adam_n_iter = 256;- params.adam_alpha = 1e-3f;- params.adam_min_alpha = 0;- params.adam_decay = 1e-1f;- params.adam_decay_min_ndim = 2;- params.adam_beta1 = 0.9f;- params.adam_beta2 = 0.999f;- params.adam_gclip = 1.0f;- params.adam_eps_f = 0.0f;-- return params;-}--void print_common_train_usage(int /*argc*/, char ** /*argv*/, const struct train_params_common * params) {- // fprintf(stderr, "usage: %s [options]\n", argv[0]);- // fprintf(stderr, "\n");- // fprintf(stderr, "options:\n");- // fprintf(stderr, " -h, --help show this help message and exit\n");- fprintf(stderr, " --train-data FNAME path from which to load training data (default '%s')\n", params->fn_train_data);- fprintf(stderr, " --checkpoint-in FNAME path from which to load training checkpoint (default '%s')\n", params->fn_checkpoint_in);- fprintf(stderr, " --checkpoint-out FNAME path to save training checkpoint (default '%s')\n", params->fn_checkpoint_out);- fprintf(stderr, " --pattern-fn-it STR pattern in output filenames to be replaced by iteration number (default '%s')\n", params->pattern_fn_it);- fprintf(stderr, " --fn-latest STR string to use instead of iteration number for saving latest output (default '%s')\n", params->fn_latest);- fprintf(stderr, " --save-every N save checkpoint and lora every N iterations. Disabled when N <= 0. (default '%d')\n", params->save_every);- fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for -1)\n");- fprintf(stderr, " -c N, --ctx N Context size used during training (default %d)\n", params->n_ctx);- fprintf(stderr, " -t N, --threads N Number of threads (default %d)\n", params->n_threads);- fprintf(stderr, " -b N, --batch N Parallel batch size (default %d)\n", params->n_batch);- fprintf(stderr, " --grad-acc N Number of gradient accumulation steps (simulates larger batch size of batch*gradacc) (default %d)\n", params->n_gradient_accumulation);- fprintf(stderr, " --sample-start STR Sets the starting point for samples after the specified pattern. If empty use every token position as sample start. (default '%s')\n", params->sample_start.c_str());- fprintf(stderr, " --include-sample-start Include the sample start in the samples. (default off)\n");- fprintf(stderr, " --escape process sample start escapes sequences (\\n, \\r, \\t, \\', \\\", \\\\)\n");- fprintf(stderr, " --overlapping-samples Samples my overlap, will include sample-start of second and following samples. When off, samples will end at begin of next sample. (default off)\n");- fprintf(stderr, " --fill-with-next-samples Samples shorter than context length will be followed by the next (shuffled) samples. (default off)\n");- fprintf(stderr, " --separate-with-eos When fill-with-next-samples, insert end-of-sequence token between samples.%s\n", params->separate_with_eos ? " (default)" : "");- fprintf(stderr, " --separate-with-bos When fill-with-next-samples, insert begin-of-sequence token between samples.%s\n", params->separate_with_bos ? " (default)" : "");- fprintf(stderr, " --no-separate-with-eos When fill-with-next-samples, don't insert end-of-sequence token between samples.%s\n", !params->separate_with_eos ? " (default)" : "");- fprintf(stderr, " --no-separate-with-bos When fill-with-next-samples, don't insert begin-of-sequence token between samples.%s\n", !params->separate_with_bos ? " (default)" : "");- fprintf(stderr, " --sample-random-offsets Use samples beginning at random offsets. Together with fill-with-next-samples this may help for training endless text generation.%s\n", params->sample_random_offsets ? " (default)" : "");- fprintf(stderr, " --force-reshuffle Force a reshuffling of data at program start, otherwise the shuffling of loaded checkpoint is resumed.\n");- fprintf(stderr, " --no-flash Don't use flash attention \n");- fprintf(stderr, " --use-flash Use flash attention (default)\n");- fprintf(stderr, " --no-checkpointing Don't use gradient checkpointing\n");- fprintf(stderr, " --use-checkpointing Use gradient checkpointing (default)\n");- fprintf(stderr, " --warmup N Only for Adam optimizer. Number of warmup steps (default %d)\n", params->warmup);- fprintf(stderr, " --cos-decay-steps N Only for Adam optimizer. Number of cosine decay steps (default %d)\n", params->cos_decay_steps);- fprintf(stderr, " --cos-decay-restart N Only for Adam optimizer. Increase of cosine decay steps after restart (default %f)\n", params->cos_decay_restart);- fprintf(stderr, " --cos-decay-min N Only for Adam optimizer. Cosine decay minimum (default %f)\n", params->cos_decay_min);- fprintf(stderr, " --enable-restart N Only for Adam optimizer. Enable restarts of cos-decay %s\n", params->enable_restart ? "(default)" : "");- fprintf(stderr, " --disable-restart N Only for Adam optimizer. Disable restarts of cos-decay %s\n", !params->enable_restart ? "(default)" : "");- fprintf(stderr, " --opt-past N Number of optimization iterations to track for delta convergence test. Disabled when zero. (default %d)\n", params->opt_past);- fprintf(stderr, " --opt-delta N Maximum delta for delta convergence test. Disabled when <= zero. (default %f)\n", params->opt_delta);- fprintf(stderr, " --opt-max-no-improvement N Maximum number of optimization iterations with no improvement. Disabled when <= zero. (default %d)\n", params->opt_max_no_improvement);- fprintf(stderr, " --epochs N Maximum number epochs to process. (default %d)\n", params->n_epochs);- fprintf(stderr, " --adam-iter N Maximum number of Adam optimization iterations for each batch (default %d)\n", params->adam_n_iter);- fprintf(stderr, " --adam-alpha N Adam learning rate alpha (default %f)\n", params->adam_alpha);- fprintf(stderr, " --adam-min-alpha N Adam minimum learning rate alpha - including warmup phase (default %f)\n", params->adam_min_alpha);- fprintf(stderr, " --adam-decay N AdamW weight decay. Values greater zero enable AdamW instead of regular Adam. (default %f)\n", params->adam_decay);- fprintf(stderr, " --adam-decay-min-ndim N Minimum number of tensor dimensions to apply AdamW weight decay. Weight decay is not applied to tensors with less n_dims. (default %d)\n", params->adam_decay_min_ndim);- fprintf(stderr, " --adam-beta1 N AdamW beta1 in interval [0,1). How much to smooth the first moment of gradients. (default %f)\n", params->adam_beta1);- fprintf(stderr, " --adam-beta2 N AdamW beta2 in interval [0,1). How much to smooth the second moment of gradients. (default %f)\n", params->adam_beta2);- fprintf(stderr, " --adam-gclip N AdamW gradient clipping. Disabled when zero. (default %f)\n", params->adam_gclip);- fprintf(stderr, " --adam-epsf N AdamW epsilon for convergence test. Disabled when <= zero. (default %f)\n", params->adam_eps_f);- fprintf(stderr, "\n");-}--bool consume_common_train_arg(- int argc, char ** argv, int * idx, struct train_params_common * params, bool * invalid_param-) {- int& i = *idx;- std::string arg = argv[i];- const std::string arg_prefix = "--";- if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {- std::replace(arg.begin(), arg.end(), '_', '-');- }- if (arg == "--train-data") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->fn_train_data = argv[i];- } else if (arg == "--checkpoint-in") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->fn_checkpoint_in = argv[i];- } else if (arg == "--checkpoint-out") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->fn_checkpoint_out = argv[i];- } else if (arg == "--pattern-fn-it") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->pattern_fn_it = argv[i];- } else if (arg == "--fn-latest") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->fn_latest = argv[i];- } else if (arg == "--save-every") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->save_every = std::stoi(argv[i]);- } else if (arg == "-s" || arg == "--seed") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->seed = std::stoi(argv[i]);- } else if (arg == "-c" || arg == "--ctx") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->n_ctx = std::stoi(argv[i]);- params->custom_n_ctx = true;- } else if (arg == "-t" || arg == "--threads") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->n_threads = std::stoi(argv[i]);- } else if (arg == "-b" || arg == "--batch") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->n_batch = std::stoi(argv[i]);- } else if (arg == "--grad-acc") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->n_gradient_accumulation = std::max(1, std::stoi(argv[i]));- } else if (arg == "--sample-start") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->sample_start = std::string(argv[i]);- } else if (arg == "--escape") {- params->escape = true;- } else if (arg == "--include-sample-start") {- params->include_sample_start = true;- } else if (arg == "--overlapping-samples") {- params->overlapping_samples = true;- } else if (arg == "--fill-with-next-samples") {- params->fill_with_next_samples = true;- } else if (arg == "--separate-with-eos") {- params->separate_with_eos = true;- } else if (arg == "--separate-with-bos") {- params->separate_with_bos = true;- } else if (arg == "--no-separate-with-eos") {- params->separate_with_eos = false;- } else if (arg == "--no-separate-with-bos") {- params->separate_with_bos = false;- } else if (arg == "--sample-random-offsets") {- params->sample_random_offsets = true;- } else if (arg == "--force-reshuffle") {- params->force_reshuffle = true;- } else if (arg == "--no-flash") {- params->use_flash = false;- } else if (arg == "--use-flash") {- params->use_flash = true;- } else if (arg == "--no-checkpointing") {- params->use_checkpointing = false;- } else if (arg == "--use-checkpointing") {- params->use_checkpointing = true;- } else if (arg == "--warmup") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->warmup = std::stoi(argv[i]);- } else if (arg == "--cos-decay-steps") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->cos_decay_steps = std::stoi(argv[i]);- } else if (arg == "--cos-decay-restart") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->cos_decay_restart = std::stof(argv[i]);- } else if (arg == "--cos-decay-min") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->cos_decay_min = std::stof(argv[i]);- } else if (arg == "--enable-restart") {- params->enable_restart = true;- } else if (arg == "--disable-restart") {- params->enable_restart = false;- } else if (arg == "--opt-past") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->opt_past = std::stoi(argv[i]);- } else if (arg == "--opt-delta") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->opt_delta = std::stof(argv[i]);- } else if (arg == "--opt-max-no-improvement") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->opt_max_no_improvement = std::stoi(argv[i]);- } else if (arg == "--adam-epsf") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_eps_f = std::stof(argv[i]);- } else if (arg == "--epochs") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->n_epochs = std::stoi(argv[i]);- } else if (arg == "--adam-iter") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_n_iter = std::stoi(argv[i]);- } else if (arg == "--adam-alpha") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_alpha = std::stof(argv[i]);- } else if (arg == "--adam-min-alpha") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_min_alpha = std::stof(argv[i]);- } else if (arg == "--adam-decay") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_decay = std::stof(argv[i]);- } else if (arg == "--adam-decay-min-ndim") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_decay_min_ndim = std::stoi(argv[i]);- } else if (arg == "--adam-beta1") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_beta1 = std::stof(argv[i]);- } else if (arg == "--adam-beta2") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_beta2 = std::stof(argv[i]);- } else if (arg == "--adam-gclip") {- if (++i >= argc) {- *invalid_param = true;- return true;- }- params->adam_gclip = std::stof(argv[i]);- } else if (arg == "-h" || arg == "--help") {- params->print_usage = true;- return true;- } else {- return false;- }- return true;-}--void finish_processing_train_args(struct train_params_common * params) {- if (params->escape) {- process_escapes(params->sample_start);- }-}--void train_opt_callback(void * vdata, int accum_step, float * sched, bool * cancel) {- struct train_opt_callback_data * data = (struct train_opt_callback_data *) vdata;- struct train_params_common * params = data->params;- struct train_state * train = data->train;- struct ggml_opt_context * opt = train->opt;- int n_batch = params->n_batch;- int n_ctx = params->n_ctx;-- if (accum_step == 0) {- // time measurement- int64_t now = ggml_time_ms();- if (now > data->last_time && opt->iter > data->first_iter) {- double dt = (double) (now - data->last_time);- if (data->millis_per_iter == 0.0) {- data->millis_per_iter = dt;- } else {- const double gain = 0.7;- data->millis_per_iter = data->millis_per_iter*(1.0-gain) + dt*gain;- }- }-- double remaining_millis = 0.0;- if (data->millis_per_iter > 0.0) {- const int n_iter = params->adam_n_iter;- const int done_iter = opt->iter - data->first_iter;- const int remaining_iter = n_iter - done_iter;- remaining_millis = remaining_iter * data->millis_per_iter;- }-- // file saving- const bool save_now = (params->save_every > 0) && (opt->iter - data->last_save_iter >= params->save_every);- if (save_now) {- int new_iters = opt->iter - data->last_save_iter;- train->train_its += new_iters;- train->train_tokens += new_iters * opt->params.n_gradient_accumulation * n_batch * n_ctx;-- if (data->save_cb) {- data->save_cb(data->save_data, train);- }-- data->last_save_iter = opt->iter;- }-- // exclude file saving from time measurement, by measuring last_time after saving- data->last_time = ggml_time_ms();-- *sched = learning_schedule(- opt->iter,- params->warmup,- params->cos_decay_steps,- params->adam_alpha,- params->adam_min_alpha,- params->cos_decay_min,- params->cos_decay_restart,- params->enable_restart);-- int impr_plot = -(int)(1 + (opt->loss_before - opt->loss_after) * 10.0f + 0.5f);- if (impr_plot > 0) impr_plot = 0;- if (std::isnan(opt->loss_before) || std::isnan(opt->loss_after)) impr_plot = 0;- printf("%s: iter=%6d sample=%zu/%zu sched=%f loss=%f",- __func__, opt->iter, std::min(1+train->shuffle_next_sample, train->shuffle_sample_count), train->shuffle_sample_count,- *sched, opt->loss_after);--- if (data->millis_per_iter > 0) {- printf(" dt=");- print_duration(data->millis_per_iter);- printf(" eta=");- print_duration(remaining_millis);- }-- float improvement = opt->loss_before - opt->loss_after;- const float plot_scale = 10.0f;- int bar_len = (int)(1 + improvement*plot_scale + 0.5);- printf(" |");- for (int i=0; i<bar_len; ++i) {- printf("-");- }- printf(">");- printf("\n");- }-- int64_t used_samples = get_example_targets_batch(- data->lctx,- data->tokens_input,- data->target_probs,- train->shuffle_next_sample,- data->shuffled_samples_offs,- data->shuffled_samples_begin,- data->shuffled_samples_size,- data->samples_count,- data->tokens_data,- data->tokens_size,- params->separate_with_eos,- params->separate_with_bos,- params->fill_with_next_samples,- params->sample_random_offsets);-- train->train_samples += used_samples;- train->shuffle_next_sample += used_samples;-- if (train->shuffle_next_sample >= train->shuffle_sample_count) {- ++train->train_epochs;- printf("%s: reshuffle samples. completed epochs: %llu\n", __func__, (long long unsigned) train->train_epochs);- // note: we may have used some samples from the current shuffling more than once- train->shuffle_rng_state_current = train->shuffle_rng_state_next;- train->shuffle_rng_state_next = shuffle_samples(- train->shuffle_rng_state_current,- data->shuffled_samples_offs,- data->shuffled_samples_begin,- data->shuffled_samples_size,- data->samples_begin,- data->samples_size,- data->samples_count);- train->shuffle_next_sample = 0;- }-- const bool last_epoch_reached = (params->n_epochs > 0 && (int64_t) train->train_epochs - data->first_epoch >= params->n_epochs);- if (last_epoch_reached) {- // allow optimization iteration at last epoch to be completed before canceling- if (data->iter_at_last_epoch < 0) {- data->iter_at_last_epoch = opt->iter;- } else if (opt->iter > data->iter_at_last_epoch) {- *cancel = true;- }- }-}diff --git a/llamafile/cuda.bat b/llamafile/cuda.batindex 55d8c38c..9eb3c30e 100644--- a/llamafile/cuda.bat+++ b/llamafile/cuda.bat@@ -1,3 +1,9 @@+:: Compiles distributable DLL for NVIDIA GPU support+::+:: The artifact will only depend on KERNEL32.DLL and NVCUDA.DLL.+:: NVCUDA DLLs are provided by the installation of the windows GPU+:: driver on a Windows system that has a CUDA-capable GPU installed.+nvcc -arch=all ^--shared ^--forward-unknown-to-host-compiler ^@@ -12,4 +18,5 @@ nvcc -arch=all ^-DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 ^-DGGML_USE_TINYBLAS ^-o ggml-cuda.dll ^- ggml-cuda.cu+ ggml-cuda.cu ^+ -lcudadiff --git a/llamafile/cuda.c b/llamafile/cuda.cindex dffde900..f75ff4d7 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -33,6 +33,7 @@#include <stdatomic.h>#include "llamafile/log.h"#include "llama.cpp/ggml-cuda.h"+#include "llama.cpp/ggml-backend-impl.h"__static_yoink("llama.cpp/ggml.h");__static_yoink("llamafile/compcap.cu");@@ -51,7 +52,7 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");#define WIND_ONLY(x) (!IsWindows() ? "-DIGNORE" STRINGIFY(__COUNTER__) : x)#define BLAS_ONLY(x) (FLAG_tinyblas ? "-DIGNORE" STRINGIFY(__COUNTER__) : x)-#define NVCC_LIBS BLAS_ONLY("-lcublas")+#define NVCC_LIBS BLAS_ONLY("-lcublas"), "-lcuda"#define NVCC_FLAGS "--shared", \"--forward-unknown-to-host-compiler", \@@ -71,6 +72,8 @@ __static_yoink("llama.cpp/ggml-backend-impl.h");? "-DGGML_USE_TINYBLAS" \: "-DGGML_USE_CUBLAS")+int ggml_backend_cuda_reg_devices(void);+static const struct Source {const char *zip;const char *name;@@ -112,8 +115,9 @@ static struct Cuda {typeof(ggml_cuda_compute_forward) *compute_forward;typeof(ggml_cuda_get_device_count) *get_device_count;typeof(ggml_cuda_get_device_description) *get_device_description;- typeof(ggml_backend_reg_cuda_init) *reg_cuda_init;typeof(ggml_backend_cuda_buffer_type) *buffer_type;+ typeof(ggml_backend_reg_cuda_init) *backend_reg_init;+ typeof(ggml_backend_cuda_host_buffer_type) *backend_host_buffer_type;} ggml_cuda;static const char *Dlerror(void) {@@ -616,8 +620,8 @@ static bool LinkCudaDso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.compute_forward = cosmo_dlsym(lib, "ggml_cuda_compute_forward"));ok &= !!(ggml_cuda.get_device_count = cosmo_dlsym(lib, "ggml_cuda_get_device_count"));ok &= !!(ggml_cuda.get_device_description = cosmo_dlsym(lib, "ggml_cuda_get_device_description"));- ok &= !!(ggml_cuda.reg_cuda_init = cosmo_dlsym(lib, "ggml_backend_reg_cuda_init"));- ok &= !!(ggml_cuda.buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_buffer_type"));+ ok &= !!(ggml_cuda.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_cuda_init"));+ ok &= !!(ggml_cuda.backend_host_buffer_type = cosmo_dlsym(lib, "ggml_backend_cuda_host_buffer_type"));if (!ok) {tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);return false;@@ -939,13 +943,28 @@ void ggml_cuda_get_device_description(int device,description_size);}-ggml_backend_t ggml_backend_reg_cuda_init(const char * params,- void * user_data) {+ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {if (!ggml_cuda_supported()) return 0;- return ggml_cuda.reg_cuda_init(params, user_data);+ return ggml_cuda.buffer_type(device);}-ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) {+ggml_backend_t ggml_backend_reg_cuda_init(const char * params, void * user_data) {if (!ggml_cuda_supported()) return 0;- return ggml_cuda.buffer_type(device);+ ggml_cuda.backend_reg_init(params, user_data);+}++ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type() {+ if (!ggml_cuda_supported()) return 0;+ ggml_cuda.backend_host_buffer_type();+}++int ggml_backend_cuda_reg_devices(void) {+ int device_count = ggml_cuda_get_device_count();+ //int device_count = 1; // DEBUG: some tools require delaying CUDA initialization+ for (int i = 0; i < device_count; i++) {+ char name[128];+ snprintf(name, sizeof(name), "%s%d", GGML_CUDA_NAME, i);+ ggml_backend_register(name, ggml_backend_reg_cuda_init, ggml_backend_cuda_buffer_type(i), (void *) (intptr_t) i);+ }+ return device_count;}diff --git a/llamafile/metal.c b/llamafile/metal.cindex 022cfc92..a3761e13 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -42,6 +42,8 @@ __static_yoink("llama.cpp/ggml-backend.h");__static_yoink("llama.cpp/ggml-metal.metal");__static_yoink("llama.cpp/ggml-backend-impl.h");+ggml_backend_t ggml_backend_reg_metal_init(const char *, void *);+static const struct Source {const char *zip;const char *name;@@ -72,6 +74,9 @@ static struct Metal {typeof(ggml_metal_init) *init;typeof(ggml_metal_set_n_cb) *set_n_cb;typeof(ggml_backend_metal_init) *backend_init;+ typeof(ggml_backend_metal_buffer_type) *backend_buffer_type;+ typeof(ggml_backend_reg_metal_init) *backend_reg_init;+ typeof(ggml_backend_metal_buffer_from_ptr) *backend_buffer_from_ptr;} ggml_metal;static const char *Dlerror(void) {@@ -209,6 +214,9 @@ static bool LinkMetal(const char *dso) {ok &= !!(ggml_metal.init = cosmo_dlsym(lib, "ggml_metal_init"));ok &= !!(ggml_metal.set_n_cb = cosmo_dlsym(lib, "ggml_metal_set_n_cb"));ok &= !!(ggml_metal.backend_init = cosmo_dlsym(lib, "ggml_backend_metal_init"));+ ok &= !!(ggml_metal.backend_buffer_type = cosmo_dlsym(lib, "ggml_backend_metal_buffer_type"));+ ok &= !!(ggml_metal.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_metal_init"));+ ok &= !!(ggml_metal.backend_buffer_from_ptr = cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));if (!ok) {tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);return false;@@ -317,3 +325,18 @@ void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {if (!ggml_metal_supported()) return;return ggml_metal.set_n_cb(ctx, n_cb);}++ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {+ if (!ggml_metal_supported()) return 0;+ return ggml_metal.backend_buffer_type();+}++ggml_backend_t ggml_backend_reg_metal_init(const char * params, void * user_data) {+ if (!ggml_metal_supported()) return 0;+ return ggml_metal.backend_reg_init(params, user_data);+}++ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size) {+ if (!ggml_metal_supported()) return 0;+ return ggml_metal.backend_buffer_from_ptr(data, size, max_size);+}diff --git a/llamafile/tinyblas.cu b/llamafile/tinyblas.cuindex a989bb58..1f951ae3 100644--- a/llamafile/tinyblas.cu+++ b/llamafile/tinyblas.cu@@ -494,3 +494,28 @@ tinyblasStatus_t tinyblasGemmStridedBatchedEx(tinyblasHandle_t stream,return TINYBLAS_STATUS_SUCCESS;}++const char *tinyblasGetStatusString(tinyblasStatus_t err) {+ switch (err) {+ case TINYBLAS_STATUS_SUCCESS:+ return "TINYBLAS_STATUS_SUCCESS";+ case TINYBLAS_STATUS_NOT_INITIALIZED:+ return "TINYBLAS_STATUS_NOT_INITIALIZED";+ case TINYBLAS_STATUS_ALLOC_FAILED:+ return "TINYBLAS_STATUS_ALLOC_FAILED";+ case TINYBLAS_STATUS_INVALID_VALUE:+ return "TINYBLAS_STATUS_INVALID_VALUE";+ case TINYBLAS_STATUS_ARCH_MISMATCH:+ return "TINYBLAS_STATUS_ARCH_MISMATCH";+ case TINYBLAS_STATUS_MAPPING_ERROR:+ return "TINYBLAS_STATUS_MAPPING_ERROR";+ case TINYBLAS_STATUS_EXECUTION_FAILED:+ return "TINYBLAS_STATUS_EXECUTION_FAILED";+ case TINYBLAS_STATUS_INTERNAL_ERROR:+ return "TINYBLAS_STATUS_INTERNAL_ERROR";+ case TINYBLAS_STATUS_NOT_SUPPORTED:+ return "TINYBLAS_STATUS_NOT_SUPPORTED";+ default:+ return "unknown error";+ }+}diff --git a/llamafile/tinyblas.h b/llamafile/tinyblas.hindex 56a42942..78d8645a 100644--- a/llamafile/tinyblas.h+++ b/llamafile/tinyblas.h@@ -15,8 +15,15 @@ enum tinyblasOperation_t {};enum tinyblasStatus_t {- TINYBLAS_STATUS_SUCCESS = 0,- TINYBLAS_STATUS_NOT_SUPPORTED = 7,+ TINYBLAS_STATUS_SUCCESS,+ TINYBLAS_STATUS_NOT_INITIALIZED,+ TINYBLAS_STATUS_ALLOC_FAILED,+ TINYBLAS_STATUS_INVALID_VALUE,+ TINYBLAS_STATUS_ARCH_MISMATCH,+ TINYBLAS_STATUS_MAPPING_ERROR,+ TINYBLAS_STATUS_EXECUTION_FAILED,+ TINYBLAS_STATUS_INTERNAL_ERROR,+ TINYBLAS_STATUS_NOT_SUPPORTED,};enum tinyblasComputeType_t {@@ -30,6 +37,8 @@ enum tinyblasGemmAlgo_t {#define tinyblasHandle_t cudaStream_t+const char *tinyblasGetStatusString(tinyblasStatus_t);+tinyblasStatus_t tinyblasSgemm(tinyblasHandle_t handle,tinyblasOperation_t transa,tinyblasOperation_t transb,