Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Restore Apple Metal GPU support
Justine Tunney912e616b5307
diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.cindex 55ee1cc4..414e7844 100644--- a/llama.cpp/ggml-backend.c+++ b/llama.cpp/ggml-backend.c@@ -74,7 +74,8 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {if (buffer->iface.free_buffer != NULL) {buffer->iface.free_buffer(buffer);}- free(buffer);+ // TODO(llama.cpp): delete this file+ // free(buffer);}size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.mindex 537b2512..6c43ae7d 100644--- a/llama.cpp/ggml-metal.m+++ b/llama.cpp/ggml-metal.m@@ -260,6 +260,11 @@ static const char * ggml_op_name(enum ggml_op op) {return "REDACTED!GGML_OP_NAME[op]";}+#define ggml_op_desc ggml_op_desc_+static const char * ggml_op_desc(const struct ggml_tensor * t) {+ return "REDACTED!GGML_OP_DESC[t]";+}+// END: COPIED FROM GGML.C////////////////////////////////////////////////////////////////////////////////@@ -421,7 +426,7 @@ struct ggml_metal_context {ggml_log_callback ggml_metal_log_callback = NULL;void * ggml_metal_log_user_data = NULL;-static void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {+void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {ggml_metal_log_callback = log_callback;ggml_metal_log_user_data = user_data;}diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 8fbf10f1..7b6ec9e4 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -172,13 +172,12 @@ inline static void * ggml_aligned_malloc(size_t size) {return NULL;}void * aligned_memory = NULL;-#ifdef GGML_USE_CPU_HBM- int result = hbw_posix_memalign(&aligned_memory, 16, size);-#elif GGML_USE_METAL- int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);-#else- int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);-#endif+ int result;+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {+ result = posix_memalign(&aligned_memory, 16384, size);+ } else {+ result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);+ }if (result != 0) {// Handle allocation failureconst char *error_desc = "unknown allocation error";diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 0ea6af28..b8e48b80 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -6580,11 +6580,11 @@ static int llama_decode_internal(ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);#endif-#ifdef GGML_USE_METAL+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {if (ggml_backend_is_metal(lctx.backend)) {ggml_backend_metal_set_n_cb(lctx.backend, n_threads);}-#endif+ }if (ggml_backend_is_cpu(lctx.backend)) {ggml_backend_cpu_set_n_threads(lctx.backend, n_threads);@@ -9379,9 +9379,9 @@ struct llama_model_params llama_model_default_params() {/*.use_mlock =*/ false,};-#ifdef GGML_USE_METAL+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {result.n_gpu_layers = 1;-#endif+ }return result;}@@ -9573,14 +9573,15 @@ struct llama_context * llama_new_context_with_model(// reserve memory for context buffersif (!hparams.vocab_only) {// initialize backend-#ifdef GGML_USE_METAL+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {if (model->n_gpu_layers > 0) {ctx->backend = ggml_backend_metal_init();if (ctx->backend == nullptr) {LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__);}}-#elif defined(LLAMA_GGML_BACKEND_CUDA_TEST)+ }+#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {// for testing onlyif (model->n_gpu_layers > 0) {diff --git a/llamafile/metal.c b/llamafile/metal.cindex a3761e13..7beba4a8 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -77,6 +77,8 @@ static struct Metal {typeof(ggml_backend_metal_buffer_type) *backend_buffer_type;typeof(ggml_backend_reg_metal_init) *backend_reg_init;typeof(ggml_backend_metal_buffer_from_ptr) *backend_buffer_from_ptr;+ typeof(ggml_backend_is_metal) *backend_is_metal;+ typeof(ggml_backend_metal_set_n_cb) *backend_set_n_cb;} ggml_metal;static const char *Dlerror(void) {@@ -217,6 +219,8 @@ static bool LinkMetal(const char *dso) {ok &= !!(ggml_metal.backend_buffer_type = cosmo_dlsym(lib, "ggml_backend_metal_buffer_type"));ok &= !!(ggml_metal.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_metal_init"));ok &= !!(ggml_metal.backend_buffer_from_ptr = cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));+ ok &= !!(ggml_metal.backend_is_metal = cosmo_dlsym(lib, "ggml_backend_is_metal"));+ ok &= !!(ggml_metal.backend_set_n_cb = cosmo_dlsym(lib, "ggml_backend_metal_set_n_cb"));if (!ok) {tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);return false;@@ -326,6 +330,11 @@ void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {return ggml_metal.set_n_cb(ctx, n_cb);}+ggml_backend_t ggml_backend_metal_init(void) {+ if (!ggml_metal_supported()) return 0;+ return ggml_metal.backend_init();+}+ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {if (!ggml_metal_supported()) return 0;return ggml_metal.backend_buffer_type();@@ -340,3 +349,13 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t sizif (!ggml_metal_supported()) return 0;return ggml_metal.backend_buffer_from_ptr(data, size, max_size);}++bool ggml_backend_is_metal(ggml_backend_t backend) {+ if (!ggml_metal_supported()) return 0;+ return ggml_metal.backend_is_metal(backend);+}++void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {+ if (!ggml_metal_supported()) return;+ ggml_metal.backend_set_n_cb(backend, n_cb);+}