Skip to content

Distribute and run LLMs with a single file.

Restore Apple Metal GPU support

Justine Tunney912e616b5307
diff --git a/llama.cpp/ggml-backend.c b/llama.cpp/ggml-backend.c
index 55ee1cc4..414e7844 100644
--- a/llama.cpp/ggml-backend.c
+++ b/llama.cpp/ggml-backend.c
@@ -74,7 +74,8 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
if (buffer->iface.free_buffer != NULL) {
buffer->iface.free_buffer(buffer);
}
- free(buffer);
+ // TODO(llama.cpp): delete this file
+ // free(buffer);
}
size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
diff --git a/llama.cpp/ggml-metal.m b/llama.cpp/ggml-metal.m
index 537b2512..6c43ae7d 100644
--- a/llama.cpp/ggml-metal.m
+++ b/llama.cpp/ggml-metal.m
@@ -260,6 +260,11 @@ static const char * ggml_op_name(enum ggml_op op) {
return "REDACTED!GGML_OP_NAME[op]";
}
+#define ggml_op_desc ggml_op_desc_
+static const char * ggml_op_desc(const struct ggml_tensor * t) {
+ return "REDACTED!GGML_OP_DESC[t]";
+}
+
// END: COPIED FROM GGML.C
////////////////////////////////////////////////////////////////////////////////
@@ -421,7 +426,7 @@ struct ggml_metal_context {
ggml_log_callback ggml_metal_log_callback = NULL;
void * ggml_metal_log_user_data = NULL;
-static void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
+void ggml_metal_log_set_callback(ggml_log_callback log_callback, void * user_data) {
ggml_metal_log_callback = log_callback;
ggml_metal_log_user_data = user_data;
}
diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.c
index 8fbf10f1..7b6ec9e4 100644
--- a/llama.cpp/ggml.c
+++ b/llama.cpp/ggml.c
@@ -172,13 +172,12 @@ inline static void * ggml_aligned_malloc(size_t size) {
return NULL;
}
void * aligned_memory = NULL;
-#ifdef GGML_USE_CPU_HBM
- int result = hbw_posix_memalign(&aligned_memory, 16, size);
-#elif GGML_USE_METAL
- int result = posix_memalign(&aligned_memory, sysconf(_SC_PAGESIZE), size);
-#else
- int result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
-#endif
+ int result;
+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {
+ result = posix_memalign(&aligned_memory, 16384, size);
+ } else {
+ result = posix_memalign(&aligned_memory, GGML_MEM_ALIGN, size);
+ }
if (result != 0) {
// Handle allocation failure
const char *error_desc = "unknown allocation error";
diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cpp
index 0ea6af28..b8e48b80 100644
--- a/llama.cpp/llama.cpp
+++ b/llama.cpp/llama.cpp
@@ -6580,11 +6580,11 @@ static int llama_decode_internal(
ggml_mpi_graph_compute_pre(lctx.ctx_mpi, gf, n_layer);
#endif
-#ifdef GGML_USE_METAL
+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {
if (ggml_backend_is_metal(lctx.backend)) {
ggml_backend_metal_set_n_cb(lctx.backend, n_threads);
}
-#endif
+ }
if (ggml_backend_is_cpu(lctx.backend)) {
ggml_backend_cpu_set_n_threads(lctx.backend, n_threads);
@@ -9379,9 +9379,9 @@ struct llama_model_params llama_model_default_params() {
/*.use_mlock =*/ false,
};
-#ifdef GGML_USE_METAL
+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {
result.n_gpu_layers = 1;
-#endif
+ }
return result;
}
@@ -9573,14 +9573,15 @@ struct llama_context * llama_new_context_with_model(
// reserve memory for context buffers
if (!hparams.vocab_only) {
// initialize backend
-#ifdef GGML_USE_METAL
+ if (llamafile_gpu_supported() == LLAMAFILE_GPU_APPLE) {
if (model->n_gpu_layers > 0) {
ctx->backend = ggml_backend_metal_init();
if (ctx->backend == nullptr) {
LLAMA_LOG_ERROR("%s: failed to initialize Metal backend\n", __func__);
}
}
-#elif defined(LLAMA_GGML_BACKEND_CUDA_TEST)
+ }
+#if defined(LLAMA_GGML_BACKEND_CUDA_TEST)
if (llamafile_gpu_supported() == LLAMAFILE_GPU_NVIDIA) {
// for testing only
if (model->n_gpu_layers > 0) {
diff --git a/llamafile/metal.c b/llamafile/metal.c
index a3761e13..7beba4a8 100644
--- a/llamafile/metal.c
+++ b/llamafile/metal.c
@@ -77,6 +77,8 @@ static struct Metal {
typeof(ggml_backend_metal_buffer_type) *backend_buffer_type;
typeof(ggml_backend_reg_metal_init) *backend_reg_init;
typeof(ggml_backend_metal_buffer_from_ptr) *backend_buffer_from_ptr;
+ typeof(ggml_backend_is_metal) *backend_is_metal;
+ typeof(ggml_backend_metal_set_n_cb) *backend_set_n_cb;
} ggml_metal;
static const char *Dlerror(void) {
@@ -217,6 +219,8 @@ static bool LinkMetal(const char *dso) {
ok &= !!(ggml_metal.backend_buffer_type = cosmo_dlsym(lib, "ggml_backend_metal_buffer_type"));
ok &= !!(ggml_metal.backend_reg_init = cosmo_dlsym(lib, "ggml_backend_reg_metal_init"));
ok &= !!(ggml_metal.backend_buffer_from_ptr = cosmo_dlsym(lib, "ggml_backend_metal_buffer_from_ptr"));
+ ok &= !!(ggml_metal.backend_is_metal = cosmo_dlsym(lib, "ggml_backend_is_metal"));
+ ok &= !!(ggml_metal.backend_set_n_cb = cosmo_dlsym(lib, "ggml_backend_metal_set_n_cb"));
if (!ok) {
tinylog(Dlerror(), ": not all symbols could be imported\n", NULL);
return false;
@@ -326,6 +330,11 @@ void ggml_metal_set_n_cb(struct ggml_metal_context * ctx, int n_cb) {
return ggml_metal.set_n_cb(ctx, n_cb);
}
+ggml_backend_t ggml_backend_metal_init(void) {
+ if (!ggml_metal_supported()) return 0;
+ return ggml_metal.backend_init();
+}
+
ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void) {
if (!ggml_metal_supported()) return 0;
return ggml_metal.backend_buffer_type();
@@ -340,3 +349,13 @@ ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t siz
if (!ggml_metal_supported()) return 0;
return ggml_metal.backend_buffer_from_ptr(data, size, max_size);
}
+
+bool ggml_backend_is_metal(ggml_backend_t backend) {
+ if (!ggml_metal_supported()) return 0;
+ return ggml_metal.backend_is_metal(backend);
+}
+
+void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb) {
+ if (!ggml_metal_supported()) return;
+ ggml_metal.backend_set_n_cb(backend, n_cb);
+}