Skip to content

Distribute and run LLMs with a single file.

11.7 KB
Raw Blame
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
--- a/llama.cpp/ggml/src/ggml-backend.cpp
+++ b/llama.cpp/ggml/src/ggml-backend.cpp
@@ -108,34 +108,22 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
 
         struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
 
-        // allocate tensors in the current buffer
+        // allocate the parent tensors in the current buffer; views are initialized
+        // in a single pass below, once every buffer exists (llamafile: carries
+        // upstream PR #25584 — a range made only of views is never planned when it
+        // follows a tensor larger than max_size, so its views kept data == NULL)
         struct ggml_tensor * t_failed = NULL;
         for (int j = item.first; j < item.last; j++) {
             struct ggml_tensor * t = tensors[j];
-            if (t->data == NULL) {
-                if (t->view_src == NULL) {
-                    if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
-                        t_failed = t;
-                        break;
-                    }
-                } else if (t->buffer == NULL) {
-                    if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
-                        t_failed = t;
-                        break;
-                    }
-                }
-            } else {
-                if (t->view_src != NULL && t->buffer == NULL) {
-                    // view of a pre-allocated tensor
-                    if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
-                        t_failed = t;
-                        break;
-                    }
+            if (t->view_src == NULL && t->data == NULL) {
+                if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
+                    t_failed = t;
+                    break;
                 }
             }
         }
         if (t_failed != NULL) {
-            GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
+            GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t_failed->name);
             for (ggml_backend_buffer_t b : buffers) {
                 ggml_backend_buffer_free(b);
             }
@@ -150,6 +138,19 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
         return NULL;
     }
 
+    for (int j = 0; j < n_tensors; j++) {
+        struct ggml_tensor * t = tensors[j];
+        if (t->view_src != NULL && t->buffer == NULL) {
+            if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
+                GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);
+                for (ggml_backend_buffer_t b : buffers) {
+                    ggml_backend_buffer_free(b);
+                }
+                return NULL;
+            }
+        }
+    }
+
     if (buffers.size() == 1) {
         return buffers[0];
     }
@@ -261,7 +262,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
     if (buffer->iface.free_buffer != NULL) {
         buffer->iface.free_buffer(buffer);
     }
-    delete buffer;
+
+    // Use free_struct if provided (for cross-module memory management,
+    // e.g., when the buffer was allocated by a dynamically loaded library)
+    if (buffer->iface.free_struct != NULL) {
+        buffer->iface.free_struct(buffer);
+    } else {
+        delete buffer;
+    }
 }
 
 size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
@@ -822,7 +830,7 @@ struct ggml_backend_multi_buffer_context {
     size_t n_buffers;
 };
 
-static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
     GGML_ASSERT(buffer);
     ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
     for (size_t i = 0; i < ctx->n_buffers; i++) {
@@ -833,7 +841,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
     free(ctx);
 }
 
-static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
     GGML_ASSERT(buffer);
     ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
     for (size_t i = 0; i < ctx->n_buffers; i++) {
@@ -853,6 +861,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
     /* .cpy_tensor      = */ NULL,
     /* .clear           = */ ggml_backend_multi_buffer_clear,
     /* .reset           = */ NULL,
+    /* .free_struct     = */ NULL,
 };
 
 ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
@@ -1211,6 +1220,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
 }
 
 // assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
+// [jart] The backends invoke this through ggml_backend_graph_optimize_params,
+// and they live in dynamically loaded modules built with the host ABI, so the
+// callback has to be GGML_CALL. A lambda cannot carry the attribute, so this is
+// a named function rather than the lambda upstream uses.
+static void GGML_CALL ggml_backend_sched_add_alloc_dep(void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
+    auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;
+    std::vector<ggml_tensor *> & keep = deps[until];
+    if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
+        keep.push_back(tensor);
+    }
+}
+
 void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
     // reset splits
     sched->n_splits = 0;
@@ -1609,13 +1630,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
     std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;
 
     struct ggml_backend_graph_optimize_params opt_params = {
-        /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
-            auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;
-            std::vector<ggml_tensor *> & keep = deps[until];
-            if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
-                keep.push_back(tensor);
-            }
-        },
+        /* .add_alloc_dep = */ ggml_backend_sched_add_alloc_dep,
         /* .user_data     = */ &alloc_deps,
     };
 
@@ -2466,7 +2481,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
 
 // CPU backend - buffer
 
-static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
+static void * GGML_CALL ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
     GGML_ASSERT(buffer);
     uintptr_t data = (uintptr_t)buffer->context;
 
@@ -2478,33 +2493,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
     return (void *)data;
 }
 
-static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
     GGML_ASSERT(buffer);
     ggml_aligned_free(buffer->context, buffer->size);
 }
 
-static void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
     GGML_ASSERT(tensor);
     memset((char *)tensor->data + offset, value, size);
 
     GGML_UNUSED(buffer);
 }
 
-static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
     GGML_ASSERT(tensor);
     memcpy((char *)tensor->data + offset, data, size);
 
     GGML_UNUSED(buffer);
 }
 
-static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
     GGML_ASSERT(tensor);
     memcpy(data, (const char *)tensor->data + offset, size);
 
     GGML_UNUSED(buffer);
 }
 
-static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
     GGML_ASSERT(src);
     if (ggml_backend_buffer_is_host(src->buffer)) {
         memcpy(dst->data, src->data, ggml_nbytes(src));
@@ -2515,7 +2530,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
     GGML_UNUSED(buffer);
 }
 
-static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
     GGML_ASSERT(buffer);
     memset(buffer->context, value, buffer->size);
 }
@@ -2532,6 +2547,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
     /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cpu_buffer_clear,
     /* .reset           = */ NULL,
+    /* .free_struct     = */ NULL,
 };
 
 static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
@@ -2546,19 +2562,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
     /* .cpy_tensor      = */ ggml_backend_cpu_buffer_cpy_tensor,
     /* .clear           = */ ggml_backend_cpu_buffer_clear,
     /* .reset           = */ NULL,
+    /* .free_struct     = */ NULL,
 };
 
 // CPU backend buffer type
 
 // this buffer type is defined here to make it available to all backends
 
-static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
     return "CPU";
 
     GGML_UNUSED(buft);
 }
 
-static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
     void * data = ggml_aligned_malloc(size);
 
     if (data == NULL) {
@@ -2569,13 +2586,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
     return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
 }
 
-static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
     return TENSOR_ALIGNMENT;
 
     GGML_UNUSED(buft);
 }
 
-static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
     return true;
 
     GGML_UNUSED(buft);
@@ -2600,7 +2617,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
     return &ggml_backend_cpu_buffer_type;
 }
 
-static const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {
     return "CPU_Mapped";
 
     GGML_UNUSED(buft);