Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
--- a/llama.cpp/ggml/src/ggml-backend.cpp
+++ b/llama.cpp/ggml/src/ggml-backend.cpp
@@ -108,34 +108,22 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
- // allocate tensors in the current buffer
+ // allocate the parent tensors in the current buffer; views are initialized
+ // in a single pass below, once every buffer exists (llamafile: carries
+ // upstream PR #25584 — a range made only of views is never planned when it
+ // follows a tensor larger than max_size, so its views kept data == NULL)
struct ggml_tensor * t_failed = NULL;
for (int j = item.first; j < item.last; j++) {
struct ggml_tensor * t = tensors[j];
- if (t->data == NULL) {
- if (t->view_src == NULL) {
- if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
- t_failed = t;
- break;
- }
- } else if (t->buffer == NULL) {
- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
- t_failed = t;
- break;
- }
- }
- } else {
- if (t->view_src != NULL && t->buffer == NULL) {
- // view of a pre-allocated tensor
- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
- t_failed = t;
- break;
- }
+ if (t->view_src == NULL && t->data == NULL) {
+ if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
+ t_failed = t;
+ break;
}
}
}
if (t_failed != NULL) {
- GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
+ GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t_failed->name);
for (ggml_backend_buffer_t b : buffers) {
ggml_backend_buffer_free(b);
}
@@ -150,6 +138,19 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
return NULL;
}
+ for (int j = 0; j < n_tensors; j++) {
+ struct ggml_tensor * t = tensors[j];
+ if (t->view_src != NULL && t->buffer == NULL) {
+ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
+ GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);
+ for (ggml_backend_buffer_t b : buffers) {
+ ggml_backend_buffer_free(b);
+ }
+ return NULL;
+ }
+ }
+ }
+
if (buffers.size() == 1) {
return buffers[0];
}
@@ -261,7 +262,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
if (buffer->iface.free_buffer != NULL) {
buffer->iface.free_buffer(buffer);
}
- delete buffer;
+
+ // Use free_struct if provided (for cross-module memory management,
+ // e.g., when the buffer was allocated by a dynamically loaded library)
+ if (buffer->iface.free_struct != NULL) {
+ buffer->iface.free_struct(buffer);
+ } else {
+ delete buffer;
+ }
}
size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
@@ -822,7 +830,7 @@ struct ggml_backend_multi_buffer_context {
size_t n_buffers;
};
-static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer) {
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
@@ -833,7 +841,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
free(ctx);
}
-static void ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_multi_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
@@ -853,6 +861,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_multi_buffer_clear,
/* .reset = */ NULL,
+ /* .free_struct = */ NULL,
};
ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
@@ -1211,6 +1220,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
}
// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
+// [jart] The backends invoke this through ggml_backend_graph_optimize_params,
+// and they live in dynamically loaded modules built with the host ABI, so the
+// callback has to be GGML_CALL. A lambda cannot carry the attribute, so this is
+// a named function rather than the lambda upstream uses.
+static void GGML_CALL ggml_backend_sched_add_alloc_dep(void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
+ auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;
+ std::vector<ggml_tensor *> & keep = deps[until];
+ if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
+ keep.push_back(tensor);
+ }
+}
+
void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
// reset splits
sched->n_splits = 0;
@@ -1609,13 +1630,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;
struct ggml_backend_graph_optimize_params opt_params = {
- /* .add_alloc_dep = */ [](void * user_data, ggml_tensor * tensor, ggml_tensor * until) {
- auto & deps = *(std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> *) user_data;
- std::vector<ggml_tensor *> & keep = deps[until];
- if (std::find(keep.begin(), keep.end(), tensor) == keep.end()) {
- keep.push_back(tensor);
- }
- },
+ /* .add_alloc_dep = */ ggml_backend_sched_add_alloc_dep,
/* .user_data = */ &alloc_deps,
};
@@ -2466,7 +2481,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
// CPU backend - buffer
-static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
+static void * GGML_CALL ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
GGML_ASSERT(buffer);
uintptr_t data = (uintptr_t)buffer->context;
@@ -2478,33 +2493,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
return (void *)data;
}
-static void ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_cpu_buffer_free_buffer(ggml_backend_buffer_t buffer) {
GGML_ASSERT(buffer);
ggml_aligned_free(buffer->context, buffer->size);
}
-static void ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_memset_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
GGML_ASSERT(tensor);
memset((char *)tensor->data + offset, value, size);
GGML_UNUSED(buffer);
}
-static void ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_set_tensor(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
GGML_ASSERT(tensor);
memcpy((char *)tensor->data + offset, data, size);
GGML_UNUSED(buffer);
}
-static void ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cpu_buffer_get_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size) {
GGML_ASSERT(tensor);
memcpy(data, (const char *)tensor->data + offset, size);
GGML_UNUSED(buffer);
}
-static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst) {
GGML_ASSERT(src);
if (ggml_backend_buffer_is_host(src->buffer)) {
memcpy(dst->data, src->data, ggml_nbytes(src));
@@ -2515,7 +2530,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
GGML_UNUSED(buffer);
}
-static void ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_cpu_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
GGML_ASSERT(buffer);
memset(buffer->context, value, buffer->size);
}
@@ -2532,6 +2547,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
+ /* .free_struct = */ NULL,
};
static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
@@ -2546,19 +2562,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
+ /* .free_struct = */ NULL,
};
// CPU backend buffer type
// this buffer type is defined here to make it available to all backends
-static const char * ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_cpu_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
return "CPU";
GGML_UNUSED(buft);
}
-static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
void * data = ggml_aligned_malloc(size);
if (data == NULL) {
@@ -2569,13 +2586,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
}
-static size_t ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_cpu_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
return TENSOR_ALIGNMENT;
GGML_UNUSED(buft);
}
-static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
return true;
GGML_UNUSED(buft);
@@ -2600,7 +2617,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
return &ggml_backend_cpu_buffer_type;
}
-static const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_buffer_type_t buft) {
return "CPU_Mapped";
GGML_UNUSED(buft);