Skip to content

Distribute and run LLMs with a single file.

Update llama.cpp submodule to 7b8443ac7 (#951)

Davide Eynardbec4d28e0aa6
diff --git a/llama.cpp b/llama.cpp
index 5e9c6354..7b8443ac 160000
--- a/llama.cpp
+++ b/llama.cpp
@@ -1 +1 @@
-Subproject commit 5e9c6354638858f4c9aa0a70fd4a41a73f894dcc
+Subproject commit 7b8443ac786c06438e0f407b7adaa72c220b5099
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index fd6fbdda..d4489c24 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -24,6 +24,7 @@ GGML_SRCS_C := \
GGML_SRCS_CPP := \
llama.cpp/ggml/src/ggml-backend-dl.cpp \
+ llama.cpp/ggml/src/ggml-backend-meta.cpp \
llama.cpp/ggml/src/ggml-backend-reg.cpp \
llama.cpp/ggml/src/ggml-backend.cpp \
llama.cpp/ggml/src/ggml-opt.cpp \
@@ -108,7 +109,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/lfm2.cpp \
llama.cpp/src/models/llada-moe.cpp \
llama.cpp/src/models/llada.cpp \
- llama.cpp/src/models/llama-iswa.cpp \
+ llama.cpp/src/models/llama4.cpp \
llama.cpp/src/models/llama.cpp \
llama.cpp/src/models/maincoder.cpp \
llama.cpp/src/models/mamba.cpp \
@@ -160,8 +161,8 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/starcoder.cpp \
llama.cpp/src/models/step35-iswa.cpp \
llama.cpp/src/models/starcoder2.cpp \
- llama.cpp/src/models/t5-dec.cpp \
- llama.cpp/src/models/t5-enc.cpp \
+ llama.cpp/src/models/t5.cpp \
+ llama.cpp/src/models/t5encoder.cpp \
llama.cpp/src/models/wavtokenizer-dec.cpp \
llama.cpp/src/models/xverse.cpp \
llama.cpp/src/llama-adapter.cpp \
@@ -208,6 +209,7 @@ COMMON_SRCS_CPP := \
llama.cpp/common/console.cpp \
llama.cpp/common/debug.cpp \
llama.cpp/common/download.cpp \
+ llama.cpp/common/fit.cpp \
llama.cpp/common/hf-cache.cpp \
llama.cpp/common/jinja/caps.cpp \
llama.cpp/common/jinja/lexer.cpp \
@@ -249,6 +251,10 @@ COMMON_SRCS_CPP += o/$(MODE)/llama.cpp/common/build-info.cpp
COMMON_OBJS := $(COMMON_SRCS_CPP:%.cpp=o/$(MODE)/%.cpp.o)
+# build-info.cpp #includes "build-info.h" from llama.cpp/common; tests build the
+# single-prefix object directly via the generic rule, so add the include path.
+o/$(MODE)/llama.cpp/common/build-info.cpp.o: private CPPFLAGS += -iquote llama.cpp/common
+
# ==============================================================================
# Additional support files
# ==============================================================================
@@ -281,6 +287,8 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/cogvlm.cpp \
llama.cpp/tools/mtmd/models/deepseekocr.cpp \
llama.cpp/tools/mtmd/models/conformer.cpp \
+ llama.cpp/tools/mtmd/models/dotsocr.cpp \
+ llama.cpp/tools/mtmd/models/gemma4a.cpp \
llama.cpp/tools/mtmd/models/gemma4v.cpp \
llama.cpp/tools/mtmd/models/glm4v.cpp \
llama.cpp/tools/mtmd/models/hunyuanocr.cpp \
@@ -295,10 +303,12 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/paddleocr.cpp \
llama.cpp/tools/mtmd/models/pixtral.cpp \
llama.cpp/tools/mtmd/models/qwen2vl.cpp \
+ llama.cpp/tools/mtmd/models/qwen3a.cpp \
llama.cpp/tools/mtmd/models/qwen3vl.cpp \
llama.cpp/tools/mtmd/models/siglip.cpp \
llama.cpp/tools/mtmd/models/step3vl.cpp \
llama.cpp/tools/mtmd/models/whisper-enc.cpp \
+ llama.cpp/tools/mtmd/models/yasa2.cpp \
llama.cpp/tools/mtmd/models/youtuvl.cpp
MTMD_OBJS := $(MTMD_SRCS_CPP:%.cpp=o/$(MODE)/%.cpp.o)
@@ -342,6 +352,7 @@ TOOL_BENCH_SRCS := llama.cpp/tools/llama-bench/llama-bench.cpp
TOOL_SERVER_SRCS := \
llama.cpp/tools/server/server.cpp \
+ llama.cpp/tools/server/server-chat.cpp \
llama.cpp/tools/server/server-common.cpp \
llama.cpp/tools/server/server-context.cpp \
llama.cpp/tools/server/server-http.cpp \
diff --git a/llama.cpp.patches/patches/common_arg.cpp.patch b/llama.cpp.patches/patches/common_arg.cpp.patch
index 4626cffa..153f0ac0 100644
--- a/llama.cpp.patches/patches/common_arg.cpp.patch
+++ b/llama.cpp.patches/patches/common_arg.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/arg.cpp b/common/arg.cpp
--- a/llama.cpp/common/arg.cpp
+++ b/llama.cpp/common/arg.cpp
-@@ -37,6 +37,8 @@
+@@ -38,6 +38,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patch
index a09c318f..dfbbcbfd 100644
--- a/llama.cpp.patches/patches/common_common.cpp.patch
+++ b/llama.cpp.patches/patches/common_common.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/common.cpp b/common/common.cpp
--- a/llama.cpp/common/common.cpp
+++ b/llama.cpp/common/common.cpp
-@@ -970,6 +970,16 @@ std::string fs_get_cache_directory() {
+@@ -972,6 +972,16 @@ std::string fs_get_cache_directory() {
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
@@ -18,7 +18,7 @@ diff --git a/common/common.cpp b/common/common.cpp
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
-@@ -1146,10 +1156,31 @@ common_init_result::common_init_result(common_params & params) :
+@@ -1148,10 +1158,31 @@ common_init_result::common_init_result(common_params & params) :
if (params.fit_params) {
LOG_INF("%s: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on\n", __func__);
@@ -43,7 +43,7 @@ diff --git a/common/common.cpp b/common/common.cpp
+ }
+ }
+
- llama_params_fit(params.model.path.c_str(), &mparams, &cparams,
+ common_fit_params(params.model.path.c_str(), &mparams, &cparams,
params.tensor_split,
params.tensor_buft_overrides.data(),
- params.fit_params_target.data(),
diff --git a/llama.cpp.patches/patches/common_download.cpp.patch b/llama.cpp.patches/patches/common_download.cpp.patch
index ec9a78e8..f50ed915 100644
--- a/llama.cpp.patches/patches/common_download.cpp.patch
+++ b/llama.cpp.patches/patches/common_download.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/download.cpp b/common/download.cpp
--- a/llama.cpp/common/download.cpp
+++ b/llama.cpp/common/download.cpp
-@@ -25,6 +25,8 @@
+@@ -26,6 +26,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/common_log.cpp.patch b/llama.cpp.patches/patches/common_log.cpp.patch
index 173fa913..b99c8094 100644
--- a/llama.cpp.patches/patches/common_log.cpp.patch
+++ b/llama.cpp.patches/patches/common_log.cpp.patch
@@ -9,7 +9,7 @@ diff --git a/common/log.cpp b/common/log.cpp
#endif // defined(_WIN32)
int common_log_verbosity_thold = LOG_DEFAULT_LLAMA;
-@@ -257,10 +258,27 @@ public:
+@@ -261,10 +262,27 @@ public:
running = true;
thrd = std::thread([this]() {
diff --git a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
index 68debcbc..d0443c58 100644
--- a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
+++ b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
@@ -18,10 +18,10 @@ diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
#ifdef __cplusplus
extern "C" {
#endif
-@@ -197,19 +207,19 @@ extern "C" {
- // Common functions that may be obtained using ggml_backend_reg_get_proc_address
+@@ -208,19 +218,19 @@ extern "C" {
+ typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
- // Split buffer type for tensor parallelism
+ // Split buffer type for tensor parallelism (old)
- typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
+ typedef ggml_backend_buffer_type_t (GGML_CALL *ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
// Set the number of threads for the backend
diff --git a/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch
index c083b232..a2b00a01 100644
--- a/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch
+++ b/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch
@@ -1,8 +1,8 @@
diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h
--- a/llama.cpp/ggml/include/ggml-cuda.h
+++ b/llama.cpp/ggml/include/ggml-cuda.h
-@@ -28,7 +28,7 @@ GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);
- GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
+@@ -31,7 +31,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int de
+ GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);
// split tensor buffer that splits matrices by rows across multiple devices
-GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);
@@ -10,7 +10,7 @@ diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h
// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
-@@ -37,8 +37,8 @@ GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);
+@@ -40,8 +40,8 @@ GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);
GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
index cf3c05c7..3aaec764 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend_buffer_type {
-@@ -39,22 +39,26 @@ extern "C" {
+@@ -39,26 +39,30 @@ extern "C" {
//
struct ggml_backend_buffer_i {
@@ -46,6 +46,12 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
+ void (GGML_CALL *memset_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
+ void (GGML_CALL *set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+ void (GGML_CALL *get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+ // (optional) 2d data copies
+- void (*set_tensor_2d)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+- void (*get_tensor_2d)(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
++ void (GGML_CALL *set_tensor_2d)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
++ void (GGML_CALL *get_tensor_2d)(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+
// (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)
- bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);
+ bool (GGML_CALL *cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);
@@ -62,7 +68,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend_buffer {
-@@ -85,38 +89,38 @@ extern "C" {
+@@ -103,40 +107,40 @@ extern "C" {
//
struct ggml_backend_i {
@@ -73,11 +79,15 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
+ void (GGML_CALL *free)(ggml_backend_t backend);
// (optional) asynchronous tensor data access
-- void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-- void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+- void (*set_tensor_async) (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
+- void (*get_tensor_async) (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
+- void (*set_tensor_2d_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+- void (*get_tensor_2d_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
- bool (*cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
-+ void (GGML_CALL *set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
-+ void (GGML_CALL *get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
++ void (GGML_CALL *set_tensor_async) (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
++ void (GGML_CALL *get_tensor_async) (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
++ void (GGML_CALL *set_tensor_2d_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
++ void (GGML_CALL *get_tensor_2d_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);
+ bool (GGML_CALL *cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
// (optional) complete all pending operations (required if the backend supports async operations)
@@ -115,7 +125,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend {
-@@ -139,46 +143,46 @@ extern "C" {
+@@ -159,46 +163,46 @@ extern "C" {
// the current functions to obtain the properties can remain, since they are more convenient for often used properties
struct ggml_backend_device_i {
// device name: short identifier for this device, such as "CPU" or "CUDA0"
@@ -177,7 +187,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend_device {
-@@ -192,15 +196,15 @@ extern "C" {
+@@ -212,15 +216,15 @@ extern "C" {
//
struct ggml_backend_reg_i {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
new file mode 100644
index 00000000..2821ecb5
--- /dev/null
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
@@ -0,0 +1,261 @@
+diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
+--- a/llama.cpp/ggml/src/ggml-backend-meta.cpp
++++ b/llama.cpp/ggml/src/ggml-backend-meta.cpp
+@@ -83,19 +83,19 @@ struct ggml_backend_meta_device_context {
+
+ static bool ggml_backend_dev_is_meta(ggml_backend_dev_t dev);
+
+-static const char * ggml_backend_meta_device_get_name(ggml_backend_dev_t dev) {
++static const char * GGML_CALL ggml_backend_meta_device_get_name(ggml_backend_dev_t dev) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+ return meta_dev_ctx->name.c_str();
+ }
+
+-static const char * ggml_backend_meta_device_get_description(ggml_backend_dev_t dev) {
++static const char * GGML_CALL ggml_backend_meta_device_get_description(ggml_backend_dev_t dev) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+ return meta_dev_ctx->description.c_str();
+ }
+
+-static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
++static void GGML_CALL ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+ *free = 0;
+@@ -108,13 +108,13 @@ static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t *
+ }
+ }
+
+-static enum ggml_backend_dev_type ggml_backend_meta_device_get_type(ggml_backend_dev_t dev) {
++static enum ggml_backend_dev_type GGML_CALL ggml_backend_meta_device_get_type(ggml_backend_dev_t dev) {
+ return GGML_BACKEND_DEVICE_TYPE_META;
+
+ GGML_UNUSED(dev);
+ }
+
+-static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
++static void GGML_CALL ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+
+@@ -142,20 +142,20 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back
+ }
+ }
+
+-static ggml_backend_t ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params);
++static ggml_backend_t GGML_CALL ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params);
+
+-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev);
++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev);
+
+-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev);
++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev);
+
+-static bool ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
++static bool GGML_CALL ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+ return std::all_of(meta_dev_ctx->simple_devs.begin(), meta_dev_ctx->simple_devs.end(),
+ [op](ggml_backend_dev_t simple_dev) { return ggml_backend_dev_supports_op(simple_dev, op); });
+ }
+
+-static bool ggml_backend_meta_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
++static bool GGML_CALL ggml_backend_meta_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ ggml_backend_dev_t dev_buft = ggml_backend_buft_get_device(buft);
+ if (!ggml_backend_dev_is_meta(dev_buft)) {
+@@ -272,7 +272,7 @@ static size_t ggml_backend_meta_buft_n_bufts(ggml_backend_buffer_type_t meta_buf
+ return meta_buft_ctx->simple_bufts.size();
+ }
+
+-static const char * ggml_backend_meta_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
++static const char * GGML_CALL ggml_backend_meta_buffer_type_get_name(ggml_backend_buffer_type_t buft) {
+ GGML_ASSERT(ggml_backend_buft_is_meta(buft));
+ const ggml_backend_meta_buffer_type_context * meta_buft_ctx = (const ggml_backend_meta_buffer_type_context *) buft->context;
+ return meta_buft_ctx->name.c_str();
+@@ -285,9 +285,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen
+ return meta_buft_ctx->simple_bufts[index];
+ }
+
+-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
+
+-static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+ size_t max_alignment = 1;
+ for (size_t i = 0; i < n_simple_bufts; i++) {
+@@ -298,7 +298,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty
+ return max_alignment;
+ }
+
+-static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+ size_t max_size = SIZE_MAX;
+ for (size_t i = 0; i < n_simple_bufts; i++) {
+@@ -307,7 +307,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ
+ return max_size;
+ }
+
+-static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+ size_t max_alloc_size = 0;
+ for (size_t i = 0; i < n_simple_bufts; i++) {
+@@ -317,7 +317,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t
+ return max_alloc_size;
+ }
+
+-static bool ggml_backend_meta_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
++static bool GGML_CALL ggml_backend_meta_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+ for (size_t i = 0; i < n_simple_bufts; i++) {
+ if (!ggml_backend_buft_is_host(ggml_backend_meta_buft_simple_buft(buft, i))) {
+@@ -340,7 +340,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {
+ return buft != nullptr && buft->iface.get_name == ggml_backend_meta_buffer_type_iface.get_name;
+ }
+
+-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev) {
++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev) {
+ static std::map<ggml_backend_dev_t, struct ggml_backend_buffer_type> meta_bufts;
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ {
+@@ -367,7 +367,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
+ return &result.first->second;
+ }
+
+-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev) {
++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev) {
+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+
+@@ -414,7 +414,7 @@ struct ggml_backend_meta_buffer_context {
+ }
+ };
+
+-static void ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) {
++static void GGML_CALL ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
+ for (auto & [ctx, buf] : buf_ctx->buf_configs) {
+@@ -1077,12 +1077,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
+ return ret;
+ }
+
+-static void * ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) {
++static void * GGML_CALL ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) {
+ GGML_UNUSED(buffer);
+ return (void *) 0x1000000000000000; // FIXME
+ }
+
+-static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
++static enum ggml_status GGML_CALL ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
+ const size_t n_simple_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
+@@ -1197,7 +1197,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer
+ return GGML_STATUS_SUCCESS;
+ }
+
+-static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
++static void GGML_CALL ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
+ GGML_ASSERT(ggml_is_contiguous(tensor));
+
+@@ -1304,7 +1304,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+ }
+ }
+
+-static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
++static void GGML_CALL ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
+ GGML_ASSERT(ggml_is_contiguous(tensor));
+
+@@ -1397,14 +1397,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+ }
+ }
+
+-static void ggml_backend_meta_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
++static void GGML_CALL ggml_backend_meta_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+ const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer);
+ for (size_t i = 0; i < n_buffers; i++) {
+ ggml_backend_buffer_clear(ggml_backend_meta_buffer_simple_buffer(buffer, i), value);
+ }
+ }
+
+-static void ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {
++static void GGML_CALL ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {
+ const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer);
+ for (size_t i = 0; i < n_buffers; i++) {
+ ggml_backend_buffer_reset(ggml_backend_meta_buffer_simple_buffer(buffer, i));
+@@ -1429,7 +1429,7 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {
+ return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer;
+ }
+
+-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+
+ ggml_init_params params = {
+@@ -1567,20 +1567,20 @@ struct ggml_backend_meta_context {
+ }
+ };
+
+-static const char * ggml_backend_meta_get_name(ggml_backend_t backend) {
++static const char * GGML_CALL ggml_backend_meta_get_name(ggml_backend_t backend) {
+ GGML_ASSERT(ggml_backend_is_meta(backend));
+ const ggml_backend_meta_context * backend_ctx = (const ggml_backend_meta_context *) backend->context;
+ return backend_ctx->name.c_str();
+ }
+
+-static void ggml_backend_meta_free(ggml_backend_t backend) {
++static void GGML_CALL ggml_backend_meta_free(ggml_backend_t backend) {
+ GGML_ASSERT(ggml_backend_is_meta(backend));
+ ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;
+ delete backend_ctx;
+ delete backend;
+ }
+
+-static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
++static void GGML_CALL ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+ const size_t n_backends = ggml_backend_meta_n_backends(backend);
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(ggml_is_contiguous(tensor));
+@@ -1621,7 +1621,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens
+ }
+ }
+
+-static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
++static void GGML_CALL ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+ const size_t n_backends = ggml_backend_meta_n_backends(backend);
+ GGML_ASSERT(offset == 0);
+ GGML_ASSERT(ggml_is_contiguous(tensor));
+@@ -1662,14 +1662,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm
+ }
+ }
+
+-static void ggml_backend_meta_synchronize(ggml_backend_t backend) {
++static void GGML_CALL ggml_backend_meta_synchronize(ggml_backend_t backend) {
+ const size_t n_backends = ggml_backend_meta_n_backends(backend);
+ for (size_t i = 0; i < n_backends; i++) {
+ ggml_backend_synchronize(ggml_backend_meta_simple_backend(backend, i));
+ }
+ }
+
+-static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
++static enum ggml_status GGML_CALL ggml_backend_meta_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
+ GGML_ASSERT(cgraph->grads == nullptr);
+ const size_t n_backends = ggml_backend_meta_n_backends(backend);
+ ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;
+@@ -2101,7 +2101,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {
+ return backend != nullptr && backend->iface.get_name == ggml_backend_meta_i.get_name;
+ }
+
+-static ggml_backend_t ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params) {
++static ggml_backend_t GGML_CALL ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params) {
+ ggml_backend_meta_context * backend_ctx = new ggml_backend_meta_context(dev, params);
+
+ ggml_backend_t backend = new struct ggml_backend;
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
index 64ec872e..ab59ee37 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
--- a/llama.cpp/ggml/src/ggml-backend-reg.cpp
+++ b/llama.cpp/ggml/src/ggml-backend-reg.cpp
-@@ -485,7 +485,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
+@@ -497,7 +497,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
if (ec) {
GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());
} else {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
index fad678f3..ba7bc249 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
}
size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
-@@ -584,7 +591,7 @@ struct ggml_backend_multi_buffer_context {
+@@ -671,7 +678,7 @@ struct ggml_backend_multi_buffer_context {
size_t n_buffers;
};
@@ -26,7 +26,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
-@@ -595,7 +602,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
+@@ -682,7 +689,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
free(ctx);
}
@@ -35,7 +35,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
-@@ -613,6 +620,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
+@@ -702,6 +709,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_multi_buffer_clear,
/* .reset = */ NULL,
@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
};
ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
-@@ -2113,7 +2121,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
+@@ -2210,7 +2218,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
// CPU backend - buffer
@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
uintptr_t data = (uintptr_t)buffer->context;
-@@ -2125,33 +2133,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
+@@ -2222,33 +2230,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
return (void *)data;
}
@@ -91,7 +91,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(src);
if (ggml_backend_buffer_is_host(src->buffer)) {
memcpy(dst->data, src->data, ggml_nbytes(src));
-@@ -2162,7 +2170,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
+@@ -2259,7 +2267,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
GGML_UNUSED(buffer);
}
@@ -100,7 +100,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
memset(buffer->context, value, buffer->size);
}
-@@ -2177,6 +2185,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
+@@ -2276,6 +2284,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
@@ -108,7 +108,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
};
static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
-@@ -2189,19 +2198,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
+@@ -2290,19 +2299,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
void * data = ggml_aligned_malloc(size);
if (data == NULL) {
-@@ -2212,13 +2222,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
+@@ -2313,13 +2323,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
}
@@ -147,7 +147,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
return true;
GGML_UNUSED(buft);
-@@ -2241,7 +2251,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
+@@ -2342,7 +2352,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
return &ggml_backend_cpu_buffer_type;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch
index d3d2b18a..c42352f1 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch
@@ -45,7 +45,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
memset(buffer->context, value, buffer->size);
}
-@@ -116,13 +116,13 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {
+@@ -118,13 +118,13 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {
/* .reset = */ nullptr,
};
@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
void * data = ggml_aligned_malloc(size);
if (data == NULL) {
fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);
-@@ -132,7 +132,7 @@ static ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_back
+@@ -134,7 +134,7 @@ static ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_back
return ggml_backend_buffer_init(buft, ggml_backend_amx_buffer_interface, data, size);
}
@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
return TENSOR_ALIGNMENT;
GGML_UNUSED(buft);
-@@ -198,7 +198,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
+@@ -200,7 +200,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
};
} // namespace ggml::cpu::amx
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
index 1634f0b1..4e4cf7d1 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
@@ -62,7 +62,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
-@@ -248,7 +248,7 @@ bool ggml_backend_is_cpu(ggml_backend_t backend) {
+@@ -250,7 +250,7 @@ bool ggml_backend_is_cpu(ggml_backend_t backend) {
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());
}
@@ -71,7 +71,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
-@@ -267,7 +267,7 @@ void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool
+@@ -269,7 +269,7 @@ void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool
ctx->threadpool = threadpool;
}
@@ -80,7 +80,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
-@@ -348,19 +348,19 @@ struct ggml_backend_cpu_device_context {
+@@ -350,19 +350,19 @@ struct ggml_backend_cpu_device_context {
}
};
@@ -103,7 +103,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
#ifdef _WIN32
MEMORYSTATUSEX status;
status.dwLength = sizeof(status);
-@@ -379,13 +379,13 @@ static void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -381,13 +381,13 @@ static void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t *
GGML_UNUSED(dev);
}
@@ -119,7 +119,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
props->name = ggml_backend_cpu_device_get_name(dev);
props->description = ggml_backend_cpu_device_get_description(dev);
props->type = ggml_backend_cpu_device_get_type(dev);
-@@ -398,27 +398,27 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm
+@@ -400,27 +400,27 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm
};
}
@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
-@@ -471,7 +471,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
+@@ -473,7 +473,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
}
}
@@ -160,7 +160,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);
GGML_UNUSED(dev);
}
-@@ -496,19 +496,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
+@@ -498,19 +498,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
// CPU backend - backend (reg)
@@ -183,7 +183,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
GGML_ASSERT(index == 0);
static ggml_backend_cpu_device_context ctx;
-@@ -523,7 +523,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg
+@@ -525,7 +525,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg
// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,
// and additionally to allow other backends to expose their own list of features that applications can query using the same API
@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
static std::vector<ggml_backend_feature> features = []() {
ggml_cpu_init();
-@@ -637,7 +637,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
+@@ -639,7 +639,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
index 74e0a341..84db581e 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh
-@@ -292,9 +292,15 @@ static bool fp16_mma_hardware_available(const int cc) {
+@@ -294,9 +294,15 @@ static bool fp16_mma_hardware_available(const int cc) {
}
static bool bf16_mma_hardware_available(const int cc) {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
index f187c4bd..bf0329de 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
-@@ -596,7 +596,7 @@ struct ggml_backend_cuda_buffer_context {
+@@ -631,7 +631,7 @@ struct ggml_backend_cuda_buffer_context {
}
};
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
delete ctx;
}
-@@ -605,12 +605,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
+@@ -640,12 +640,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;
}
@@ -25,34 +25,52 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
if (tensor->view_src != NULL) {
-@@ -631,7 +631,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
+@@ -666,7 +666,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
return GGML_STATUS_SUCCESS;
}
-static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -639,7 +639,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
+@@ -674,7 +674,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
-static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -647,7 +647,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -682,7 +682,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
-static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -655,7 +655,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -690,7 +690,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+ }
+
+-static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data,
++static void GGML_CALL ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data,
+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
+
+@@ -700,7 +700,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
+ }
+
+-static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data,
++static void GGML_CALL ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data,
+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
+
+@@ -710,7 +710,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -61,7 +79,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (ggml_backend_buffer_is_cuda(src->buffer)) {
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
-@@ -676,7 +676,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
+@@ -731,7 +731,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
GGML_UNUSED(buffer);
}
@@ -70,7 +88,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -684,6 +684,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
+@@ -739,6 +739,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -85,7 +103,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
/* .get_base = */ ggml_backend_cuda_buffer_get_base,
-@@ -694,6 +702,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
+@@ -751,6 +759,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cuda_buffer_clear,
/* .reset = */ NULL,
@@ -93,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
};
// cuda buffer type
-@@ -702,7 +711,7 @@ struct ggml_backend_cuda_buffer_type_context {
+@@ -759,7 +768,7 @@ struct ggml_backend_cuda_buffer_type_context {
std::string name;
};
@@ -102,7 +120,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -712,7 +721,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
+@@ -769,7 +778,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;
}
@@ -111,7 +129,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
ggml_cuda_set_device(buft_ctx->device);
-@@ -731,13 +740,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
+@@ -788,13 +797,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
}
@@ -127,7 +145,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t size = ggml_nbytes(tensor);
int64_t ne0 = tensor->ne[0];
-@@ -851,19 +860,19 @@ struct ggml_backend_cuda_split_buffer_context {
+@@ -908,19 +917,19 @@ struct ggml_backend_cuda_split_buffer_context {
};
@@ -150,7 +168,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -913,7 +922,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
+@@ -970,7 +979,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
return GGML_STATUS_SUCCESS;
}
@@ -159,7 +177,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -952,7 +961,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
+@@ -1009,7 +1018,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
}
}
@@ -168,7 +186,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -991,7 +1000,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
+@@ -1048,7 +1057,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
}
}
@@ -177,7 +195,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(buffer);
GGML_UNUSED(value);
}
-@@ -1006,11 +1015,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
+@@ -1065,11 +1074,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_cuda_split_buffer_clear,
/* .reset = */ NULL,
@@ -191,7 +209,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -1020,7 +1030,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
+@@ -1079,7 +1089,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name;
}
@@ -200,7 +218,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
// instead, we allocate them for each tensor separately in init_tensor
// however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,
-@@ -1030,13 +1040,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
+@@ -1089,13 +1099,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);
}
@@ -216,7 +234,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -1064,7 +1074,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
+@@ -1123,7 +1133,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
return total_size;
}
@@ -225,16 +243,16 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return false;
GGML_UNUSED(buft);
-@@ -1079,7 +1089,7 @@ static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_inte
- /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,
- };
+@@ -1280,7 +1290,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg
+ #endif // GGML_USE_NCCL
+ }
-ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {
+ggml_backend_buffer_type_t GGML_CALL ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {
static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex);
-@@ -1123,7 +1133,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
+@@ -1324,7 +1334,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
// host buffer type
@@ -243,7 +261,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return GGML_CUDA_NAME "_Host";
GGML_UNUSED(buft);
-@@ -1133,7 +1143,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
+@@ -1334,7 +1344,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
}
@@ -252,7 +270,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
CUDA_CHECK(cudaFreeHost(buffer->context));
}
-@@ -1155,17 +1165,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
+@@ -1356,17 +1366,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
return ptr;
}
@@ -280,7 +298,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return buffer;
}
-@@ -1293,8 +1311,13 @@ static void ggml_cuda_op_mul_mat_cublas(
+@@ -1494,8 +1512,13 @@ static void ggml_cuda_op_mul_mat_cublas(
const int cc = ggml_cuda_info().devices[id].cc;
@@ -294,7 +312,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
const bool use_fp16 =
src0->type != GGML_TYPE_NVFP4 &&
-@@ -2831,20 +2854,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+@@ -2969,20 +2992,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
// backend
@@ -315,20 +333,38 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
-static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2853,7 +2876,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
- CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
+@@ -2991,7 +3014,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
+ CUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
-static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2862,7 +2885,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
- CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
+@@ -3000,7 +3023,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *) tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
+ }
+
+-static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data,
++static void GGML_CALL ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data,
+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+@@ -3011,7 +3034,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct
+ (char *) tensor->data + offset, stride_tensor, data, stride_data, size, n_copies, cudaMemcpyHostToDevice, cuda_ctx->stream()));
+ }
+
+-static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data,
++static void GGML_CALL ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data,
+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
+@@ -3022,7 +3045,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const
+ data, stride_data, (const char *) tensor->data + offset, stride_tensor, size, n_copies, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
-static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
@@ -336,7 +372,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
-@@ -2917,7 +2940,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
+@@ -3077,7 +3100,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
return true;
}
@@ -345,7 +381,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
-@@ -4010,7 +4033,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
+@@ -4215,7 +4238,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
}
#endif // USE_CUDA_GRAPH
@@ -354,7 +390,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_cuda_set_device(cuda_ctx->device);
-@@ -4069,13 +4092,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
+@@ -4274,13 +4297,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
return GGML_STATUS_SUCCESS;
}
@@ -370,7 +406,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
if (ggml_backend_is_cuda(backend)) {
-@@ -4094,7 +4117,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
+@@ -4299,7 +4322,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
}
}
@@ -379,7 +415,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
#ifdef USE_CUDA_GRAPH
-@@ -4380,7 +4403,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
+@@ -4587,7 +4610,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
CUDA_CHECK(cudaMemGetInfo(free, total));
}
@@ -388,7 +424,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return false;
}
-@@ -4403,7 +4426,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+@@ -4610,7 +4633,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)
}
@@ -397,7 +433,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return;
}
-@@ -4426,12 +4449,12 @@ struct ggml_backend_cuda_device_context {
+@@ -4633,12 +4656,12 @@ struct ggml_backend_cuda_device_context {
int op_offload_min_batch_size;
};
@@ -412,7 +448,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
return ctx->description.c_str();
}
-@@ -4512,7 +4535,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
+@@ -4719,7 +4742,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
}
#endif // defined(__linux__)
@@ -421,7 +457,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
ggml_cuda_set_device(ctx->device);
CUDA_CHECK(cudaMemGetInfo(free, total));
-@@ -4542,12 +4565,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -4749,12 +4772,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
}
@@ -436,7 +472,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
props->name = ggml_backend_cuda_device_get_name(dev);
-@@ -4571,24 +4594,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
+@@ -4778,24 +4801,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
};
}
@@ -465,7 +501,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
// split buffers can only be used with GGML_OP_MUL_MAT
-@@ -4970,7 +4993,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5179,7 +5202,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
}
@@ -474,7 +510,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;
return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)));
-@@ -4991,13 +5014,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -5200,13 +5223,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -490,7 +526,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
#ifdef GGML_CUDA_NO_PEER_COPY
return nullptr;
#else
-@@ -5015,14 +5038,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+@@ -5224,14 +5247,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
#endif
}
@@ -507,7 +543,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(dev);
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
}
-@@ -5051,23 +5074,23 @@ struct ggml_backend_cuda_reg_context {
+@@ -5260,23 +5283,23 @@ struct ggml_backend_cuda_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
@@ -535,12 +571,12 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::vector<ggml_backend_feature> features = []() {
std::vector<ggml_backend_feature> features;
#define _STRINGIFY(...) #__VA_ARGS__
-@@ -5128,7 +5151,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
+@@ -5337,7 +5360,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
GGML_UNUSED(reg);
}
-static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
+static void * GGML_CALL ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {
GGML_UNUSED(reg);
- if (strcmp(name, "ggml_backend_split_buffer_type") == 0) {
- return (void *)ggml_backend_cuda_split_buffer_type;
+ if (strcmp(name, "ggml_backend_comm_init") == 0) {
+ return (void *)ggml_backend_cuda_comm_init;
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
index 3b48ffe6..95be7ca5 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda
#include <cuda_bf16.h>
#include <cuda_fp16.h>
-@@ -17,8 +27,10 @@
+@@ -21,8 +31,10 @@
#if CUDART_VERSION < 11020
#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
index f99567d3..1c1f68cb 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
@@ -79,7 +79,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
/* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer,
/* .get_base = */ ggml_backend_metal_buffer_shared_get_base,
-@@ -93,11 +101,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
+@@ -95,11 +103,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {
/* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,
/* .clear = */ ggml_backend_metal_buffer_shared_clear,
/* .reset = */ NULL,
@@ -93,7 +93,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -105,7 +114,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t
+@@ -107,7 +116,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t
ggml_metal_buffer_free(ctx);
}
@@ -102,7 +102,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -113,7 +122,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b
+@@ -115,7 +124,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b
return ggml_metal_buffer_get_base(ctx);
}
@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -121,7 +130,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_
+@@ -123,7 +132,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_
ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);
}
@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -129,7 +138,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b
+@@ -131,7 +140,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);
}
@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -137,7 +146,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
+@@ -139,7 +148,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
}
@@ -138,7 +138,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -149,7 +158,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
+@@ -151,7 +160,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
return false;
}
@@ -147,15 +147,15 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
-@@ -167,6 +176,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
- /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
- /* .clear = */ ggml_backend_metal_buffer_private_clear,
- /* .reset = */ NULL,
-+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
+@@ -171,6 +180,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
+ /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
+ /* .clear = */ ggml_backend_metal_buffer_private_clear,
+ /* .reset = */ NULL,
++ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
};
static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {
-@@ -239,33 +249,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
+@@ -243,33 +253,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
// default (shared) buffer type
@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -315,33 +325,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
+@@ -319,33 +329,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
// default (private) buffer type
@@ -235,7 +235,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -390,34 +400,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
+@@ -394,34 +404,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
// mapped buffer type
@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -468,13 +478,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
+@@ -472,13 +482,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
// backend
@@ -292,7 +292,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_t ctx = (ggml_metal_t)backend->context;
// wait for any ongoing async operations to finish
-@@ -485,25 +495,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
+@@ -489,25 +499,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
free(backend);
}
@@ -322,7 +322,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
if (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {
return false;
}
-@@ -524,27 +534,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
+@@ -528,27 +538,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
return ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);
}
@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_t ctx = (ggml_metal_t)backend->context;
ggml_metal_graph_optimize(ctx, cgraph);
-@@ -634,7 +644,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
+@@ -640,7 +650,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
// backend device
@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -642,25 +652,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
+@@ -648,25 +658,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
return props_dev->name;
}
@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
props->name = ggml_backend_metal_device_get_name(dev);
props->description = ggml_backend_metal_device_get_description(dev);
props->type = ggml_backend_metal_device_get_type(dev);
-@@ -675,7 +685,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
+@@ -681,7 +691,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
};
}
@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_t ctx = ggml_metal_init(ctx_dev);
-@@ -700,7 +710,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
+@@ -706,7 +716,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
GGML_UNUSED(params);
}
@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -708,7 +718,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
+@@ -714,7 +724,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
return props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);
}
@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);
-@@ -718,13 +728,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
+@@ -724,13 +734,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
return ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);
}
@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return
buft->device == dev && (
buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||
-@@ -745,7 +755,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -751,7 +761,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -445,7 +445,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
return (op->op == GGML_OP_MUL_MAT ||
-@@ -753,7 +763,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
+@@ -759,7 +769,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
get_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;
}
@@ -454,7 +454,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);
-@@ -767,7 +777,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
+@@ -773,7 +783,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
return ev;
}
@@ -463,7 +463,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
-@@ -777,7 +787,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
+@@ -783,7 +793,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
delete event;
}
@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t evt = (ggml_metal_event_t)event->context;
-@@ -829,18 +839,18 @@ struct ggml_backend_metal_reg_deleter {
+@@ -835,18 +845,18 @@ struct ggml_backend_metal_reg_deleter {
typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;
@@ -494,7 +494,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;
GGML_ASSERT(index < ctx->devices.size());
return ctx->devices[index];
-@@ -853,13 +863,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
+@@ -859,13 +869,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
{ NULL, NULL },
};
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
index 81b8e2ce..53a01fbc 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
-@@ -238,11 +238,11 @@ struct vk_queue {
+@@ -252,11 +252,11 @@ struct vk_queue {
}
};
@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
/* .get_name = */ ggml_backend_vk_buffer_type_name,
/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
-@@ -2074,7 +2074,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
+@@ -2089,7 +2089,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
@@ -27,7 +27,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {
const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},
-@@ -3391,20 +3391,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
+@@ -3469,20 +3469,15 @@ static void ggml_vk_load_shaders(vk_device& device) {
if (!pipeline->needed || pipeline->compiled) {
continue;
}
@@ -54,7 +54,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
}
};
-@@ -13428,20 +13423,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
+@@ -13569,20 +13564,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;
}
@@ -86,7 +86,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
-@@ -13449,7 +13452,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
+@@ -13590,7 +13593,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
return GGML_STATUS_SUCCESS;
}
@@ -95,7 +95,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13462,7 +13465,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
+@@ -13603,7 +13606,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);
}
@@ -104,7 +104,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13474,7 +13477,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
+@@ -13615,7 +13618,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
@@ -113,7 +113,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
-@@ -13487,7 +13490,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
+@@ -13628,7 +13631,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
@@ -122,7 +122,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (ggml_nbytes(src) == 0) {
return true;
}
-@@ -13508,7 +13511,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
+@@ -13649,7 +13652,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
UNUSED(buffer);
}
@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);
-@@ -13524,16 +13527,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
+@@ -13667,16 +13670,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
/* .clear = */ ggml_backend_vk_buffer_clear,
/* .reset = */ NULL,
@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
-@@ -13549,17 +13553,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
+@@ -13692,17 +13696,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);
}
@@ -172,7 +172,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return ggml_nbytes(tensor);
UNUSED(buft);
-@@ -13577,24 +13581,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
+@@ -13720,24 +13724,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
// host buffer type
@@ -201,7 +201,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");
size += 32; // Behave like the CPU buffer type
-@@ -13610,19 +13614,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
+@@ -13753,19 +13757,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
buffer->buft = buft;
buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;
@@ -224,7 +224,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return vk_instance.devices[0]->suballocation_block_size;
UNUSED(buft);
-@@ -13654,13 +13659,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
+@@ -13797,13 +13802,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
// backend
@@ -240,7 +240,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
-@@ -13676,7 +13681,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
+@@ -13819,7 +13824,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
return &ctx->device->buffer_type;
}
@@ -249,7 +249,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_set_tensor_async(" << size << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13723,7 +13728,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor
+@@ -13866,7 +13871,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor
}
}
@@ -258,7 +258,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_get_tensor_async(" << size << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13757,7 +13762,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_
+@@ -13900,7 +13905,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_
}
}
@@ -267,7 +267,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;
-@@ -13882,7 +13887,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
+@@ -14025,7 +14030,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
}
}
@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_synchronize()");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14304,7 +14309,7 @@ static int32_t find_first_set(uint32_t x) {
+@@ -14446,7 +14451,7 @@ static int32_t find_first_set(uint32_t x) {
return ret;
}
@@ -285,7 +285,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14684,7 +14689,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+@@ -14828,7 +14833,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
}
// Sort the graph for improved parallelism.
@@ -294,7 +294,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
{
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14922,7 +14927,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
+@@ -15066,7 +15071,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
}
}
@@ -303,7 +303,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -14960,7 +14965,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
+@@ -15104,7 +15109,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
ctx->compute_ctx.reset();
}
@@ -312,7 +312,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -15055,7 +15060,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
+@@ -15201,7 +15206,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
*total += heap.size;
if (membudget_supported && i < budgetprops.heapUsage.size()) {
@@ -324,7 +324,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
} else {
*free += heap.size;
}
-@@ -15123,38 +15131,38 @@ struct ggml_backend_vk_device_context {
+@@ -15269,38 +15277,38 @@ struct ggml_backend_vk_device_context {
int op_offload_min_batch_size;
};
@@ -370,7 +370,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
props->name = ggml_backend_vk_device_get_name(dev);
-@@ -15170,13 +15178,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
+@@ -15316,13 +15324,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
};
}
@@ -386,7 +386,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
const vk_device& device = ggml_vk_get_device(ctx->device);
-@@ -15714,7 +15722,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
+@@ -15868,7 +15876,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
UNUSED(dev);
}
@@ -395,7 +395,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {
return false;
}
-@@ -15740,13 +15748,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
+@@ -15894,13 +15902,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15769,7 +15777,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
+@@ -15923,7 +15931,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
};
}
@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15789,7 +15797,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
+@@ -15943,7 +15951,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
delete event;
}
@@ -429,7 +429,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15846,7 +15854,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
+@@ -16000,7 +16008,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
return buf;
}
@@ -438,7 +438,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
GGML_UNUSED(max_tensor_size);
-@@ -15884,17 +15892,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
+@@ -16038,17 +16046,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,
};
diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
index 0900d663..52d68c2b 100644
--- a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
--- a/llama.cpp/tools/server/server-models.cpp
+++ b/llama.cpp/tools/server/server-models.cpp
-@@ -27,6 +27,7 @@
+@@ -30,6 +30,7 @@
#include <netinet/in.h>
#include <arpa/inet.h>
#include <unistd.h>
@@ -9,7 +9,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
extern char **environ;
#endif
-@@ -519,9 +520,12 @@ void server_models::unload_lru() {
+@@ -523,9 +524,12 @@ void server_models::unload_lru() {
// wait for unload to complete
{
std::unique_lock<std::mutex> lk(mutex);
@@ -25,7 +25,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
}
}
-@@ -623,6 +627,14 @@ void server_models::load(const std::string & name) {
+@@ -627,6 +631,14 @@ void server_models::load(const std::string & name) {
});
std::thread stopping_thread([&]() {
@@ -40,7 +40,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
// thread to monitor stopping signal OR child crash
auto is_stopping = [this, &name]() {
return this->stopping_models.find(name) != this->stopping_models.end();
-@@ -632,7 +644,10 @@ void server_models::load(const std::string & name) {
+@@ -636,7 +648,10 @@ void server_models::load(const std::string & name) {
};
{
std::unique_lock<std::mutex> lk(this->mutex);
@@ -52,7 +52,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
// child may have already exited (e.g. crashed) — skip shutdown sequence
if (!subprocess_alive(child_proc.get())) {
-@@ -753,13 +768,14 @@ void server_models::update_status(const std::string & name, server_model_status
+@@ -762,13 +777,14 @@ void server_models::update_status(const std::string & name, server_model_status
void server_models::wait_until_loading_finished(const std::string & name) {
std::unique_lock<std::mutex> lk(mutex);
diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patch
index 54cabb6e..8a48a3ff 100644
--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
--- a/llama.cpp/tools/server/server.cpp
+++ b/llama.cpp/tools/server/server.cpp
-@@ -12,6 +12,7 @@
+@@ -14,6 +14,7 @@
#include <atomic>
#include <clocale>
#include <exception>
@@ -9,7 +9,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
-@@ -19,6 +20,11 @@
+@@ -21,6 +22,11 @@
#include <windows.h>
#endif
@@ -21,7 +21,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
-@@ -69,12 +75,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
+@@ -71,12 +77,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
};
}
@@ -45,7 +45,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
common_init();
if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
-@@ -293,6 +309,13 @@ int main(int argc, char ** argv) {
+@@ -295,6 +311,13 @@ int main(int argc, char ** argv) {
// this will unblock start_loop()
ctx_server.terminate();
};
@@ -59,7 +59,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
}
// TODO: refactor in common/console
-@@ -324,6 +347,11 @@ int main(int argc, char ** argv) {
+@@ -326,6 +349,11 @@ int main(int argc, char ** argv) {
LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());
LOG_INF("%s: starting the main loop...\n", __func__);
@@ -71,7 +71,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
// optionally, notify router server that this instance is ready
std::thread monitor_thread;
if (server_models::is_child_server()) {
-@@ -347,5 +375,48 @@ int main(int argc, char ** argv) {
+@@ -349,5 +377,48 @@ int main(int argc, char ** argv) {
}
}
diff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
index 82a8f41e..8c350166 100644
--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp
+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp
-@@ -6214,6 +6214,15 @@ void ThreadPool::cleanup_finished_threads() {
+@@ -6257,6 +6257,15 @@ void ThreadPool::cleanup_finished_threads() {
}
void ThreadPool::worker(bool is_dynamic) {
@@ -17,7 +17,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
for (;;) {
std::function<void()> fn;
{
-@@ -6231,7 +6240,13 @@ void ThreadPool::worker(bool is_dynamic) {
+@@ -6274,7 +6283,13 @@ void ThreadPool::worker(bool is_dynamic) {
break;
}
} else {
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index de077711..2aef7863 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -219,6 +219,7 @@ LLAMAFILE_HIGHLIGHT_KEYWORDS := $(LLAMAFILE_HIGHLIGHT_GPERF_FILES:%.gperf=o/$(MO
# Server objects for llamafile
LLAMAFILE_SERVER_SUPPORT_OBJS := \
+ o/$(MODE)/llama.cpp/tools/server/server-chat.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-common.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-context.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-http.cpp.o \
@@ -233,6 +234,7 @@ LLAMAFILE_METAL_SOURCES := \
o/$(MODE)/llama.cpp/ggml/src/ggml.c.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-alloc.c.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-backend.cpp.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-backend-meta.cpp.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-quants.c.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-threading.cpp.zip.o \
o/$(MODE)/llama.cpp/ggml/include/ggml.h.zip.o \
@@ -240,6 +242,7 @@ LLAMAFILE_METAL_SOURCES := \
o/$(MODE)/llama.cpp/ggml/include/ggml-cpu.h.zip.o \
o/$(MODE)/llama.cpp/ggml/include/ggml-alloc.h.zip.o \
o/$(MODE)/llama.cpp/ggml/include/ggml-backend.h.zip.o \
+ o/$(MODE)/llama.cpp/ggml/include/ggml-cpp.h.zip.o \
o/$(MODE)/llama.cpp/ggml/include/ggml-metal.h.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-impl.h.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-common.h.zip.o \
diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.sh
index 2d82d99f..3f81934d 100644
--- a/llamafile/build-functions.sh
+++ b/llamafile/build-functions.sh
@@ -175,6 +175,7 @@ compile_ggml_core() {
$llama_cpp_dir/ggml/src/ggml.c \
$llama_cpp_dir/ggml/src/ggml-alloc.c \
$llama_cpp_dir/ggml/src/ggml-backend.cpp \
+ $llama_cpp_dir/ggml/src/ggml-backend-meta.cpp \
$llama_cpp_dir/ggml/src/ggml-quants.c \
$llama_cpp_dir/ggml/src/ggml-threading.cpp"
diff --git a/llamafile/cuda.bat b/llamafile/cuda.bat
index 7168d975..ad0b428d 100644
--- a/llamafile/cuda.bat
+++ b/llamafile/cuda.bat
@@ -249,7 +249,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do (
)
:: C++ sources
-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (
+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (
set "SRC=%GGML_SRC_DIR%\%%f"
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"
diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.bat
index 50ed8493..65c63aec 100644
--- a/llamafile/cuda_parallel.bat
+++ b/llamafile/cuda_parallel.bat
@@ -268,7 +268,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do (
)
:: C++ sources
-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (
+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (
set "SRC=%GGML_SRC_DIR%\%%f"
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"
diff --git a/llamafile/metal.c b/llamafile/metal.c
index 778ef1b0..0f1a7187 100644
--- a/llamafile/metal.c
+++ b/llamafile/metal.c
@@ -54,6 +54,7 @@ extern char **environ;
__static_yoink("llama.cpp/ggml/src/ggml.c");
__static_yoink("llama.cpp/ggml/src/ggml-alloc.c");
__static_yoink("llama.cpp/ggml/src/ggml-backend.cpp");
+__static_yoink("llama.cpp/ggml/src/ggml-backend-meta.cpp");
__static_yoink("llama.cpp/ggml/src/ggml-quants.c");
__static_yoink("llama.cpp/ggml/src/ggml-threading.cpp");
@@ -63,6 +64,7 @@ __static_yoink("llama.cpp/ggml/include/gguf.h");
__static_yoink("llama.cpp/ggml/include/ggml-cpu.h");
__static_yoink("llama.cpp/ggml/include/ggml-alloc.h");
__static_yoink("llama.cpp/ggml/include/ggml-backend.h");
+__static_yoink("llama.cpp/ggml/include/ggml-cpp.h");
__static_yoink("llama.cpp/ggml/include/ggml-metal.h");
__static_yoink("llama.cpp/ggml/src/ggml-impl.h");
__static_yoink("llama.cpp/ggml/src/ggml-common.h");
@@ -96,6 +98,7 @@ static const struct MetalSource {
{"/zip/llama.cpp/ggml/include/ggml-cpu.h", "ggml-cpu.h"},
{"/zip/llama.cpp/ggml/include/ggml-alloc.h", "ggml-alloc.h"},
{"/zip/llama.cpp/ggml/include/ggml-backend.h", "ggml-backend.h"},
+ {"/zip/llama.cpp/ggml/include/ggml-cpp.h", "ggml-cpp.h"},
{"/zip/llama.cpp/ggml/include/ggml-metal.h", "ggml-metal.h"},
{"/zip/llama.cpp/ggml/src/ggml-impl.h", "ggml-impl.h"},
{"/zip/llama.cpp/ggml/src/ggml-common.h", "ggml-common.h"},
@@ -108,6 +111,7 @@ static const struct MetalSource {
{"/zip/llama.cpp/ggml/src/ggml.c", "ggml.c"},
{"/zip/llama.cpp/ggml/src/ggml-alloc.c", "ggml-alloc.c"},
{"/zip/llama.cpp/ggml/src/ggml-backend.cpp", "ggml-backend.cpp"},
+ {"/zip/llama.cpp/ggml/src/ggml-backend-meta.cpp", "ggml-backend-meta.cpp"},
{"/zip/llama.cpp/ggml/src/ggml-quants.c", "ggml-quants.c"},
{"/zip/llama.cpp/ggml/src/ggml-threading.cpp", "ggml-threading.cpp"},
@@ -379,6 +383,7 @@ static bool BuildMetal(const char *dso) {
"ggml-alloc.c",
"ggml-quants.c",
"ggml-backend.cpp",
+ "ggml-backend-meta.cpp",
"ggml-threading.cpp",
"ggml-metal.cpp",
"ggml-metal-device.cpp",
diff --git a/llamafile/rocm.bat b/llamafile/rocm.bat
index c136a82a..e4914d09 100644
--- a/llamafile/rocm.bat
+++ b/llamafile/rocm.bat
@@ -235,7 +235,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do (
)
:: C++ sources
-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (
+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (
set "SRC=%GGML_SRC_DIR%\%%f"
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"
diff --git a/llamafile/rocm_parallel.bat b/llamafile/rocm_parallel.bat
index ee1c441c..36ce7684 100644
--- a/llamafile/rocm_parallel.bat
+++ b/llamafile/rocm_parallel.bat
@@ -254,7 +254,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do (
)
:: C++ sources
-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (
+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (
set "SRC=%GGML_SRC_DIR%\%%f"
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"
diff --git a/llamafile/vulkan.bat b/llamafile/vulkan.bat
index 4cb9d19d..53f941ad 100644
--- a/llamafile/vulkan.bat
+++ b/llamafile/vulkan.bat
@@ -292,7 +292,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do (
)
:: C++ sources
-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (
+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (
set "SRC=%GGML_SRC_DIR%\%%f"
set "BASE=%%~nf"
set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"
diff --git a/llamafile/vulkan.sh b/llamafile/vulkan.sh
index a988972a..f1824a1f 100755
--- a/llamafile/vulkan.sh
+++ b/llamafile/vulkan.sh
@@ -69,15 +69,44 @@ if [ -z "$GLSLC" ]; then
GLSLC="/usr/bin/glslc"
else
echo "Error: glslc not found. Please install the Vulkan SDK."
- echo " Linux (Ubuntu/Debian): sudo apt install vulkan-sdk"
- echo " Linux (Fedora): sudo dnf install vulkan-tools shaderc"
- echo " Linux (Arch): sudo pacman -S vulkan-devel shaderc"
+ echo " Linux (Ubuntu/Debian): sudo apt install vulkan-sdk spirv-headers"
+ echo " Linux (Fedora): sudo dnf install vulkan-tools shaderc spirv-headers-devel"
+ echo " Linux (Arch): sudo pacman -S vulkan-devel shaderc spirv-headers"
echo " macOS: brew install vulkan-sdk"
echo " Or set VULKAN_SDK environment variable"
exit 1
fi
fi
+# Find SPIR-V headers (required by ggml-vulkan.cpp since llama.cpp PR #21572).
+# Probe the same cascade as the C++ source, plus VULKAN_SDK. For each hit,
+# strip the matched relative path off to get the -I root.
+SPIRV_INCLUDE=""
+probe() {
+ local rel="$1" base
+ for base in \
+ "$VULKAN_SDK/include" \
+ /usr/include \
+ /usr/local/include
+ do
+ [ -z "$base" ] && continue
+ if [ -f "$base/$rel" ]; then
+ SPIRV_INCLUDE="-I$base"
+ return 0
+ fi
+ done
+ return 1
+}
+probe spirv/unified1/spirv.hpp || probe spirv-headers/spirv.hpp || probe spirv.hpp || true
+if [ -z "$SPIRV_INCLUDE" ]; then
+ echo "Error: SPIR-V headers not found (spirv.hpp). Required by ggml-vulkan.cpp."
+ echo " Linux (Ubuntu/Debian): sudo apt install spirv-headers"
+ echo " Linux (Fedora): sudo dnf install spirv-headers-devel"
+ echo " Linux (Arch): sudo pacman -S spirv-headers"
+ echo " Or install the LunarG Vulkan SDK and set VULKAN_SDK."
+ exit 1
+fi
+
# Directory setup
LLAMAFILE_DIR="$SCRIPT_DIR"
LLAMA_CPP_DIR="$SCRIPT_DIR/../llama.cpp"
@@ -249,6 +278,7 @@ if [ ! -f "$VULKAN_OBJ" ] || [ "$VULKAN_SRC" -nt "$VULKAN_OBJ" ] || [ "$SHADERS_
echo " Compiling ggml-vulkan.cpp..."
$CXX -c $CXX_FLAGS \
-I$GGML_VULKAN_DIR \
+ $SPIRV_INCLUDE \
-o "$VULKAN_OBJ" "$VULKAN_SRC"
else
echo " ggml-vulkan.o is up to date"