Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Update llama.cpp submodule to 7b8443ac7 (#951)
Davide Eynardbec4d28e0aa6
diff --git a/llama.cpp b/llama.cppindex 5e9c6354..7b8443ac 160000--- a/llama.cpp+++ b/llama.cpp@@ -1 +1 @@-Subproject commit 5e9c6354638858f4c9aa0a70fd4a41a73f894dcc+Subproject commit 7b8443ac786c06438e0f407b7adaa72c220b5099diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mkindex fd6fbdda..d4489c24 100644--- a/llama.cpp.patches/llamafile-files/BUILD.mk+++ b/llama.cpp.patches/llamafile-files/BUILD.mk@@ -24,6 +24,7 @@ GGML_SRCS_C := \GGML_SRCS_CPP := \llama.cpp/ggml/src/ggml-backend-dl.cpp \+ llama.cpp/ggml/src/ggml-backend-meta.cpp \llama.cpp/ggml/src/ggml-backend-reg.cpp \llama.cpp/ggml/src/ggml-backend.cpp \llama.cpp/ggml/src/ggml-opt.cpp \@@ -108,7 +109,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/lfm2.cpp \llama.cpp/src/models/llada-moe.cpp \llama.cpp/src/models/llada.cpp \- llama.cpp/src/models/llama-iswa.cpp \+ llama.cpp/src/models/llama4.cpp \llama.cpp/src/models/llama.cpp \llama.cpp/src/models/maincoder.cpp \llama.cpp/src/models/mamba.cpp \@@ -160,8 +161,8 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/starcoder.cpp \llama.cpp/src/models/step35-iswa.cpp \llama.cpp/src/models/starcoder2.cpp \- llama.cpp/src/models/t5-dec.cpp \- llama.cpp/src/models/t5-enc.cpp \+ llama.cpp/src/models/t5.cpp \+ llama.cpp/src/models/t5encoder.cpp \llama.cpp/src/models/wavtokenizer-dec.cpp \llama.cpp/src/models/xverse.cpp \llama.cpp/src/llama-adapter.cpp \@@ -208,6 +209,7 @@ COMMON_SRCS_CPP := \llama.cpp/common/console.cpp \llama.cpp/common/debug.cpp \llama.cpp/common/download.cpp \+ llama.cpp/common/fit.cpp \llama.cpp/common/hf-cache.cpp \llama.cpp/common/jinja/caps.cpp \llama.cpp/common/jinja/lexer.cpp \@@ -249,6 +251,10 @@ COMMON_SRCS_CPP += o/$(MODE)/llama.cpp/common/build-info.cppCOMMON_OBJS := $(COMMON_SRCS_CPP:%.cpp=o/$(MODE)/%.cpp.o)+# build-info.cpp #includes "build-info.h" from llama.cpp/common; tests build the+# single-prefix object directly via the generic rule, so add the include path.+o/$(MODE)/llama.cpp/common/build-info.cpp.o: private CPPFLAGS += -iquote llama.cpp/common+# ==============================================================================# Additional support files# ==============================================================================@@ -281,6 +287,8 @@ MTMD_SRCS_CPP := \llama.cpp/tools/mtmd/models/cogvlm.cpp \llama.cpp/tools/mtmd/models/deepseekocr.cpp \llama.cpp/tools/mtmd/models/conformer.cpp \+ llama.cpp/tools/mtmd/models/dotsocr.cpp \+ llama.cpp/tools/mtmd/models/gemma4a.cpp \llama.cpp/tools/mtmd/models/gemma4v.cpp \llama.cpp/tools/mtmd/models/glm4v.cpp \llama.cpp/tools/mtmd/models/hunyuanocr.cpp \@@ -295,10 +303,12 @@ MTMD_SRCS_CPP := \llama.cpp/tools/mtmd/models/paddleocr.cpp \llama.cpp/tools/mtmd/models/pixtral.cpp \llama.cpp/tools/mtmd/models/qwen2vl.cpp \+ llama.cpp/tools/mtmd/models/qwen3a.cpp \llama.cpp/tools/mtmd/models/qwen3vl.cpp \llama.cpp/tools/mtmd/models/siglip.cpp \llama.cpp/tools/mtmd/models/step3vl.cpp \llama.cpp/tools/mtmd/models/whisper-enc.cpp \+ llama.cpp/tools/mtmd/models/yasa2.cpp \llama.cpp/tools/mtmd/models/youtuvl.cppMTMD_OBJS := $(MTMD_SRCS_CPP:%.cpp=o/$(MODE)/%.cpp.o)@@ -342,6 +352,7 @@ TOOL_BENCH_SRCS := llama.cpp/tools/llama-bench/llama-bench.cppTOOL_SERVER_SRCS := \llama.cpp/tools/server/server.cpp \+ llama.cpp/tools/server/server-chat.cpp \llama.cpp/tools/server/server-common.cpp \llama.cpp/tools/server/server-context.cpp \llama.cpp/tools/server/server-http.cpp \diff --git a/llama.cpp.patches/patches/common_arg.cpp.patch b/llama.cpp.patches/patches/common_arg.cpp.patchindex 4626cffa..153f0ac0 100644--- a/llama.cpp.patches/patches/common_arg.cpp.patch+++ b/llama.cpp.patches/patches/common_arg.cpp.patch@@ -1,7 +1,7 @@diff --git a/common/arg.cpp b/common/arg.cpp--- a/llama.cpp/common/arg.cpp+++ b/llama.cpp/common/arg.cpp-@@ -37,6 +37,8 @@+@@ -38,6 +38,8 @@#ifndef __EMSCRIPTEN__#ifdef __linux__#include <linux/limits.h>diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patchindex a09c318f..dfbbcbfd 100644--- a/llama.cpp.patches/patches/common_common.cpp.patch+++ b/llama.cpp.patches/patches/common_common.cpp.patch@@ -1,7 +1,7 @@diff --git a/common/common.cpp b/common/common.cpp--- a/llama.cpp/common/common.cpp+++ b/llama.cpp/common/common.cpp-@@ -970,6 +970,16 @@ std::string fs_get_cache_directory() {+@@ -972,6 +972,16 @@ std::string fs_get_cache_directory() {cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");#elif defined(_WIN32)cache_directory = std::getenv("LOCALAPPDATA");@@ -18,7 +18,7 @@ diff --git a/common/common.cpp b/common/common.cpp#elif defined(__EMSCRIPTEN__)GGML_ABORT("not implemented on this platform");#else-@@ -1146,10 +1156,31 @@ common_init_result::common_init_result(common_params & params) :+@@ -1148,10 +1158,31 @@ common_init_result::common_init_result(common_params & params) :if (params.fit_params) {LOG_INF("%s: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on\n", __func__);@@ -43,7 +43,7 @@ diff --git a/common/common.cpp b/common/common.cpp+ }+ }+- llama_params_fit(params.model.path.c_str(), &mparams, &cparams,+ common_fit_params(params.model.path.c_str(), &mparams, &cparams,params.tensor_split,params.tensor_buft_overrides.data(),- params.fit_params_target.data(),diff --git a/llama.cpp.patches/patches/common_download.cpp.patch b/llama.cpp.patches/patches/common_download.cpp.patchindex ec9a78e8..f50ed915 100644--- a/llama.cpp.patches/patches/common_download.cpp.patch+++ b/llama.cpp.patches/patches/common_download.cpp.patch@@ -1,7 +1,7 @@diff --git a/common/download.cpp b/common/download.cpp--- a/llama.cpp/common/download.cpp+++ b/llama.cpp/common/download.cpp-@@ -25,6 +25,8 @@+@@ -26,6 +26,8 @@#ifndef __EMSCRIPTEN__#ifdef __linux__#include <linux/limits.h>diff --git a/llama.cpp.patches/patches/common_log.cpp.patch b/llama.cpp.patches/patches/common_log.cpp.patchindex 173fa913..b99c8094 100644--- a/llama.cpp.patches/patches/common_log.cpp.patch+++ b/llama.cpp.patches/patches/common_log.cpp.patch@@ -9,7 +9,7 @@ diff --git a/common/log.cpp b/common/log.cpp#endif // defined(_WIN32)int common_log_verbosity_thold = LOG_DEFAULT_LLAMA;-@@ -257,10 +258,27 @@ public:+@@ -261,10 +262,27 @@ public:running = true;thrd = std::thread([this]() {diff --git a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patchindex 68debcbc..d0443c58 100644--- a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch+++ b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch@@ -18,10 +18,10 @@ diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h#ifdef __cplusplusextern "C" {#endif-@@ -197,19 +207,19 @@ extern "C" {- // Common functions that may be obtained using ggml_backend_reg_get_proc_address+@@ -208,19 +218,19 @@ extern "C" {+ typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);- // Split buffer type for tensor parallelism+ // Split buffer type for tensor parallelism (old)- typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);+ typedef ggml_backend_buffer_type_t (GGML_CALL *ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);// Set the number of threads for the backenddiff --git a/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patchindex c083b232..a2b00a01 100644--- a/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch+++ b/llama.cpp.patches/patches/ggml_include_ggml-cuda.h.patch@@ -1,8 +1,8 @@diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h--- a/llama.cpp/ggml/include/ggml-cuda.h+++ b/llama.cpp/ggml/include/ggml-cuda.h-@@ -28,7 +28,7 @@ GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend);- GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);+@@ -31,7 +31,7 @@ GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int de+ GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends);// split tensor buffer that splits matrices by rows across multiple devices-GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split);@@ -10,7 +10,7 @@ diff --git a/ggml/include/ggml-cuda.h b/ggml/include/ggml-cuda.h// pinned host buffer for use with the CPU backend for faster copies between CPU and GPUGGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);-@@ -37,8 +37,8 @@ GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);+@@ -40,8 +40,8 @@ GGML_BACKEND_API int ggml_backend_cuda_get_device_count(void);GGML_BACKEND_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);GGML_BACKEND_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patchindex cf3c05c7..3aaec764 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend_buffer_type {-@@ -39,22 +39,26 @@ extern "C" {+@@ -39,26 +39,30 @@ extern "C" {//struct ggml_backend_buffer_i {@@ -46,6 +46,12 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h+ void (GGML_CALL *memset_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);+ void (GGML_CALL *set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+ void (GGML_CALL *get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+ // (optional) 2d data copies+- void (*set_tensor_2d)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+- void (*get_tensor_2d)(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);++ void (GGML_CALL *set_tensor_2d)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);++ void (GGML_CALL *get_tensor_2d)(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+// (optional) tensor copy: dst is in the buffer, src may be in any buffer, including buffers from a different backend (return false if not supported)- bool (*cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);+ bool (GGML_CALL *cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst);@@ -62,7 +68,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend_buffer {-@@ -85,38 +89,38 @@ extern "C" {+@@ -103,40 +107,40 @@ extern "C" {//struct ggml_backend_i {@@ -73,11 +79,15 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h+ void (GGML_CALL *free)(ggml_backend_t backend);// (optional) asynchronous tensor data access-- void (*set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);-- void (*get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+- void (*set_tensor_async) (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);+- void (*get_tensor_async) (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);+- void (*set_tensor_2d_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+- void (*get_tensor_2d_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);- bool (*cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);-+ void (GGML_CALL *set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);-+ void (GGML_CALL *get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);++ void (GGML_CALL *set_tensor_async) (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);++ void (GGML_CALL *get_tensor_async) (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);++ void (GGML_CALL *set_tensor_2d_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);++ void (GGML_CALL *get_tensor_2d_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data);+ bool (GGML_CALL *cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);// (optional) complete all pending operations (required if the backend supports async operations)@@ -115,7 +125,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend {-@@ -139,46 +143,46 @@ extern "C" {+@@ -159,46 +163,46 @@ extern "C" {// the current functions to obtain the properties can remain, since they are more convenient for often used propertiesstruct ggml_backend_device_i {// device name: short identifier for this device, such as "CPU" or "CUDA0"@@ -177,7 +187,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend_device {-@@ -192,15 +196,15 @@ extern "C" {+@@ -212,15 +216,15 @@ extern "C" {//struct ggml_backend_reg_i {diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patchnew file mode 100644index 00000000..2821ecb5--- /dev/null+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch@@ -0,0 +1,261 @@+diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp+--- a/llama.cpp/ggml/src/ggml-backend-meta.cpp++++ b/llama.cpp/ggml/src/ggml-backend-meta.cpp+@@ -83,19 +83,19 @@ struct ggml_backend_meta_device_context {++ static bool ggml_backend_dev_is_meta(ggml_backend_dev_t dev);++-static const char * ggml_backend_meta_device_get_name(ggml_backend_dev_t dev) {++static const char * GGML_CALL ggml_backend_meta_device_get_name(ggml_backend_dev_t dev) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;+ return meta_dev_ctx->name.c_str();+ }++-static const char * ggml_backend_meta_device_get_description(ggml_backend_dev_t dev) {++static const char * GGML_CALL ggml_backend_meta_device_get_description(ggml_backend_dev_t dev) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;+ return meta_dev_ctx->description.c_str();+ }++-static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {++static void GGML_CALL ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;+ *free = 0;+@@ -108,13 +108,13 @@ static void ggml_backend_meta_device_get_memory(ggml_backend_dev_t dev, size_t *+ }+ }++-static enum ggml_backend_dev_type ggml_backend_meta_device_get_type(ggml_backend_dev_t dev) {++static enum ggml_backend_dev_type GGML_CALL ggml_backend_meta_device_get_type(ggml_backend_dev_t dev) {+ return GGML_BACKEND_DEVICE_TYPE_META;++ GGML_UNUSED(dev);+ }++-static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {++static void GGML_CALL ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;++@@ -142,20 +142,20 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back+ }+ }++-static ggml_backend_t ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params);++static ggml_backend_t GGML_CALL ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params);++-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev);++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev);++-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev);++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev);++-static bool ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {++static bool GGML_CALL ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;+ return std::all_of(meta_dev_ctx->simple_devs.begin(), meta_dev_ctx->simple_devs.end(),+ [op](ggml_backend_dev_t simple_dev) { return ggml_backend_dev_supports_op(simple_dev, op); });+ }++-static bool ggml_backend_meta_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {++static bool GGML_CALL ggml_backend_meta_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ ggml_backend_dev_t dev_buft = ggml_backend_buft_get_device(buft);+ if (!ggml_backend_dev_is_meta(dev_buft)) {+@@ -272,7 +272,7 @@ static size_t ggml_backend_meta_buft_n_bufts(ggml_backend_buffer_type_t meta_buf+ return meta_buft_ctx->simple_bufts.size();+ }++-static const char * ggml_backend_meta_buffer_type_get_name(ggml_backend_buffer_type_t buft) {++static const char * GGML_CALL ggml_backend_meta_buffer_type_get_name(ggml_backend_buffer_type_t buft) {+ GGML_ASSERT(ggml_backend_buft_is_meta(buft));+ const ggml_backend_meta_buffer_type_context * meta_buft_ctx = (const ggml_backend_meta_buffer_type_context *) buft->context;+ return meta_buft_ctx->name.c_str();+@@ -285,9 +285,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen+ return meta_buft_ctx->simple_bufts[index];+ }++-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);++-static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);+ size_t max_alignment = 1;+ for (size_t i = 0; i < n_simple_bufts; i++) {+@@ -298,7 +298,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty+ return max_alignment;+ }++-static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);+ size_t max_size = SIZE_MAX;+ for (size_t i = 0; i < n_simple_bufts; i++) {+@@ -307,7 +307,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ+ return max_size;+ }++-static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {++static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);+ size_t max_alloc_size = 0;+ for (size_t i = 0; i < n_simple_bufts; i++) {+@@ -317,7 +317,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t+ return max_alloc_size;+ }++-static bool ggml_backend_meta_buffer_type_is_host(ggml_backend_buffer_type_t buft) {++static bool GGML_CALL ggml_backend_meta_buffer_type_is_host(ggml_backend_buffer_type_t buft) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);+ for (size_t i = 0; i < n_simple_bufts; i++) {+ if (!ggml_backend_buft_is_host(ggml_backend_meta_buft_simple_buft(buft, i))) {+@@ -340,7 +340,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {+ return buft != nullptr && buft->iface.get_name == ggml_backend_meta_buffer_type_iface.get_name;+ }++-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev) {++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_buffer_type(ggml_backend_dev_t dev) {+ static std::map<ggml_backend_dev_t, struct ggml_backend_buffer_type> meta_bufts;+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ {+@@ -367,7 +367,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_+ return &result.first->second;+ }++-static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev) {++static ggml_backend_buffer_type_t GGML_CALL ggml_backend_meta_device_get_host_buffer_type(ggml_backend_dev_t dev) {+ GGML_ASSERT(ggml_backend_dev_is_meta(dev));+ const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;++@@ -414,7 +414,7 @@ struct ggml_backend_meta_buffer_context {+ }+ };++-static void ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) {++static void GGML_CALL ggml_backend_meta_buffer_free_buffer(ggml_backend_buffer_t buffer) {+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;+ for (auto & [ctx, buf] : buf_ctx->buf_configs) {+@@ -1077,12 +1077,12 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co+ return ret;+ }++-static void * ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) {++static void * GGML_CALL ggml_backend_meta_buffer_get_base(ggml_backend_buffer_t buffer) {+ GGML_UNUSED(buffer);+ return (void *) 0x1000000000000000; // FIXME+ }++-static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {++static enum ggml_status GGML_CALL ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {+ GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));+ ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;+ const size_t n_simple_bufs = ggml_backend_meta_buffer_n_bufs(buffer);+@@ -1197,7 +1197,7 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor(ggml_backend_buffer+ return GGML_STATUS_SUCCESS;+ }++-static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {++static void GGML_CALL ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);+ GGML_ASSERT(ggml_is_contiguous(tensor));++@@ -1304,7 +1304,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg+ }+ }++-static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {++static void GGML_CALL ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);+ GGML_ASSERT(ggml_is_contiguous(tensor));++@@ -1397,14 +1397,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co+ }+ }++-static void ggml_backend_meta_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {++static void GGML_CALL ggml_backend_meta_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {+ const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer);+ for (size_t i = 0; i < n_buffers; i++) {+ ggml_backend_buffer_clear(ggml_backend_meta_buffer_simple_buffer(buffer, i), value);+ }+ }++-static void ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {++static void GGML_CALL ggml_backend_meta_buffer_reset(ggml_backend_buffer_t buffer) {+ const size_t n_buffers = ggml_backend_meta_buffer_n_bufs(buffer);+ for (size_t i = 0; i < n_buffers; i++) {+ ggml_backend_buffer_reset(ggml_backend_meta_buffer_simple_buffer(buffer, i));+@@ -1429,7 +1429,7 @@ bool ggml_backend_buffer_is_meta(ggml_backend_buffer_t buf) {+ return buf != nullptr && buf->iface.free_buffer == ggml_backend_meta_buffer_iface.free_buffer;+ }++-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);++ ggml_init_params params = {+@@ -1567,20 +1567,20 @@ struct ggml_backend_meta_context {+ }+ };++-static const char * ggml_backend_meta_get_name(ggml_backend_t backend) {++static const char * GGML_CALL ggml_backend_meta_get_name(ggml_backend_t backend) {+ GGML_ASSERT(ggml_backend_is_meta(backend));+ const ggml_backend_meta_context * backend_ctx = (const ggml_backend_meta_context *) backend->context;+ return backend_ctx->name.c_str();+ }++-static void ggml_backend_meta_free(ggml_backend_t backend) {++static void GGML_CALL ggml_backend_meta_free(ggml_backend_t backend) {+ GGML_ASSERT(ggml_backend_is_meta(backend));+ ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;+ delete backend_ctx;+ delete backend;+ }++-static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {++static void GGML_CALL ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+ const size_t n_backends = ggml_backend_meta_n_backends(backend);+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(ggml_is_contiguous(tensor));+@@ -1621,7 +1621,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens+ }+ }++-static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {++static void GGML_CALL ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+ const size_t n_backends = ggml_backend_meta_n_backends(backend);+ GGML_ASSERT(offset == 0);+ GGML_ASSERT(ggml_is_contiguous(tensor));+@@ -1662,14 +1662,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm+ }+ }++-static void ggml_backend_meta_synchronize(ggml_backend_t backend) {++static void GGML_CALL ggml_backend_meta_synchronize(ggml_backend_t backend) {+ const size_t n_backends = ggml_backend_meta_n_backends(backend);+ for (size_t i = 0; i < n_backends; i++) {+ ggml_backend_synchronize(ggml_backend_meta_simple_backend(backend, i));+ }+ }++-static enum ggml_status ggml_backend_meta_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {++static enum ggml_status GGML_CALL ggml_backend_meta_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {+ GGML_ASSERT(cgraph->grads == nullptr);+ const size_t n_backends = ggml_backend_meta_n_backends(backend);+ ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;+@@ -2101,7 +2101,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {+ return backend != nullptr && backend->iface.get_name == ggml_backend_meta_i.get_name;+ }++-static ggml_backend_t ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params) {++static ggml_backend_t GGML_CALL ggml_backend_meta_device_init_backend(ggml_backend_dev_t dev, const char * params) {+ ggml_backend_meta_context * backend_ctx = new ggml_backend_meta_context(dev, params);++ ggml_backend_t backend = new struct ggml_backend;diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patchindex 64ec872e..ab59ee37 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp--- a/llama.cpp/ggml/src/ggml-backend-reg.cpp+++ b/llama.cpp/ggml/src/ggml-backend-reg.cpp-@@ -485,7 +485,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,+@@ -497,7 +497,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,if (ec) {GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());} else {diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patchindex fad678f3..ba7bc249 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp}size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {-@@ -584,7 +591,7 @@ struct ggml_backend_multi_buffer_context {+@@ -671,7 +678,7 @@ struct ggml_backend_multi_buffer_context {size_t n_buffers;};@@ -26,7 +26,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;for (size_t i = 0; i < ctx->n_buffers; i++) {-@@ -595,7 +602,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)+@@ -682,7 +689,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)free(ctx);}@@ -35,7 +35,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;for (size_t i = 0; i < ctx->n_buffers; i++) {-@@ -613,6 +620,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {+@@ -702,6 +709,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {/* .cpy_tensor = */ NULL,/* .clear = */ ggml_backend_multi_buffer_clear,/* .reset = */ NULL,@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp};ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {-@@ -2113,7 +2121,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t+@@ -2210,7 +2218,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t// CPU backend - buffer@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);uintptr_t data = (uintptr_t)buffer->context;-@@ -2125,33 +2133,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {+@@ -2222,33 +2230,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {return (void *)data;}@@ -91,7 +91,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(src);if (ggml_backend_buffer_is_host(src->buffer)) {memcpy(dst->data, src->data, ggml_nbytes(src));-@@ -2162,7 +2170,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con+@@ -2259,7 +2267,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, conGGML_UNUSED(buffer);}@@ -100,7 +100,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);memset(buffer->context, value, buffer->size);}-@@ -2177,6 +2185,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {+@@ -2276,6 +2284,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,/* .reset = */ NULL,@@ -108,7 +108,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp};static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {-@@ -2189,19 +2198,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {+@@ -2290,19 +2299,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,/* .reset = */ NULL,@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppvoid * data = ggml_aligned_malloc(size);if (data == NULL) {-@@ -2212,13 +2222,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back+@@ -2313,13 +2323,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backreturn ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);}@@ -147,7 +147,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppreturn true;GGML_UNUSED(buft);-@@ -2241,7 +2251,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {+@@ -2342,7 +2352,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {return &ggml_backend_cpu_buffer_type;}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patchindex d3d2b18a..c42352f1 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_amx_amx.cpp.patch@@ -45,7 +45,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cppmemset(buffer->context, value, buffer->size);}-@@ -116,13 +116,13 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {+@@ -118,13 +118,13 @@ static ggml_backend_buffer_i ggml_backend_amx_buffer_interface = {/* .reset = */ nullptr,};@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cppvoid * data = ggml_aligned_malloc(size);if (data == NULL) {fprintf(stderr, "%s: failed to allocate buffer of size %zu\n", __func__, size);-@@ -132,7 +132,7 @@ static ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_back+@@ -134,7 +134,7 @@ static ggml_backend_buffer_t ggml_backend_amx_buffer_type_alloc_buffer(ggml_backreturn ggml_backend_buffer_init(buft, ggml_backend_amx_buffer_interface, data, size);}@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cppreturn TENSOR_ALIGNMENT;GGML_UNUSED(buft);-@@ -198,7 +198,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {+@@ -200,7 +200,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {};} // namespace ggml::cpu::amxdiff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patchindex 1634f0b1..4e4cf7d1 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch@@ -62,7 +62,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppstruct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);-@@ -248,7 +248,7 @@ bool ggml_backend_is_cpu(ggml_backend_t backend) {+@@ -250,7 +250,7 @@ bool ggml_backend_is_cpu(ggml_backend_t backend) {return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_cpu_guid());}@@ -71,7 +71,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppGGML_ASSERT(ggml_backend_is_cpu(backend_cpu));struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;-@@ -267,7 +267,7 @@ void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpool+@@ -269,7 +269,7 @@ void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_threadpoolctx->threadpool = threadpool;}@@ -80,7 +80,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppGGML_ASSERT(ggml_backend_is_cpu(backend_cpu));struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;-@@ -348,19 +348,19 @@ struct ggml_backend_cpu_device_context {+@@ -350,19 +350,19 @@ struct ggml_backend_cpu_device_context {}};@@ -103,7 +103,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp#ifdef _WIN32MEMORYSTATUSEX status;status.dwLength = sizeof(status);-@@ -379,13 +379,13 @@ static void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t *+@@ -381,13 +381,13 @@ static void ggml_backend_cpu_device_get_memory(ggml_backend_dev_t dev, size_t *GGML_UNUSED(dev);}@@ -119,7 +119,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppprops->name = ggml_backend_cpu_device_get_name(dev);props->description = ggml_backend_cpu_device_get_description(dev);props->type = ggml_backend_cpu_device_get_type(dev);-@@ -398,27 +398,27 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm+@@ -400,27 +400,27 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm};}@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppconst struct ggml_tensor * src0 = op->src[0];const struct ggml_tensor * src1 = op->src[1];-@@ -471,7 +471,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st+@@ -473,7 +473,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st}}@@ -160,7 +160,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppreturn ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);GGML_UNUSED(dev);}-@@ -496,19 +496,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {+@@ -498,19 +498,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {// CPU backend - backend (reg)@@ -183,7 +183,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppGGML_ASSERT(index == 0);static ggml_backend_cpu_device_context ctx;-@@ -523,7 +523,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg+@@ -525,7 +525,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,// and additionally to allow other backends to expose their own list of features that applications can query using the same API@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppstatic std::vector<ggml_backend_feature> features = []() {ggml_cpu_init();-@@ -637,7 +637,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r+@@ -639,7 +639,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t rGGML_UNUSED(reg);}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patchindex 74e0a341..84db581e 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh+++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh-@@ -292,9 +292,15 @@ static bool fp16_mma_hardware_available(const int cc) {+@@ -294,9 +294,15 @@ static bool fp16_mma_hardware_available(const int cc) {}static bool bf16_mma_hardware_available(const int cc) {diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patchindex f187c4bd..bf0329de 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu-@@ -596,7 +596,7 @@ struct ggml_backend_cuda_buffer_context {+@@ -631,7 +631,7 @@ struct ggml_backend_cuda_buffer_context {}};@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;delete ctx;}-@@ -605,12 +605,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {+@@ -640,12 +640,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;}@@ -25,34 +25,52 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;if (tensor->view_src != NULL) {-@@ -631,7 +631,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer+@@ -666,7 +666,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_bufferreturn GGML_STATUS_SUCCESS;}-static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {+static void GGML_CALL ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -639,7 +639,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,+@@ -674,7 +674,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}-static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+static void GGML_CALL ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -647,7 +647,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg+@@ -682,7 +682,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggCUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}-static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+static void GGML_CALL ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -655,7 +655,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co+@@ -690,7 +690,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+ }++-static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data,++static void GGML_CALL ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data,+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;++@@ -700,7 +700,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,+ CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));+ }++-static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data,++static void GGML_CALL ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data,+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {+ ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;++@@ -710,7 +710,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -61,7 +79,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (ggml_backend_buffer_is_cuda(src->buffer)) {ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;-@@ -676,7 +676,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co+@@ -731,7 +731,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, coGGML_UNUSED(buffer);}@@ -70,7 +88,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;ggml_cuda_set_device(ctx->device);-@@ -684,6 +684,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t+@@ -739,6 +739,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_tCUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -85,7 +103,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.custatic const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,/* .get_base = */ ggml_backend_cuda_buffer_get_base,-@@ -694,6 +702,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {+@@ -751,6 +759,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,/* .clear = */ ggml_backend_cuda_buffer_clear,/* .reset = */ NULL,@@ -93,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu};// cuda buffer type-@@ -702,7 +711,7 @@ struct ggml_backend_cuda_buffer_type_context {+@@ -759,7 +768,7 @@ struct ggml_backend_cuda_buffer_type_context {std::string name;};@@ -102,7 +120,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;return ctx->name.c_str();-@@ -712,7 +721,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {+@@ -769,7 +778,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;}@@ -111,7 +129,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;ggml_cuda_set_device(buft_ctx->device);-@@ -731,13 +740,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac+@@ -788,13 +797,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bacreturn ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);}@@ -127,7 +145,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cusize_t size = ggml_nbytes(tensor);int64_t ne0 = tensor->ne[0];-@@ -851,19 +860,19 @@ struct ggml_backend_cuda_split_buffer_context {+@@ -908,19 +917,19 @@ struct ggml_backend_cuda_split_buffer_context {};@@ -150,7 +168,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuGGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supportedGGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");-@@ -913,7 +922,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_+@@ -970,7 +979,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_return GGML_STATUS_SUCCESS;}@@ -159,7 +177,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu// split tensors must always be set in their entirety at onceGGML_ASSERT(offset == 0);GGML_ASSERT(size == ggml_nbytes(tensor));-@@ -952,7 +961,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff+@@ -1009,7 +1018,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff}}@@ -168,7 +186,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu// split tensors must always be set in their entirety at onceGGML_ASSERT(offset == 0);GGML_ASSERT(size == ggml_nbytes(tensor));-@@ -991,7 +1000,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff+@@ -1048,7 +1057,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff}}@@ -177,7 +195,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuGGML_UNUSED(buffer);GGML_UNUSED(value);}-@@ -1006,11 +1015,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {+@@ -1065,11 +1074,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {/* .cpy_tensor = */ NULL,/* .clear = */ ggml_backend_cuda_split_buffer_clear,/* .reset = */ NULL,@@ -191,7 +209,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;return ctx->name.c_str();-@@ -1020,7 +1030,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {+@@ -1079,7 +1089,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name;}@@ -200,7 +218,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point// instead, we allocate them for each tensor separately in init_tensor// however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,-@@ -1030,13 +1040,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg+@@ -1089,13 +1099,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(ggreturn ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);}@@ -216,7 +234,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");-@@ -1064,7 +1074,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu+@@ -1123,7 +1133,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bureturn total_size;}@@ -225,16 +243,16 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cureturn false;GGML_UNUSED(buft);-@@ -1079,7 +1089,7 @@ static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_inte- /* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,- };+@@ -1280,7 +1290,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg+ #endif // GGML_USE_NCCL+ }-ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {+ggml_backend_buffer_type_t GGML_CALL ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split) {static std::mutex mutex;std::lock_guard<std::mutex> lock(mutex);-@@ -1123,7 +1133,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,+@@ -1324,7 +1334,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,// host buffer type@@ -243,7 +261,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cureturn GGML_CUDA_NAME "_Host";GGML_UNUSED(buft);-@@ -1133,7 +1143,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {+@@ -1334,7 +1344,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;}@@ -252,7 +270,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuCUDA_CHECK(cudaFreeHost(buffer->context));}-@@ -1155,17 +1165,25 @@ static void * ggml_cuda_host_malloc(size_t size) {+@@ -1356,17 +1366,25 @@ static void * ggml_cuda_host_malloc(size_t size) {return ptr;}@@ -280,7 +298,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cureturn buffer;}-@@ -1293,8 +1311,13 @@ static void ggml_cuda_op_mul_mat_cublas(+@@ -1494,8 +1512,13 @@ static void ggml_cuda_op_mul_mat_cublas(const int cc = ggml_cuda_info().devices[id].cc;@@ -294,7 +312,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuconst bool use_fp16 =src0->type != GGML_TYPE_NVFP4 &&-@@ -2831,20 +2854,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg+@@ -2969,20 +2992,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg// backend@@ -315,20 +333,38 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu-static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {+static void GGML_CALL ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2853,7 +2876,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens- CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));+@@ -2991,7 +3014,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens+ CUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));}-static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {+static void GGML_CALL ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {- ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2862,7 +2885,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm- CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));+@@ -3000,7 +3023,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm+ CUDA_CHECK(cudaMemcpyAsync(data, (const char *) tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));+ }++-static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data,++static void GGML_CALL ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data,+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;+@@ -3011,7 +3034,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct+ (char *) tensor->data + offset, stride_tensor, data, stride_data, size, n_copies, cudaMemcpyHostToDevice, cuda_ctx->stream()));+ }++-static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data,++static void GGML_CALL ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data,+ size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {+ ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;+ ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;+@@ -3022,7 +3045,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const+ data, stride_data, (const char *) tensor->data + offset, stride_tensor, size, n_copies, cudaMemcpyDeviceToHost, cuda_ctx->stream()));}-static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {@@ -336,7 +372,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;-@@ -2917,7 +2940,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_+@@ -3077,7 +3100,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_return true;}@@ -345,7 +381,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));-@@ -4010,7 +4033,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co+@@ -4215,7 +4238,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co}#endif // USE_CUDA_GRAPH@@ -354,7 +390,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_cuda_set_device(cuda_ctx->device);-@@ -4069,13 +4092,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,+@@ -4274,13 +4297,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,return GGML_STATUS_SUCCESS;}@@ -370,7 +406,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;if (ggml_backend_is_cuda(backend)) {-@@ -4094,7 +4117,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev+@@ -4299,7 +4322,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev}}@@ -379,7 +415,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;#ifdef USE_CUDA_GRAPH-@@ -4380,7 +4403,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot+@@ -4587,7 +4610,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * totCUDA_CHECK(cudaMemGetInfo(free, total));}@@ -388,7 +424,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {return false;}-@@ -4403,7 +4426,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {+@@ -4610,7 +4633,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)}@@ -397,7 +433,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {return;}-@@ -4426,12 +4449,12 @@ struct ggml_backend_cuda_device_context {+@@ -4633,12 +4656,12 @@ struct ggml_backend_cuda_device_context {int op_offload_min_batch_size;};@@ -412,7 +448,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;return ctx->description.c_str();}-@@ -4512,7 +4535,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k+@@ -4719,7 +4742,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k}#endif // defined(__linux__)@@ -421,7 +457,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;ggml_cuda_set_device(ctx->device);CUDA_CHECK(cudaMemGetInfo(free, total));-@@ -4542,12 +4565,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *+@@ -4749,12 +4772,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *}@@ -436,7 +472,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;props->name = ggml_backend_cuda_device_get_name(dev);-@@ -4571,24 +4594,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back+@@ -4778,24 +4801,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back};}@@ -465,7 +501,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;// split buffers can only be used with GGML_OP_MUL_MAT-@@ -4970,7 +4993,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -5179,7 +5202,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g}}@@ -474,7 +510,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)));-@@ -4991,13 +5014,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {+@@ -5200,13 +5223,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {}}@@ -490,7 +526,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu#ifdef GGML_CUDA_NO_PEER_COPYreturn nullptr;#else-@@ -5015,14 +5038,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_+@@ -5224,14 +5247,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_#endif}@@ -507,7 +543,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuGGML_UNUSED(dev);CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));}-@@ -5051,23 +5074,23 @@ struct ggml_backend_cuda_reg_context {+@@ -5260,23 +5283,23 @@ struct ggml_backend_cuda_reg_context {std::vector<ggml_backend_dev_t> devices;};@@ -535,12 +571,12 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.custatic std::vector<ggml_backend_feature> features = []() {std::vector<ggml_backend_feature> features;#define _STRINGIFY(...) #__VA_ARGS__-@@ -5128,7 +5151,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t+@@ -5337,7 +5360,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_tGGML_UNUSED(reg);}-static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {+static void * GGML_CALL ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, const char * name) {GGML_UNUSED(reg);- if (strcmp(name, "ggml_backend_split_buffer_type") == 0) {- return (void *)ggml_backend_cuda_split_buffer_type;+ if (strcmp(name, "ggml_backend_comm_init") == 0) {+ return (void *)ggml_backend_cuda_comm_init;diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patchindex 3b48ffe6..95be7ca5 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda#include <cuda_bf16.h>#include <cuda_fp16.h>-@@ -17,8 +27,10 @@+@@ -21,8 +31,10 @@#if CUDART_VERSION < 11020#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTEDdiff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patchindex f99567d3..1c1f68cb 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch@@ -79,7 +79,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalstatic ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {/* .free_buffer = */ ggml_backend_metal_buffer_shared_free_buffer,/* .get_base = */ ggml_backend_metal_buffer_shared_get_base,-@@ -93,11 +101,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {+@@ -95,11 +103,12 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_shared_i = {/* .cpy_tensor = */ ggml_backend_metal_buffer_shared_cpy_tensor,/* .clear = */ ggml_backend_metal_buffer_shared_clear,/* .reset = */ NULL,@@ -93,7 +93,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -105,7 +114,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t+@@ -107,7 +116,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_tggml_metal_buffer_free(ctx);}@@ -102,7 +102,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -113,7 +122,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b+@@ -115,7 +124,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t breturn ggml_metal_buffer_get_base(ctx);}@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -121,7 +130,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_+@@ -123,7 +132,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);}@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -129,7 +138,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b+@@ -131,7 +140,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t bggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);}@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -137,7 +146,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b+@@ -139,7 +148,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t bggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);}@@ -138,7 +138,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -149,7 +158,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b+@@ -151,7 +160,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t breturn false;}@@ -147,15 +147,15 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));-@@ -167,6 +176,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {- /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,- /* .clear = */ ggml_backend_metal_buffer_private_clear,- /* .reset = */ NULL,-+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,+@@ -171,6 +180,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {+ /* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,+ /* .clear = */ ggml_backend_metal_buffer_private_clear,+ /* .reset = */ NULL,++ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,};static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {-@@ -239,33 +249,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_+@@ -243,33 +253,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_// default (shared) buffer type@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -315,33 +325,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi+@@ -319,33 +329,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi// default (private) buffer type@@ -235,7 +235,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -390,34 +400,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev+@@ -394,34 +404,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev// mapped buffer type@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -468,13 +478,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi+@@ -472,13 +482,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi// backend@@ -292,7 +292,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_t ctx = (ggml_metal_t)backend->context;// wait for any ongoing async operations to finish-@@ -485,25 +495,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {+@@ -489,25 +499,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {free(backend);}@@ -322,7 +322,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalif (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {return false;}-@@ -524,27 +534,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml+@@ -528,27 +538,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggmlreturn ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);}@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_t ctx = (ggml_metal_t)backend->context;ggml_metal_graph_optimize(ctx, cgraph);-@@ -634,7 +644,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {+@@ -640,7 +650,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {// backend device@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);-@@ -642,25 +652,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {+@@ -648,25 +658,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {return props_dev->name;}@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalprops->name = ggml_backend_metal_device_get_name(dev);props->description = ggml_backend_metal_device_get_description(dev);props->type = ggml_backend_metal_device_get_type(dev);-@@ -675,7 +685,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac+@@ -681,7 +691,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac};}@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_t ctx = ggml_metal_init(ctx_dev);-@@ -700,7 +710,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t+@@ -706,7 +716,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_tGGML_UNUSED(params);}@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);-@@ -708,7 +718,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml+@@ -714,7 +724,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggmlreturn props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);}@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);-@@ -718,13 +728,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen+@@ -724,13 +734,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backenreturn ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);}@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturnbuft->device == dev && (buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||-@@ -745,7 +755,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {+@@ -751,7 +761,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {}}@@ -445,7 +445,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;return (op->op == GGML_OP_MUL_MAT ||-@@ -753,7 +763,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g+@@ -759,7 +769,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const gget_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;}@@ -454,7 +454,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);-@@ -767,7 +777,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev+@@ -773,7 +783,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_devreturn ev;}@@ -463,7 +463,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t ev = (ggml_metal_event_t)event->context;-@@ -777,7 +787,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba+@@ -783,7 +793,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_badelete event;}@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t evt = (ggml_metal_event_t)event->context;-@@ -829,18 +839,18 @@ struct ggml_backend_metal_reg_deleter {+@@ -835,18 +845,18 @@ struct ggml_backend_metal_reg_deleter {typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;@@ -494,7 +494,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;GGML_ASSERT(index < ctx->devices.size());return ctx->devices[index];-@@ -853,13 +863,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {+@@ -859,13 +869,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {{ NULL, NULL },};diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patchindex 81b8e2ce..53a01fbc 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp-@@ -238,11 +238,11 @@ struct vk_queue {+@@ -252,11 +252,11 @@ struct vk_queue {}};@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vustatic ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {/* .get_name = */ ggml_backend_vk_buffer_type_name,/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,-@@ -2074,7 +2074,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *+@@ -2089,7 +2089,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);@@ -27,7 +27,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vustatic VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},-@@ -3391,20 +3391,15 @@ static void ggml_vk_load_shaders(vk_device& device) {+@@ -3469,20 +3469,15 @@ static void ggml_vk_load_shaders(vk_device& device) {if (!pipeline->needed || pipeline->compiled) {continue;}@@ -54,7 +54,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu}};-@@ -13428,20 +13423,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {+@@ -13569,20 +13564,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;}@@ -86,7 +86,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");if (tensor->view_src != nullptr) {GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);-@@ -13449,7 +13452,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t+@@ -13590,7 +13593,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_treturn GGML_STATUS_SUCCESS;}@@ -95,7 +95,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;-@@ -13462,7 +13465,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g+@@ -13603,7 +13606,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, gggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);}@@ -104,7 +104,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;-@@ -13474,7 +13477,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml+@@ -13615,7 +13618,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggmlggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}@@ -113,7 +113,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;-@@ -13487,7 +13490,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons+@@ -13628,7 +13631,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, consggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}@@ -122,7 +122,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuif (ggml_nbytes(src) == 0) {return true;}-@@ -13508,7 +13511,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons+@@ -13649,7 +13652,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, consUNUSED(buffer);}@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);-@@ -13524,16 +13527,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {+@@ -13667,16 +13670,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,/* .clear = */ ggml_backend_vk_buffer_clear,/* .reset = */ NULL,@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;-@@ -13549,17 +13553,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe+@@ -13692,17 +13696,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backereturn ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);}@@ -172,7 +172,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vureturn ggml_nbytes(tensor);UNUSED(buft);-@@ -13577,24 +13581,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {+@@ -13720,24 +13724,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {// host buffer type@@ -201,7 +201,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");size += 32; // Behave like the CPU buffer type-@@ -13610,19 +13614,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_+@@ -13753,19 +13757,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);buffer->buft = buft;buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;@@ -224,7 +224,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vureturn vk_instance.devices[0]->suballocation_block_size;UNUSED(buft);-@@ -13654,13 +13659,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {+@@ -13797,13 +13802,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {// backend@@ -240,7 +240,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");-@@ -13676,7 +13681,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b+@@ -13819,7 +13824,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_breturn &ctx->device->buffer_type;}@@ -249,7 +249,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_set_tensor_async(" << size << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");-@@ -13723,7 +13728,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor+@@ -13866,7 +13871,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor}}@@ -258,7 +258,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_get_tensor_async(" << size << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");-@@ -13757,7 +13762,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_+@@ -13900,7 +13905,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_}}@@ -267,7 +267,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;-@@ -13882,7 +13887,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {+@@ -14025,7 +14030,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {}}@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_synchronize()");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -14304,7 +14309,7 @@ static int32_t find_first_set(uint32_t x) {+@@ -14446,7 +14451,7 @@ static int32_t find_first_set(uint32_t x) {return ret;}@@ -285,7 +285,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -14684,7 +14689,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg+@@ -14828,7 +14833,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg}// Sort the graph for improved parallelism.@@ -294,7 +294,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu{VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -14922,7 +14927,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *+@@ -15066,7 +15071,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *}}@@ -303,7 +303,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;vk_event *vkev = (vk_event *)event->context;-@@ -14960,7 +14965,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev+@@ -15104,7 +15109,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_evctx->compute_ctx.reset();}@@ -312,7 +312,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;vk_event *vkev = (vk_event *)event->context;-@@ -15055,7 +15060,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total+@@ -15201,7 +15206,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total*total += heap.size;if (membudget_supported && i < budgetprops.heapUsage.size()) {@@ -324,7 +324,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu} else {*free += heap.size;}-@@ -15123,38 +15131,38 @@ struct ggml_backend_vk_device_context {+@@ -15269,38 +15277,38 @@ struct ggml_backend_vk_device_context {int op_offload_min_batch_size;};@@ -370,7 +370,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;props->name = ggml_backend_vk_device_get_name(dev);-@@ -15170,13 +15178,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml+@@ -15316,13 +15324,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml};}@@ -386,7 +386,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;const vk_device& device = ggml_vk_get_device(ctx->device);-@@ -15714,7 +15722,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm+@@ -15868,7 +15876,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmUNUSED(dev);}@@ -395,7 +395,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuif (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {return false;}-@@ -15740,13 +15748,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {+@@ -15894,13 +15902,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {}}@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15769,7 +15777,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t+@@ -15923,7 +15931,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t};}@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15789,7 +15797,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe+@@ -15943,7 +15951,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backedelete event;}@@ -429,7 +429,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15846,7 +15854,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si+@@ -16000,7 +16008,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, sireturn buf;}@@ -438,7 +438,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");GGML_UNUSED(max_tensor_size);-@@ -15884,17 +15892,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {+@@ -16038,17 +16046,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,};diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patchindex 0900d663..52d68c2b 100644--- a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch@@ -1,7 +1,7 @@diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp--- a/llama.cpp/tools/server/server-models.cpp+++ b/llama.cpp/tools/server/server-models.cpp-@@ -27,6 +27,7 @@+@@ -30,6 +30,7 @@#include <netinet/in.h>#include <arpa/inet.h>#include <unistd.h>@@ -9,7 +9,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppextern char **environ;#endif-@@ -519,9 +520,12 @@ void server_models::unload_lru() {+@@ -523,9 +524,12 @@ void server_models::unload_lru() {// wait for unload to complete{std::unique_lock<std::mutex> lk(mutex);@@ -25,7 +25,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp}}}-@@ -623,6 +627,14 @@ void server_models::load(const std::string & name) {+@@ -627,6 +631,14 @@ void server_models::load(const std::string & name) {});std::thread stopping_thread([&]() {@@ -40,7 +40,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp// thread to monitor stopping signal OR child crashauto is_stopping = [this, &name]() {return this->stopping_models.find(name) != this->stopping_models.end();-@@ -632,7 +644,10 @@ void server_models::load(const std::string & name) {+@@ -636,7 +648,10 @@ void server_models::load(const std::string & name) {};{std::unique_lock<std::mutex> lk(this->mutex);@@ -52,7 +52,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp}// child may have already exited (e.g. crashed) — skip shutdown sequenceif (!subprocess_alive(child_proc.get())) {-@@ -753,13 +768,14 @@ void server_models::update_status(const std::string & name, server_model_status+@@ -762,13 +777,14 @@ void server_models::update_status(const std::string & name, server_model_statusvoid server_models::wait_until_loading_finished(const std::string & name) {std::unique_lock<std::mutex> lk(mutex);diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patchindex 54cabb6e..8a48a3ff 100644--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch@@ -1,7 +1,7 @@diff --git a/tools/server/server.cpp b/tools/server/server.cpp--- a/llama.cpp/tools/server/server.cpp+++ b/llama.cpp/tools/server/server.cpp-@@ -12,6 +12,7 @@+@@ -14,6 +14,7 @@#include <atomic>#include <clocale>#include <exception>@@ -9,7 +9,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp#include <signal.h>#include <thread> // for std::thread::hardware_concurrency-@@ -19,6 +20,11 @@+@@ -21,6 +22,11 @@#include <windows.h>#endif@@ -21,7 +21,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cppstatic std::function<void(int)> shutdown_handler;static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;-@@ -69,12 +75,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t+@@ -71,12 +77,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t};}@@ -45,7 +45,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cppcommon_init();if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {-@@ -293,6 +309,13 @@ int main(int argc, char ** argv) {+@@ -295,6 +311,13 @@ int main(int argc, char ** argv) {// this will unblock start_loop()ctx_server.terminate();};@@ -59,7 +59,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp}// TODO: refactor in common/console-@@ -324,6 +347,11 @@ int main(int argc, char ** argv) {+@@ -326,6 +349,11 @@ int main(int argc, char ** argv) {LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());LOG_INF("%s: starting the main loop...\n", __func__);@@ -71,7 +71,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp// optionally, notify router server that this instance is readystd::thread monitor_thread;if (server_models::is_child_server()) {-@@ -347,5 +375,48 @@ int main(int argc, char ** argv) {+@@ -349,5 +377,48 @@ int main(int argc, char ** argv) {}}diff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patchindex 82a8f41e..8c350166 100644--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch@@ -1,7 +1,7 @@diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp-@@ -6214,6 +6214,15 @@ void ThreadPool::cleanup_finished_threads() {+@@ -6257,6 +6257,15 @@ void ThreadPool::cleanup_finished_threads() {}void ThreadPool::worker(bool is_dynamic) {@@ -17,7 +17,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cppfor (;;) {std::function<void()> fn;{-@@ -6231,7 +6240,13 @@ void ThreadPool::worker(bool is_dynamic) {+@@ -6274,7 +6283,13 @@ void ThreadPool::worker(bool is_dynamic) {break;}} else {diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex de077711..2aef7863 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -219,6 +219,7 @@ LLAMAFILE_HIGHLIGHT_KEYWORDS := $(LLAMAFILE_HIGHLIGHT_GPERF_FILES:%.gperf=o/$(MO# Server objects for llamafileLLAMAFILE_SERVER_SUPPORT_OBJS := \+ o/$(MODE)/llama.cpp/tools/server/server-chat.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-common.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-context.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-http.cpp.o \@@ -233,6 +234,7 @@ LLAMAFILE_METAL_SOURCES := \o/$(MODE)/llama.cpp/ggml/src/ggml.c.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-alloc.c.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-backend.cpp.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-backend-meta.cpp.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-quants.c.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-threading.cpp.zip.o \o/$(MODE)/llama.cpp/ggml/include/ggml.h.zip.o \@@ -240,6 +242,7 @@ LLAMAFILE_METAL_SOURCES := \o/$(MODE)/llama.cpp/ggml/include/ggml-cpu.h.zip.o \o/$(MODE)/llama.cpp/ggml/include/ggml-alloc.h.zip.o \o/$(MODE)/llama.cpp/ggml/include/ggml-backend.h.zip.o \+ o/$(MODE)/llama.cpp/ggml/include/ggml-cpp.h.zip.o \o/$(MODE)/llama.cpp/ggml/include/ggml-metal.h.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-impl.h.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-common.h.zip.o \diff --git a/llamafile/build-functions.sh b/llamafile/build-functions.shindex 2d82d99f..3f81934d 100644--- a/llamafile/build-functions.sh+++ b/llamafile/build-functions.sh@@ -175,6 +175,7 @@ compile_ggml_core() {$llama_cpp_dir/ggml/src/ggml.c \$llama_cpp_dir/ggml/src/ggml-alloc.c \$llama_cpp_dir/ggml/src/ggml-backend.cpp \+ $llama_cpp_dir/ggml/src/ggml-backend-meta.cpp \$llama_cpp_dir/ggml/src/ggml-quants.c \$llama_cpp_dir/ggml/src/ggml-threading.cpp"diff --git a/llamafile/cuda.bat b/llamafile/cuda.batindex 7168d975..ad0b428d 100644--- a/llamafile/cuda.bat+++ b/llamafile/cuda.bat@@ -249,7 +249,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do ():: C++ sources-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (set "SRC=%GGML_SRC_DIR%\%%f"set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"diff --git a/llamafile/cuda_parallel.bat b/llamafile/cuda_parallel.batindex 50ed8493..65c63aec 100644--- a/llamafile/cuda_parallel.bat+++ b/llamafile/cuda_parallel.bat@@ -268,7 +268,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do ():: C++ sources-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (set "SRC=%GGML_SRC_DIR%\%%f"set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"diff --git a/llamafile/metal.c b/llamafile/metal.cindex 778ef1b0..0f1a7187 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -54,6 +54,7 @@ extern char **environ;__static_yoink("llama.cpp/ggml/src/ggml.c");__static_yoink("llama.cpp/ggml/src/ggml-alloc.c");__static_yoink("llama.cpp/ggml/src/ggml-backend.cpp");+__static_yoink("llama.cpp/ggml/src/ggml-backend-meta.cpp");__static_yoink("llama.cpp/ggml/src/ggml-quants.c");__static_yoink("llama.cpp/ggml/src/ggml-threading.cpp");@@ -63,6 +64,7 @@ __static_yoink("llama.cpp/ggml/include/gguf.h");__static_yoink("llama.cpp/ggml/include/ggml-cpu.h");__static_yoink("llama.cpp/ggml/include/ggml-alloc.h");__static_yoink("llama.cpp/ggml/include/ggml-backend.h");+__static_yoink("llama.cpp/ggml/include/ggml-cpp.h");__static_yoink("llama.cpp/ggml/include/ggml-metal.h");__static_yoink("llama.cpp/ggml/src/ggml-impl.h");__static_yoink("llama.cpp/ggml/src/ggml-common.h");@@ -96,6 +98,7 @@ static const struct MetalSource {{"/zip/llama.cpp/ggml/include/ggml-cpu.h", "ggml-cpu.h"},{"/zip/llama.cpp/ggml/include/ggml-alloc.h", "ggml-alloc.h"},{"/zip/llama.cpp/ggml/include/ggml-backend.h", "ggml-backend.h"},+ {"/zip/llama.cpp/ggml/include/ggml-cpp.h", "ggml-cpp.h"},{"/zip/llama.cpp/ggml/include/ggml-metal.h", "ggml-metal.h"},{"/zip/llama.cpp/ggml/src/ggml-impl.h", "ggml-impl.h"},{"/zip/llama.cpp/ggml/src/ggml-common.h", "ggml-common.h"},@@ -108,6 +111,7 @@ static const struct MetalSource {{"/zip/llama.cpp/ggml/src/ggml.c", "ggml.c"},{"/zip/llama.cpp/ggml/src/ggml-alloc.c", "ggml-alloc.c"},{"/zip/llama.cpp/ggml/src/ggml-backend.cpp", "ggml-backend.cpp"},+ {"/zip/llama.cpp/ggml/src/ggml-backend-meta.cpp", "ggml-backend-meta.cpp"},{"/zip/llama.cpp/ggml/src/ggml-quants.c", "ggml-quants.c"},{"/zip/llama.cpp/ggml/src/ggml-threading.cpp", "ggml-threading.cpp"},@@ -379,6 +383,7 @@ static bool BuildMetal(const char *dso) {"ggml-alloc.c","ggml-quants.c","ggml-backend.cpp",+ "ggml-backend-meta.cpp","ggml-threading.cpp","ggml-metal.cpp","ggml-metal-device.cpp",diff --git a/llamafile/rocm.bat b/llamafile/rocm.batindex c136a82a..e4914d09 100644--- a/llamafile/rocm.bat+++ b/llamafile/rocm.bat@@ -235,7 +235,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do ():: C++ sources-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (set "SRC=%GGML_SRC_DIR%\%%f"set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"diff --git a/llamafile/rocm_parallel.bat b/llamafile/rocm_parallel.batindex ee1c441c..36ce7684 100644--- a/llamafile/rocm_parallel.bat+++ b/llamafile/rocm_parallel.bat@@ -254,7 +254,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do ():: C++ sources-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (set "SRC=%GGML_SRC_DIR%\%%f"set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"diff --git a/llamafile/vulkan.bat b/llamafile/vulkan.batindex 4cb9d19d..53f941ad 100644--- a/llamafile/vulkan.bat+++ b/llamafile/vulkan.bat@@ -292,7 +292,7 @@ for %%f in (ggml.c ggml-alloc.c ggml-quants.c) do ():: C++ sources-for %%f in (ggml-backend.cpp ggml-threading.cpp) do (+for %%f in (ggml-backend.cpp ggml-backend-meta.cpp ggml-threading.cpp) do (set "SRC=%GGML_SRC_DIR%\%%f"set "BASE=%%~nf"set "OBJ=%BUILD_DIR%\ggml-core-!BASE!.obj"diff --git a/llamafile/vulkan.sh b/llamafile/vulkan.shindex a988972a..f1824a1f 100755--- a/llamafile/vulkan.sh+++ b/llamafile/vulkan.sh@@ -69,15 +69,44 @@ if [ -z "$GLSLC" ]; thenGLSLC="/usr/bin/glslc"elseecho "Error: glslc not found. Please install the Vulkan SDK."- echo " Linux (Ubuntu/Debian): sudo apt install vulkan-sdk"- echo " Linux (Fedora): sudo dnf install vulkan-tools shaderc"- echo " Linux (Arch): sudo pacman -S vulkan-devel shaderc"+ echo " Linux (Ubuntu/Debian): sudo apt install vulkan-sdk spirv-headers"+ echo " Linux (Fedora): sudo dnf install vulkan-tools shaderc spirv-headers-devel"+ echo " Linux (Arch): sudo pacman -S vulkan-devel shaderc spirv-headers"echo " macOS: brew install vulkan-sdk"echo " Or set VULKAN_SDK environment variable"exit 1fifi+# Find SPIR-V headers (required by ggml-vulkan.cpp since llama.cpp PR #21572).+# Probe the same cascade as the C++ source, plus VULKAN_SDK. For each hit,+# strip the matched relative path off to get the -I root.+SPIRV_INCLUDE=""+probe() {+ local rel="$1" base+ for base in \+ "$VULKAN_SDK/include" \+ /usr/include \+ /usr/local/include+ do+ [ -z "$base" ] && continue+ if [ -f "$base/$rel" ]; then+ SPIRV_INCLUDE="-I$base"+ return 0+ fi+ done+ return 1+}+probe spirv/unified1/spirv.hpp || probe spirv-headers/spirv.hpp || probe spirv.hpp || true+if [ -z "$SPIRV_INCLUDE" ]; then+ echo "Error: SPIR-V headers not found (spirv.hpp). Required by ggml-vulkan.cpp."+ echo " Linux (Ubuntu/Debian): sudo apt install spirv-headers"+ echo " Linux (Fedora): sudo dnf install spirv-headers-devel"+ echo " Linux (Arch): sudo pacman -S spirv-headers"+ echo " Or install the LunarG Vulkan SDK and set VULKAN_SDK."+ exit 1+fi+# Directory setupLLAMAFILE_DIR="$SCRIPT_DIR"LLAMA_CPP_DIR="$SCRIPT_DIR/../llama.cpp"@@ -249,6 +278,7 @@ if [ ! -f "$VULKAN_OBJ" ] || [ "$VULKAN_SRC" -nt "$VULKAN_OBJ" ] || [ "$SHADERS_echo " Compiling ggml-vulkan.cpp..."$CXX -c $CXX_FLAGS \-I$GGML_VULKAN_DIR \+ $SPIRV_INCLUDE \-o "$VULKAN_OBJ" "$VULKAN_SRC"elseecho " ggml-vulkan.o is up to date"