Skip to content

Distribute and run LLMs with a single file.

Update llama.cpp submodule to 5e9c63546 (#941)

Davide Eynardea67feae87a2
diff --git a/llama.cpp b/llama.cpp
index 7f5ee549..5e9c6354 160000
--- a/llama.cpp
+++ b/llama.cpp
@@ -1 +1 @@
-Subproject commit 7f5ee549683d600ad41db6a295a232cdd2d8eb9f
+Subproject commit 5e9c6354638858f4c9aa0a70fd4a41a73f894dcc
diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.md
index bfbc1baa..3be219ce 100644
--- a/llama.cpp.patches/README.md
+++ b/llama.cpp.patches/README.md
@@ -83,6 +83,7 @@ Cosmopolitan libc has specific behaviors with condition variables and signals th
| Patch | Description |
|-------|-------------|
| `common_log.cpp.patch` | Adds `#include <csignal>`; blocks `SIGINT`/`SIGTERM` on logger thread via `pthread_sigmask` to prevent `EINTR` exceptions; replaces `cv.wait()` with `wait_for(30s)` loop to work around XNU futex timeout bug (~72 minute expiry) |
+| `tools_server_server-models.cpp.patch` | Adds `#include <csignal>`; blocks `SIGINT`/`SIGTERM` on stopping thread; replaces `cv.wait()` with `wait_for(30s)` loops in `unload_lru`, `stopping_thread`, and `wait_until_loading_finished` |
| `tools_server_server-queue.cpp.patch` | Adds missing includes (`<cerrno>`, `<system_error>`, `<csignal>`); blocks `SIGINT`/`SIGTERM` on queue thread; replaces `wait()` with `wait_for()` loops in three locations (`wait_until_no_sleep`, main loop, `recv`) |
| `vendor_cpp-httplib_httplib.cpp.patch` | Fixes httplib thread pool with `wait_for()` instead of `wait()` for XNU futex compatibility |
@@ -116,7 +117,6 @@ These patches integrate llamafile's file handling APIs for loading models from b
| Patch | Description |
|-------|-------------|
-| `common_chat.cpp.patch` | Fixes C++ type conversion: explicitly wraps `inputs.messages` in `std::optional<json>()` for Deepseek v3.1 template |
| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach) |
| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Fixes unsigned integer underflow in `ggml_backend_vk_get_device_memory` where Vulkan's `heapUsage` can exceed `heapBudget` (clamps to zero instead of wrapping) |
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index 216e745e..fd6fbdda 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -88,6 +88,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/gemma2-iswa.cpp \
llama.cpp/src/models/gemma3.cpp \
llama.cpp/src/models/gemma3n-iswa.cpp \
+ llama.cpp/src/models/gemma4-iswa.cpp \
llama.cpp/src/models/glm4-moe.cpp \
llama.cpp/src/models/glm4.cpp \
llama.cpp/src/models/gpt2.cpp \
@@ -198,14 +199,16 @@ LLAMA_OBJS := $(LLAMA_SRCS_CPP:%.cpp=o/$(MODE)/%.cpp.o)
COMMON_SRCS_CPP := \
llama.cpp/common/arg.cpp \
- llama.cpp/common/chat-parser-xml-toolcall.cpp \
- llama.cpp/common/chat-parser.cpp \
+ llama.cpp/common/chat-auto-parser-generator.cpp \
+ llama.cpp/common/chat-auto-parser-helpers.cpp \
+ llama.cpp/common/chat-diff-analyzer.cpp \
llama.cpp/common/chat-peg-parser.cpp \
llama.cpp/common/chat.cpp \
llama.cpp/common/common.cpp \
llama.cpp/common/console.cpp \
llama.cpp/common/debug.cpp \
llama.cpp/common/download.cpp \
+ llama.cpp/common/hf-cache.cpp \
llama.cpp/common/jinja/caps.cpp \
llama.cpp/common/jinja/lexer.cpp \
llama.cpp/common/jinja/parser.cpp \
@@ -222,6 +225,7 @@ COMMON_SRCS_CPP := \
llama.cpp/common/ngram-mod.cpp \
llama.cpp/common/peg-parser.cpp \
llama.cpp/common/preset.cpp \
+ llama.cpp/common/reasoning-budget.cpp \
llama.cpp/common/regex-partial.cpp \
llama.cpp/common/sampling.cpp \
llama.cpp/common/speculative.cpp \
@@ -273,9 +277,13 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/mtmd.cpp \
llama.cpp/tools/mtmd/mtmd-helper.cpp \
llama.cpp/tools/mtmd/mtmd-audio.cpp \
+ llama.cpp/tools/mtmd/mtmd-image.cpp \
llama.cpp/tools/mtmd/models/cogvlm.cpp \
+ llama.cpp/tools/mtmd/models/deepseekocr.cpp \
llama.cpp/tools/mtmd/models/conformer.cpp \
+ llama.cpp/tools/mtmd/models/gemma4v.cpp \
llama.cpp/tools/mtmd/models/glm4v.cpp \
+ llama.cpp/tools/mtmd/models/hunyuanocr.cpp \
llama.cpp/tools/mtmd/models/internvl.cpp \
llama.cpp/tools/mtmd/models/kimik25.cpp \
llama.cpp/tools/mtmd/models/kimivl.cpp \
@@ -289,6 +297,7 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/qwen2vl.cpp \
llama.cpp/tools/mtmd/models/qwen3vl.cpp \
llama.cpp/tools/mtmd/models/siglip.cpp \
+ llama.cpp/tools/mtmd/models/step3vl.cpp \
llama.cpp/tools/mtmd/models/whisper-enc.cpp \
llama.cpp/tools/mtmd/models/youtuvl.cpp
@@ -316,7 +325,9 @@ o/$(MODE)/llama.cpp/tools/server/%.hpp: llama.cpp/tools/server/public/%
@echo 'unsigned int $(VARNAME)_len = sizeof($(VARNAME));' >> $@
SERVER_ASSETS := \
- o/$(MODE)/llama.cpp/tools/server/index.html.gz.hpp \
+ o/$(MODE)/llama.cpp/tools/server/index.html.hpp \
+ o/$(MODE)/llama.cpp/tools/server/bundle.js.hpp \
+ o/$(MODE)/llama.cpp/tools/server/bundle.css.hpp \
o/$(MODE)/llama.cpp/tools/server/loading.html.hpp
# ==============================================================================
@@ -336,7 +347,8 @@ TOOL_SERVER_SRCS := \
llama.cpp/tools/server/server-http.cpp \
llama.cpp/tools/server/server-models.cpp \
llama.cpp/tools/server/server-queue.cpp \
- llama.cpp/tools/server/server-task.cpp
+ llama.cpp/tools/server/server-task.cpp \
+ llama.cpp/tools/server/server-tools.cpp
# Tool object files
TOOL_QUANTIZE_OBJS := $(TOOL_QUANTIZE_SRCS:%.cpp=o/$(MODE)/%.cpp.o)
@@ -373,8 +385,9 @@ $(TOOL_PERPLEXITY_OBJS) $(TOOL_BENCH_OBJS) $(TOOL_SERVER_OBJS) $(MTMD_OBJS): \
-iquote o/$(MODE)/llama.cpp/tools/server \
-isystem llama.cpp/vendor
-# Server needs llamafile headers for Metal support
+# Server needs llamafile headers for Metal support and web UI
$(TOOL_SERVER_OBJS): private CPPFLAGS += -iquote llamafile
+$(TOOL_SERVER_OBJS): private CCFLAGS += -DLLAMA_BUILD_WEBUI
# Version definitions
$(GGML_OBJS): private CCFLAGS += \
diff --git a/llama.cpp.patches/patches/common_arg.cpp.patch b/llama.cpp.patches/patches/common_arg.cpp.patch
index 262d172f..4626cffa 100644
--- a/llama.cpp.patches/patches/common_arg.cpp.patch
+++ b/llama.cpp.patches/patches/common_arg.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/arg.cpp b/common/arg.cpp
--- a/llama.cpp/common/arg.cpp
+++ b/llama.cpp/common/arg.cpp
-@@ -36,6 +36,8 @@
+@@ -37,6 +37,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/common_chat.cpp.patch b/llama.cpp.patches/patches/common_chat.cpp.patch
deleted file mode 100644
index 3972308b..00000000
--- a/llama.cpp.patches/patches/common_chat.cpp.patch
+++ /dev/null
@@ -1,12 +0,0 @@
-diff --git a/common/chat.cpp b/common/chat.cpp
---- a/llama.cpp/common/chat.cpp
-+++ b/llama.cpp/common/chat.cpp
-@@ -1795,7 +1795,7 @@ static common_chat_params common_chat_params_init_deepseek_v3_1(const common_cha
- };
-
- auto prompt = apply(tmpl, inputs,
-- /* messages_override= */ inputs.messages,
-+ /* messages_override= */ std::optional<json>(inputs.messages),
- /* tools_override= */ std::nullopt,
- additional_context);
- data.prompt = prompt;
diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patch
index a413ac99..a09c318f 100644
--- a/llama.cpp.patches/patches/common_common.cpp.patch
+++ b/llama.cpp.patches/patches/common_common.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/common.cpp b/common/common.cpp
--- a/llama.cpp/common/common.cpp
+++ b/llama.cpp/common/common.cpp
-@@ -874,6 +874,16 @@ std::string fs_get_cache_directory() {
+@@ -970,6 +970,16 @@ std::string fs_get_cache_directory() {
cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");
#elif defined(_WIN32)
cache_directory = std::getenv("LOCALAPPDATA");
@@ -18,7 +18,7 @@ diff --git a/common/common.cpp b/common/common.cpp
#elif defined(__EMSCRIPTEN__)
GGML_ABORT("not implemented on this platform");
#else
-@@ -1050,10 +1060,31 @@ common_init_result::common_init_result(common_params & params) :
+@@ -1146,10 +1156,31 @@ common_init_result::common_init_result(common_params & params) :
if (params.fit_params) {
LOG_INF("%s: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on\n", __func__);
diff --git a/llama.cpp.patches/patches/common_download.cpp.patch b/llama.cpp.patches/patches/common_download.cpp.patch
index a731fcb1..ec9a78e8 100644
--- a/llama.cpp.patches/patches/common_download.cpp.patch
+++ b/llama.cpp.patches/patches/common_download.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/download.cpp b/common/download.cpp
--- a/llama.cpp/common/download.cpp
+++ b/llama.cpp/common/download.cpp
-@@ -24,6 +24,8 @@
+@@ -25,6 +25,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
index b7db17a0..64ec872e 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
--- a/llama.cpp/ggml/src/ggml-backend-reg.cpp
+++ b/llama.cpp/ggml/src/ggml-backend-reg.cpp
-@@ -478,7 +478,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
+@@ -485,7 +485,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
if (ec) {
GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());
} else {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
index 06217bfc..ae8d8d00 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
--- a/llama.cpp/ggml/src/ggml-cpu/repack.cpp
+++ b/llama.cpp/ggml/src/ggml-cpu/repack.cpp
-@@ -3521,14 +3521,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
+@@ -4723,14 +4723,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
return nullptr;
}
@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
const void * data, size_t offset, size_t size) {
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -3540,13 +3540,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buff
+@@ -4742,13 +4742,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buff
GGML_UNUSED(buffer);
}
@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
if (buffer == nullptr) {
-@@ -3561,7 +3561,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(gg
+@@ -4763,7 +4763,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(gg
return buffer;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
index 8e010dc5..f187c4bd 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
-@@ -581,7 +581,7 @@ struct ggml_backend_cuda_buffer_context {
+@@ -596,7 +596,7 @@ struct ggml_backend_cuda_buffer_context {
}
};
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
delete ctx;
}
-@@ -590,12 +590,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
+@@ -605,12 +605,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;
}
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
if (tensor->view_src != NULL) {
-@@ -616,7 +616,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
+@@ -631,7 +631,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
return GGML_STATUS_SUCCESS;
}
@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -624,7 +624,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
+@@ -639,7 +639,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -632,7 +632,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -647,7 +647,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -640,7 +640,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -655,7 +655,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (ggml_backend_buffer_is_cuda(src->buffer)) {
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
-@@ -661,7 +661,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
+@@ -676,7 +676,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
GGML_UNUSED(buffer);
}
@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -669,6 +669,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
+@@ -684,6 +684,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -85,7 +85,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
/* .get_base = */ ggml_backend_cuda_buffer_get_base,
-@@ -679,6 +687,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
+@@ -694,6 +702,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cuda_buffer_clear,
/* .reset = */ NULL,
@@ -93,7 +93,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
};
// cuda buffer type
-@@ -687,7 +696,7 @@ struct ggml_backend_cuda_buffer_type_context {
+@@ -702,7 +711,7 @@ struct ggml_backend_cuda_buffer_type_context {
std::string name;
};
@@ -102,7 +102,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -697,7 +706,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
+@@ -712,7 +721,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;
}
@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
ggml_cuda_set_device(buft_ctx->device);
-@@ -716,13 +725,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
+@@ -731,13 +740,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
}
@@ -127,7 +127,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t size = ggml_nbytes(tensor);
int64_t ne0 = tensor->ne[0];
-@@ -836,19 +845,19 @@ struct ggml_backend_cuda_split_buffer_context {
+@@ -851,19 +860,19 @@ struct ggml_backend_cuda_split_buffer_context {
};
@@ -150,7 +150,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_ASSERT(tensor->view_src == nullptr); // views of split tensors are not supported
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -898,7 +907,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
+@@ -913,7 +922,7 @@ static enum ggml_status ggml_backend_cuda_split_buffer_init_tensor(ggml_backend_
return GGML_STATUS_SUCCESS;
}
@@ -159,7 +159,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -937,7 +946,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
+@@ -952,7 +961,7 @@ static void ggml_backend_cuda_split_buffer_set_tensor(ggml_backend_buffer_t buff
}
}
@@ -168,7 +168,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// split tensors must always be set in their entirety at once
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -976,7 +985,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
+@@ -991,7 +1000,7 @@ static void ggml_backend_cuda_split_buffer_get_tensor(ggml_backend_buffer_t buff
}
}
@@ -177,7 +177,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(buffer);
GGML_UNUSED(value);
}
-@@ -991,11 +1000,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
+@@ -1006,11 +1015,12 @@ static const ggml_backend_buffer_i ggml_backend_cuda_split_buffer_interface = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_cuda_split_buffer_clear,
/* .reset = */ NULL,
@@ -191,7 +191,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -1005,7 +1015,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
+@@ -1020,7 +1030,7 @@ static bool ggml_backend_buft_is_cuda_split(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_split_buffer_type_get_name;
}
@@ -200,7 +200,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
// since we don't know the exact split after rounding, we cannot allocate the device buffers at this point
// instead, we allocate them for each tensor separately in init_tensor
// however, the size still represents the maximum cumulative size of all the device buffers after the tensors are allocated,
-@@ -1015,13 +1025,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
+@@ -1030,13 +1040,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_split_buffer_type_alloc_buffer(gg
return ggml_backend_buffer_init(buft, ggml_backend_cuda_split_buffer_interface, ctx, size);
}
@@ -216,7 +216,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_split_buffer_type_context * ctx = (ggml_backend_cuda_split_buffer_type_context *)buft->context;
GGML_ASSERT(ggml_is_contiguous(tensor) && "split buffers only supported for contiguous tensors");
-@@ -1049,7 +1059,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
+@@ -1064,7 +1074,7 @@ static size_t ggml_backend_cuda_split_buffer_type_get_alloc_size(ggml_backend_bu
return total_size;
}
@@ -225,7 +225,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return false;
GGML_UNUSED(buft);
-@@ -1064,7 +1074,7 @@ static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_inte
+@@ -1079,7 +1089,7 @@ static const ggml_backend_buffer_type_i ggml_backend_cuda_split_buffer_type_inte
/* .is_host = */ ggml_backend_cuda_split_buffer_type_is_host,
};
@@ -234,7 +234,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex);
-@@ -1108,7 +1118,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
+@@ -1123,7 +1133,7 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device,
// host buffer type
@@ -243,7 +243,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return GGML_CUDA_NAME "_Host";
GGML_UNUSED(buft);
-@@ -1118,7 +1128,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
+@@ -1133,7 +1143,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
}
@@ -252,7 +252,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
CUDA_CHECK(cudaFreeHost(buffer->context));
}
-@@ -1140,17 +1150,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
+@@ -1155,17 +1165,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
return ptr;
}
@@ -280,7 +280,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return buffer;
}
-@@ -1250,8 +1268,13 @@ static void ggml_cuda_op_mul_mat_cublas(
+@@ -1293,8 +1311,13 @@ static void ggml_cuda_op_mul_mat_cublas(
const int cc = ggml_cuda_info().devices[id].cc;
@@ -292,9 +292,9 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
(GGML_CUDA_CC_IS_MTHREADS(cc) && cc >= GGML_CUDA_CC_QY2);
+#endif
- const bool use_fp16 = (src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT;
-
-@@ -2768,20 +2791,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+ const bool use_fp16 =
+ src0->type != GGML_TYPE_NVFP4 &&
+@@ -2831,20 +2854,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
// backend
@@ -318,7 +318,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2790,7 +2813,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
+@@ -2853,7 +2876,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
CUDA_CHECK(cudaMemcpyAsync((char *)tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
@@ -327,7 +327,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2799,7 +2822,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
+@@ -2862,7 +2885,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
CUDA_CHECK(cudaMemcpyAsync(data, (const char *)tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
@@ -336,7 +336,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
-@@ -2854,7 +2877,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
+@@ -2917,7 +2940,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
return true;
}
@@ -345,7 +345,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
-@@ -3915,7 +3938,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
+@@ -4010,7 +4033,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
}
#endif // USE_CUDA_GRAPH
@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_cuda_set_device(cuda_ctx->device);
-@@ -3974,13 +3997,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
+@@ -4069,13 +4092,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
return GGML_STATUS_SUCCESS;
}
@@ -370,7 +370,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
if (ggml_backend_is_cuda(backend)) {
-@@ -3999,7 +4022,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
+@@ -4094,7 +4117,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
}
}
@@ -379,7 +379,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
#ifdef USE_CUDA_GRAPH
-@@ -4285,7 +4308,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
+@@ -4380,7 +4403,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
CUDA_CHECK(cudaMemGetInfo(free, total));
}
@@ -388,7 +388,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return false;
}
-@@ -4308,7 +4331,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+@@ -4403,7 +4426,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)
}
@@ -397,7 +397,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
return;
}
-@@ -4331,12 +4354,12 @@ struct ggml_backend_cuda_device_context {
+@@ -4426,12 +4449,12 @@ struct ggml_backend_cuda_device_context {
int op_offload_min_batch_size;
};
@@ -412,7 +412,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
return ctx->description.c_str();
}
-@@ -4417,7 +4440,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
+@@ -4512,7 +4535,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
}
#endif // defined(__linux__)
@@ -421,7 +421,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
ggml_cuda_set_device(ctx->device);
CUDA_CHECK(cudaMemGetInfo(free, total));
-@@ -4447,12 +4470,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -4542,12 +4565,12 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
}
@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
props->name = ggml_backend_cuda_device_get_name(dev);
-@@ -4476,24 +4499,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
+@@ -4571,24 +4594,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
};
}
@@ -465,7 +465,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
// split buffers can only be used with GGML_OP_MUL_MAT
-@@ -4867,7 +4890,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -4970,7 +4993,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
}
@@ -474,7 +474,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;
return (((ggml_backend_buft_is_cuda(buft) || ggml_backend_buft_is_cuda_split(buft)) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft)));
-@@ -4888,13 +4911,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -4991,13 +5014,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -490,7 +490,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
#ifdef GGML_CUDA_NO_PEER_COPY
return nullptr;
#else
-@@ -4912,14 +4935,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+@@ -5015,14 +5038,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
#endif
}
@@ -507,7 +507,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(dev);
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
}
-@@ -4948,23 +4971,23 @@ struct ggml_backend_cuda_reg_context {
+@@ -5051,23 +5074,23 @@ struct ggml_backend_cuda_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
@@ -535,7 +535,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::vector<ggml_backend_feature> features = []() {
std::vector<ggml_backend_feature> features;
#define _STRINGIFY(...) #__VA_ARGS__
-@@ -5025,7 +5048,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
+@@ -5128,7 +5151,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
index cca53e24..3b48ffe6 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_vendors_cuda.h.patch
@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda
#include <cuda_bf16.h>
#include <cuda_fp16.h>
-@@ -16,8 +26,10 @@
+@@ -17,8 +27,10 @@
#if CUDART_VERSION < 11020
#define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
index 0ad19bd4..f99567d3 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
@@ -2,72 +2,72 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal.cpp
@@ -23,7 +23,7 @@ static int g_devices = 1;
-
+
// shared buffer
-
+
-static void ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t buffer) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -31,7 +31,7 @@ static void ggml_backend_metal_buffer_shared_free_buffer(ggml_backend_buffer_t b
ggml_metal_buffer_free(ctx);
}
-
+
-static void * ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t buffer) {
+static void * GGML_CALL ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t buffer) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -39,7 +39,7 @@ static void * ggml_backend_metal_buffer_shared_get_base(ggml_backend_buffer_t bu
return ggml_metal_buffer_get_base(ctx);
}
-
+
-static void ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -47,7 +47,7 @@ static void ggml_backend_metal_buffer_shared_memset_tensor(ggml_backend_buffer_t
ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);
}
-
+
-static void ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -55,7 +55,7 @@ static void ggml_backend_metal_buffer_shared_set_tensor(ggml_backend_buffer_t bu
ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);
}
-
+
-static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -63,7 +63,7 @@ static void ggml_backend_metal_buffer_shared_get_tensor(ggml_backend_buffer_t bu
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
}
-
+
-static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -75,7 +75,7 @@ static bool ggml_backend_metal_buffer_shared_cpy_tensor(ggml_backend_buffer_t bu
return false;
}
-
+
-static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer, uint8_t value) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(ggml_metal_buffer_is_shared(ctx));
@@ -83,6 +83,14 @@ static void ggml_backend_metal_buffer_shared_clear(ggml_backend_buffer_t buffer,
ggml_metal_buffer_clear(ctx, value);
}
-
+
+// free_struct callback for cross-module memory management
+// When the Metal backend is loaded as a dynamic library, the buffer struct
+// is allocated by the dylib's 'new' operator and must be freed by the dylib's
@@ -85,67 +85,67 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
/* .reset = */ NULL,
+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
};
-
+
// private buffer
-
+
-static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t buffer) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -105,7 +114,7 @@ static void ggml_backend_metal_buffer_private_free_buffer(ggml_backend_buffer_t
ggml_metal_buffer_free(ctx);
}
-
+
-static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t buffer) {
+static void * GGML_CALL ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t buffer) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -113,7 +122,7 @@ static void * ggml_backend_metal_buffer_private_get_base(ggml_backend_buffer_t b
return ggml_metal_buffer_get_base(ctx);
}
-
+
-static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -121,7 +130,7 @@ static void ggml_backend_metal_buffer_private_memset_tensor(ggml_backend_buffer_
ggml_metal_buffer_memset_tensor(ctx, tensor, value, offset, size);
}
-
+
-static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -129,7 +138,7 @@ static void ggml_backend_metal_buffer_private_set_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_set_tensor(ctx, tensor, data, offset, size);
}
-
+
-static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -137,7 +146,7 @@ static void ggml_backend_metal_buffer_private_get_tensor(ggml_backend_buffer_t b
ggml_metal_buffer_get_tensor(ctx, tensor, data, offset, size);
}
-
+
-static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -149,7 +158,7 @@ static bool ggml_backend_metal_buffer_private_cpy_tensor(ggml_backend_buffer_t b
return false;
}
-
+
-static void ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) {
+static void GGML_CALL ggml_backend_metal_buffer_private_clear(ggml_backend_buffer_t buffer, uint8_t value) {
ggml_metal_buffer_t ctx = (ggml_metal_buffer_t)buffer->context;
-
+
GGML_ASSERT(!ggml_metal_buffer_is_shared(ctx));
@@ -167,6 +176,7 @@ static ggml_backend_buffer_i ggml_backend_metal_buffer_private_i = {
/* .cpy_tensor = */ ggml_backend_metal_buffer_private_cpy_tensor,
@@ -153,170 +153,170 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
/* .reset = */ NULL,
+ /* .free_struct = */ ggml_backend_metal_buffer_free_struct,
};
-
+
static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {
@@ -239,33 +249,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
-
+
// default (shared) buffer type
-
+
-static const char * ggml_backend_metal_buffer_type_shared_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_metal_buffer_type_shared_get_name(ggml_backend_buffer_type_t buft) {
ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;
-
+
return ctx->name.c_str();
}
-
+
-static ggml_backend_buffer_t ggml_backend_metal_buffer_type_shared_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_metal_buffer_type_shared_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, true);
}
-
+
-static size_t ggml_backend_metal_buffer_type_shared_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_shared_get_alignment(ggml_backend_buffer_type_t buft) {
return 32;
-
+
GGML_UNUSED(buft);
}
-
+
-static size_t ggml_backend_metal_buffer_type_shared_get_max_size(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_shared_get_max_size(ggml_backend_buffer_type_t buft) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;
-
+
return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;
}
-
+
-static size_t ggml_backend_metal_buffer_type_shared_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_shared_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);
}
-
+
-static bool ggml_backend_metal_buffer_type_shared_is_host(ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_metal_buffer_type_shared_is_host(ggml_backend_buffer_type_t buft) {
return false;
-
+
GGML_UNUSED(buft);
@@ -315,33 +325,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
-
+
// default (private) buffer type
-
+
-static const char * ggml_backend_metal_buffer_type_private_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_metal_buffer_type_private_get_name(ggml_backend_buffer_type_t buft) {
ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;
-
+
return ctx->name.c_str();
}
-
+
-static ggml_backend_buffer_t ggml_backend_metal_buffer_type_private_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_metal_buffer_type_private_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, false);
}
-
+
-static size_t ggml_backend_metal_buffer_type_private_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_private_get_alignment(ggml_backend_buffer_type_t buft) {
return 32;
-
+
GGML_UNUSED(buft);
}
-
+
-static size_t ggml_backend_metal_buffer_type_private_get_max_size(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_private_get_max_size(ggml_backend_buffer_type_t buft) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;
-
+
return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;
}
-
+
-static size_t ggml_backend_metal_buffer_type_private_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_private_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);
}
-
+
-static bool ggml_backend_metal_buffer_type_private_is_host(ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_metal_buffer_type_private_is_host(ggml_backend_buffer_type_t buft) {
return false;
-
+
GGML_UNUSED(buft);
@@ -390,34 +400,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
-
+
// mapped buffer type
-
+
-static const char * ggml_backend_metal_buffer_type_mapped_get_name(ggml_backend_buffer_type_t buft) {
+static const char * GGML_CALL ggml_backend_metal_buffer_type_mapped_get_name(ggml_backend_buffer_type_t buft) {
ggml_backend_metal_buffer_type * ctx = (ggml_backend_metal_buffer_type *)buft->context;
-
+
return ctx->name.c_str();
}
-
+
-static ggml_backend_buffer_t ggml_backend_metal_buffer_type_mapped_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_metal_buffer_type_mapped_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
// for mapped buffers, prefer shared memory
return ggml_backend_metal_buffer_type_alloc_buffer(buft, size, true);
}
-
+
-static size_t ggml_backend_metal_buffer_type_mapped_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_mapped_get_alignment(ggml_backend_buffer_type_t buft) {
return 32;
-
+
GGML_UNUSED(buft);
}
-
+
-static size_t ggml_backend_metal_buffer_type_mapped_get_max_size(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_mapped_get_max_size(ggml_backend_buffer_type_t buft) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)buft->device->context;
-
+
return ggml_metal_device_get_props(ctx_dev)->max_buffer_size;
}
-
+
-static size_t ggml_backend_metal_buffer_type_mapped_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
+static size_t GGML_CALL ggml_backend_metal_buffer_type_mapped_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor) {
return ggml_backend_metal_buffer_type_get_alloc_size(buft, tensor);
}
-
+
-static bool ggml_backend_metal_buffer_type_mapped_is_host(ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_metal_buffer_type_mapped_is_host(ggml_backend_buffer_type_t buft) {
return false;
-
+
GGML_UNUSED(buft);
@@ -468,13 +478,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
-
+
// backend
-
+
-static const char * ggml_backend_metal_name(ggml_backend_t backend) {
+static const char * GGML_CALL ggml_backend_metal_name(ggml_backend_t backend) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
return ggml_metal_get_name(ctx);
}
-
+
-static void ggml_backend_metal_free(ggml_backend_t backend) {
+static void GGML_CALL ggml_backend_metal_free(ggml_backend_t backend) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
// wait for any ongoing async operations to finish
@@ -485,25 +495,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
free(backend);
}
-
+
-static void ggml_backend_metal_synchronize(ggml_backend_t backend) {
+static void GGML_CALL ggml_backend_metal_synchronize(ggml_backend_t backend) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
ggml_metal_synchronize(ctx);
}
-
+
-static void ggml_backend_metal_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_set_tensor_async(ggml_backend_t backend, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
ggml_metal_set_tensor_async(ctx, tensor, data, offset, size);
}
-
+
-static void ggml_backend_metal_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
+static void GGML_CALL ggml_backend_metal_get_tensor_async(ggml_backend_t backend, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
ggml_metal_get_tensor_async(ctx, tensor, data, offset, size);
}
-
+
-static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
if (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {
@@ -325,69 +325,69 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
@@ -524,27 +534,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
return ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);
}
-
+
-static enum ggml_status ggml_backend_metal_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
+static enum ggml_status GGML_CALL ggml_backend_metal_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
return ggml_metal_graph_compute(ctx, cgraph);
}
-
+
-static void ggml_backend_metal_event_record(ggml_backend_t backend, ggml_backend_event_t event) {
+static void GGML_CALL ggml_backend_metal_event_record(ggml_backend_t backend, ggml_backend_event_t event) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
-
+
ggml_metal_event_record(ctx, ev);
}
-
+
-static void ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
+static void GGML_CALL ggml_backend_metal_event_wait(ggml_backend_t backend, ggml_backend_event_t event) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
-
+
ggml_metal_event_wait(ctx, ev);
}
-
+
-static void ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {
+static void GGML_CALL ggml_backend_metal_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph) {
ggml_metal_t ctx = (ggml_metal_t)backend->context;
-
+
ggml_metal_graph_optimize(ctx, cgraph);
@@ -634,7 +644,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
-
+
// backend device
-
+
-static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
+static const char * GGML_CALL ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
@@ -642,25 +652,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
return props_dev->name;
}
-
+
-static const char * ggml_backend_metal_device_get_description(ggml_backend_dev_t dev) {
+static const char * GGML_CALL ggml_backend_metal_device_get_description(ggml_backend_dev_t dev) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
return ggml_metal_device_get_props(ctx_dev)->desc;
}
-
+
-static void ggml_backend_metal_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
+static void GGML_CALL ggml_backend_metal_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_device_get_memory(ctx_dev, free, total);
}
-
+
-static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backend_dev_t dev) {
+static enum ggml_backend_dev_type GGML_CALL ggml_backend_metal_device_get_type(ggml_backend_dev_t dev) {
return GGML_BACKEND_DEVICE_TYPE_GPU;
-
+
GGML_UNUSED(dev);
}
-
+
-static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
+static void GGML_CALL ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
props->name = ggml_backend_metal_device_get_name(dev);
@@ -396,41 +396,41 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
@@ -675,7 +685,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
};
}
-
+
-static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t dev, const char * params) {
+static ggml_backend_t GGML_CALL ggml_backend_metal_device_init_backend(ggml_backend_dev_t dev, const char * params) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_t ctx = ggml_metal_init(ctx_dev);
@@ -700,7 +710,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
GGML_UNUSED(params);
}
-
+
-static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml_backend_dev_t dev) {
+static ggml_backend_buffer_type_t GGML_CALL ggml_backend_metal_device_get_buffer_type(ggml_backend_dev_t dev) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
@@ -708,7 +718,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
return props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);
}
-
+
-static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
+static ggml_backend_buffer_t GGML_CALL ggml_backend_metal_device_buffer_mapped(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);
@@ -718,13 +728,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
return ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);
}
-
+
-static bool ggml_backend_metal_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+static bool GGML_CALL ggml_backend_metal_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
return ggml_metal_device_supports_op(ctx_dev, op);
}
-
+
-static bool ggml_backend_metal_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
+static bool GGML_CALL ggml_backend_metal_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
return
@@ -439,56 +439,56 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
@@ -745,7 +755,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
-
+
-static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+static bool GGML_CALL ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
return (op->op == GGML_OP_MUL_MAT ||
@@ -753,7 +763,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
get_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;
}
-
+
-static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev_t dev) {
+static ggml_backend_event_t GGML_CALL ggml_backend_metal_device_event_new(ggml_backend_dev_t dev) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);
@@ -767,7 +777,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
return ev;
}
-
+
-static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {
+static void GGML_CALL ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_backend_event_t event) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
@@ -777,7 +787,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
delete event;
}
-
+
-static void ggml_backend_metal_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {
+static void GGML_CALL ggml_backend_metal_device_event_synchronize(ggml_backend_dev_t dev, ggml_backend_event_t event) {
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
-
+
ggml_metal_event_t evt = (ggml_metal_event_t)event->context;
@@ -829,18 +839,18 @@ struct ggml_backend_metal_reg_deleter {
-
+
typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;
-
+
-static const char * ggml_backend_metal_reg_get_name(ggml_backend_reg_t reg) {
+static const char * GGML_CALL ggml_backend_metal_reg_get_name(ggml_backend_reg_t reg) {
return GGML_METAL_NAME;
-
+
GGML_UNUSED(reg);
}
-
+
-static size_t ggml_backend_metal_reg_device_count(ggml_backend_reg_t reg) {
+static size_t GGML_CALL ggml_backend_metal_reg_device_count(ggml_backend_reg_t reg) {
ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;
return ctx->devices.size();
}
-
+
-static ggml_backend_dev_t ggml_backend_metal_reg_device_get(ggml_backend_reg_t reg, size_t index) {
+static ggml_backend_dev_t GGML_CALL ggml_backend_metal_reg_device_get(ggml_backend_reg_t reg, size_t index) {
ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;
@@ -497,14 +497,14 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
@@ -853,13 +863,13 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
{ NULL, NULL },
};
-
+
-static ggml_backend_feature * ggml_backend_metal_get_features(ggml_backend_reg_t reg) {
+static ggml_backend_feature * GGML_CALL ggml_backend_metal_get_features(ggml_backend_reg_t reg) {
return g_ggml_backend_metal_features;
-
+
GGML_UNUSED(reg);
}
-
+
-static void * ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {
+static void * GGML_CALL ggml_backend_metal_get_proc_address(ggml_backend_reg_t reg, const char * name) {
if (strcmp(name, "ggml_backend_get_features") == 0) {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
index 71210c63..79d955f4 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
-@@ -225,11 +225,11 @@ struct vk_queue {
+@@ -238,11 +238,11 @@ struct vk_queue {
}
};
@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
/* .get_name = */ ggml_backend_vk_buffer_type_name,
/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
-@@ -2027,7 +2027,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
+@@ -2074,7 +2074,7 @@ static void ggml_vk_check_results_1(ggml_backend_vk_context * ctx, ggml_cgraph *
typedef void (*ggml_vk_func_t)(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
@@ -27,7 +27,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
static VkDeviceSize ggml_vk_get_max_buffer_range(const ggml_backend_vk_context * ctx, const vk_buffer &buf, const VkDeviceSize offset) {
const VkDeviceSize range = std::min(VkDeviceSize{buf->size - offset},
-@@ -13222,20 +13222,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
+@@ -13428,20 +13428,28 @@ static bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;
}
@@ -59,7 +59,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
-@@ -13243,7 +13251,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
+@@ -13449,7 +13457,7 @@ static enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t
return GGML_STATUS_SUCCESS;
}
@@ -68,7 +68,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13252,7 +13260,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
+@@ -13462,7 +13470,7 @@ static void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, g
ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);
}
@@ -77,7 +77,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -13260,7 +13268,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
+@@ -13474,7 +13482,7 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
@@ -86,16 +86,16 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
-@@ -13269,7 +13277,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
+@@ -13487,7 +13495,7 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
-static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * src, ggml_tensor * dst) {
- if (ggml_backend_buffer_is_vk(src->buffer)) {
- ggml_backend_vk_buffer_context * src_buf_ctx = (ggml_backend_vk_buffer_context *)src->buffer->context;
- ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context;
-@@ -13286,7 +13294,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
+ if (ggml_nbytes(src) == 0) {
+ return true;
+ }
+@@ -13508,7 +13516,7 @@ static bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, cons
UNUSED(buffer);
}
@@ -104,7 +104,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
ggml_vk_buffer_memset(ctx->dev_buffer, 0, value, buffer->size);
-@@ -13302,16 +13310,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
+@@ -13524,16 +13532,17 @@ static ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
/* .clear = */ ggml_backend_vk_buffer_clear,
/* .reset = */ NULL,
@@ -124,7 +124,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
-@@ -13327,17 +13336,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
+@@ -13549,17 +13558,17 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);
}
@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return ggml_nbytes(tensor);
UNUSED(buft);
-@@ -13355,24 +13364,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
+@@ -13577,24 +13586,24 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
// host buffer type
@@ -174,7 +174,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");
size += 32; // Behave like the CPU buffer type
-@@ -13388,19 +13397,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
+@@ -13610,19 +13619,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
buffer->buft = buft;
buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;
@@ -197,7 +197,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return vk_instance.devices[0]->suballocation_block_size;
UNUSED(buft);
-@@ -13432,13 +13442,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
+@@ -13654,13 +13664,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
// backend
@@ -213,7 +213,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
-@@ -13454,7 +13464,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
+@@ -13676,7 +13686,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
return &ctx->device->buffer_type;
}
@@ -222,7 +222,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_set_tensor_async(" << size << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13497,7 +13507,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor
+@@ -13723,7 +13733,7 @@ static void ggml_backend_vk_set_tensor_async(ggml_backend_t backend, ggml_tensor
}
}
@@ -231,16 +231,16 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_get_tensor_async(" << size << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
GGML_ASSERT((tensor->buffer->buft == ggml_backend_vk_get_default_buffer_type(backend) || tensor->buffer->buft == ggml_backend_vk_host_buffer_type()) && "unsupported buffer type");
-@@ -13527,7 +13537,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_
+@@ -13757,7 +13767,7 @@ static void ggml_backend_vk_get_tensor_async(ggml_backend_t backend, const ggml_
}
}
-static bool ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
+static bool GGML_CALL ggml_backend_vk_cpy_tensor_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const ggml_tensor * src, ggml_tensor * dst) {
- VK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async()");
+ VK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;
-@@ -13639,7 +13649,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
+@@ -13882,7 +13892,7 @@ static void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
}
}
@@ -249,7 +249,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_synchronize()");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14061,7 +14071,7 @@ static int32_t find_first_set(uint32_t x) {
+@@ -14304,7 +14314,7 @@ static int32_t find_first_set(uint32_t x) {
return ret;
}
@@ -258,7 +258,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14441,7 +14451,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+@@ -14684,7 +14694,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
}
// Sort the graph for improved parallelism.
@@ -267,7 +267,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
{
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14679,7 +14689,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
+@@ -14922,7 +14932,7 @@ static void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph *
}
}
@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -14702,7 +14712,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
+@@ -14960,7 +14970,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
ctx->compute_ctx.reset();
}
@@ -285,7 +285,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -14796,7 +14806,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
+@@ -15055,7 +15065,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
*total += heap.size;
if (membudget_supported && i < budgetprops.heapUsage.size()) {
@@ -297,7 +297,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
} else {
*free += heap.size;
}
-@@ -14864,38 +14877,38 @@ struct ggml_backend_vk_device_context {
+@@ -15123,38 +15136,38 @@ struct ggml_backend_vk_device_context {
int op_offload_min_batch_size;
};
@@ -343,7 +343,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
props->name = ggml_backend_vk_device_get_name(dev);
-@@ -14911,13 +14924,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
+@@ -15170,13 +15183,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
};
}
@@ -359,7 +359,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
const vk_device& device = ggml_vk_get_device(ctx->device);
-@@ -15445,7 +15458,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
+@@ -15714,7 +15727,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
UNUSED(dev);
}
@@ -368,7 +368,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {
return false;
}
-@@ -15471,13 +15484,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
+@@ -15740,13 +15753,13 @@ static int64_t ggml_vk_get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -384,7 +384,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15497,7 +15510,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
+@@ -15769,7 +15782,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
};
}
@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15509,7 +15522,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
+@@ -15789,7 +15802,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
delete event;
}
@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15543,7 +15556,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
+@@ -15846,7 +15859,7 @@ static vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, si
return buf;
}
@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
GGML_UNUSED(max_tensor_size);
-@@ -15581,17 +15594,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
+@@ -15884,17 +15897,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,
};
diff --git a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
index 619929de..73195679 100644
--- a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
@@ -122,18 +122,22 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
if (is_array) {
std::vector<T> value;
try {
-@@ -394,8 +487,8 @@ bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct
+@@ -394,12 +487,8 @@ bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct
return true;
}
--struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {
+-struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) {
+- if (!file) {
+- return nullptr;
+- }
+-
- const struct gguf_reader gr(file);
+template<typename Reader>
+struct gguf_context * gguf_init_from_reader_impl(const Reader & gr, struct gguf_init_params params) {
struct gguf_context * ctx = new gguf_context;
bool ok = true;
-@@ -696,14 +789,14 @@ struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_par
+@@ -700,14 +789,14 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para
GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);
// we require the data section to be aligned, so take into account any padding
@@ -150,12 +154,15 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
// compute the total size of the data section, taking into account the alignment
{
-@@ -840,7 +933,27 @@ struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_par
+@@ -844,7 +933,30 @@ struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_para
return ctx;
}
+// Wrapper for FILE*-based reading
-+struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {
++struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params) {
++ if (!file) {
++ return nullptr;
++ }
+ const struct gguf_reader gr(file);
+ return gguf_init_from_reader_impl(gr, params);
+}
@@ -178,8 +185,8 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
FILE * file = ggml_fopen(fname, "rb");
if (!file) {
-@@ -851,6 +964,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
- struct gguf_context * result = gguf_init_from_file_impl(file, params);
+@@ -855,6 +967,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
+ struct gguf_context * result = gguf_init_from_file_ptr(file, params);
fclose(file);
return result;
+#endif
diff --git a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
index c1a9db33..a51c2b7b 100644
--- a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
+++ b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
@@ -12,16 +12,12 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#include <cstring>
#include <climits>
#include <stdexcept>
-@@ -165,6 +169,19 @@ struct llama_file::impl {
+@@ -173,6 +177,15 @@ struct llama_file::impl {
}
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
+#ifdef COSMOCC
-+ // [llamafile] Use llamafile_open_gguf for all file opening. This handles:
-+ // - Plain GGUF files (opened normally with fopen)
-+ // - /zip/ paths (memory-mapped from executable's zip)
-+ // - [email protected] syntax
-+ // - .llamafile containers
++ // [llamafile] Use llamafile_open_gguf for all file opening
+ lfile = llamafile_open_gguf(fname, mode);
+ if (lfile == NULL) {
+ throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));
@@ -32,7 +28,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
-@@ -210,6 +227,9 @@ struct llama_file::impl {
+@@ -225,6 +238,9 @@ struct llama_file::impl {
}
size_t tell() const {
@@ -42,7 +38,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
if (fd == -1) {
long ret = std::ftell(fp);
if (ret == -1) {
-@@ -227,6 +247,11 @@ struct llama_file::impl {
+@@ -242,6 +258,11 @@ struct llama_file::impl {
}
void seek(size_t offset, int whence) const {
@@ -54,7 +50,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
off_t ret = 0;
if (fd == -1) {
ret = std::fseek(fp, (long) offset, whence);
-@@ -236,12 +261,22 @@ struct llama_file::impl {
+@@ -251,12 +272,22 @@ struct llama_file::impl {
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
@@ -77,7 +73,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
errno = 0;
if (fd == -1) {
const size_t curr_off = tell();
-@@ -291,6 +326,7 @@ struct llama_file::impl {
+@@ -306,6 +337,7 @@ struct llama_file::impl {
bytes_read += (size_t) ret;
}
}
@@ -85,7 +81,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
}
void read_aligned_chunk(void * dest, size_t size) {
-@@ -351,13 +387,20 @@ struct llama_file::impl {
+@@ -366,13 +398,20 @@ struct llama_file::impl {
}
~impl() {
@@ -94,8 +90,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
+#else
if (fd != -1) {
close(fd);
-- } else {
-+ } else if (fp) {
+ } else if (owns_fp) {
std::fclose(fp);
}
+#endif
@@ -107,21 +102,18 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
std::string fname;
#endif
-@@ -382,6 +425,13 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
+@@ -401,6 +440,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
int llama_file::file_id() const {
+#ifdef COSMOCC
-+ // [llamafile] For pre-mapped files (bundled /zip/ assets), there's no fd.
-+ // Callers that need mmap should check has_premapped_content() first and use
-+ // premapped_content() instead of calling file_id() + mmap().
+ FILE * fp = llamafile_fp(pimpl->lfile);
+ return fp ? fileno(fp) : -1;
+#else
#ifdef _WIN32
return _fileno(pimpl->fp);
#else
-@@ -389,9 +439,10 @@ int llama_file::file_id() const {
+@@ -408,9 +451,10 @@ int llama_file::file_id() const {
return pimpl->fd;
}
#if defined(fileno)
@@ -134,11 +126,10 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#endif
#endif
}
-@@ -409,14 +460,64 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
+@@ -428,14 +472,56 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
-+// [llamafile] Check if file has pre-mapped content (for bundled zip assets)
+bool llama_file::has_premapped_content() const {
+#ifdef COSMOCC
+ return !llamafile_fp(pimpl->lfile);
@@ -169,25 +160,18 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
-+ bool is_owned = true; // [llamafile] false if using pre-mapped content
++ bool is_owned = true;
+#ifdef COSMOCC
-+ llamafile * lfile = nullptr; // [llamafile] for reference counting
++ llamafile * lfile = nullptr;
+#endif
impl(struct llama_file * file, size_t prefetch, bool numa) {
size = file->size();
+
-+ // [llamafile] Check if file already has pre-mapped content (bundled zip assets).
-+ // For /zip/ paths, the content is already memory-mapped by llamafile_open_zip().
-+ // We use reference counting to keep the mapping alive: llamafile_ref() here and
-+ // llamafile_unref() in the destructor. The llama_file also calls llamafile_close()
-+ // (which internally calls llamafile_unref()) in its destructor. Either destruction
-+ // order is safe: the memory is only freed when the last reference is released.
+ if (file->has_premapped_content()) {
+ addr = file->premapped_content();
+ is_owned = false;
+#ifdef COSMOCC
-+ // Keep the llamafile alive by incrementing reference count
+ lfile = (llamafile *)file->get_llamafile();
+ if (lfile) {
+ llamafile_ref(lfile);
@@ -199,11 +183,10 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
int fd = file->file_id();
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
-@@ -461,6 +562,11 @@ struct llama_mmap::impl {
+@@ -480,6 +566,10 @@ struct llama_mmap::impl {
}
void unmap_fragment(size_t first, size_t last) {
-+ // [llamafile] Don't unmap if we're using pre-mapped content
+ if (!is_owned) {
+ return;
+ }
@@ -211,14 +194,12 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
int page_size = sysconf(_SC_PAGESIZE);
align_range(&first, &last, page_size);
size_t len = last - first;
-@@ -497,6 +603,16 @@ struct llama_mmap::impl {
+@@ -516,6 +606,14 @@ struct llama_mmap::impl {
}
~impl() {
-+ // [llamafile] Don't munmap if we're using pre-mapped content
+ if (!is_owned) {
+#ifdef COSMOCC
-+ // Decrement reference count on the llamafile
+ if (lfile) {
+ llamafile_unref(lfile);
+ }
diff --git a/llama.cpp.patches/patches/src_llama-mmap.h.patch b/llama.cpp.patches/patches/src_llama-mmap.h.patch
index f3619797..39fbf730 100644
--- a/llama.cpp.patches/patches/src_llama-mmap.h.patch
+++ b/llama.cpp.patches/patches/src_llama-mmap.h.patch
@@ -1,7 +1,7 @@
diff --git a/src/llama-mmap.h b/src/llama-mmap.h
--- a/llama.cpp/src/llama-mmap.h
+++ b/llama.cpp/src/llama-mmap.h
-@@ -34,6 +34,11 @@ struct llama_file {
+@@ -35,6 +35,11 @@ struct llama_file {
size_t read_alignment() const;
bool has_direct_io() const;
diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
new file mode 100644
index 00000000..0900d663
--- /dev/null
+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
@@ -0,0 +1,74 @@
+diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
+--- a/llama.cpp/tools/server/server-models.cpp
++++ b/llama.cpp/tools/server/server-models.cpp
+@@ -27,6 +27,7 @@
+ #include <netinet/in.h>
+ #include <arpa/inet.h>
+ #include <unistd.h>
++#include <csignal>
+ extern char **environ;
+ #endif
+
+@@ -519,9 +520,12 @@ void server_models::unload_lru() {
+ // wait for unload to complete
+ {
+ std::unique_lock<std::mutex> lk(mutex);
+- cv.wait(lk, [this, &lru_model_name]() {
+- return mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED;
+- });
++ // Use wait_for() instead of wait() to work around a Cosmopolitan libc
++ // bug where untimed futex waits on XNU (macOS) expire after ~72 minutes,
++ // causing condition_variable::wait() to throw ETIMEDOUT.
++ while (mapping[lru_model_name].meta.status != SERVER_MODEL_STATUS_UNLOADED) {
++ cv.wait_for(lk, std::chrono::seconds(30));
++ }
+ }
+ }
+ }
+@@ -623,6 +627,14 @@ void server_models::load(const std::string & name) {
+ });
+
+ std::thread stopping_thread([&]() {
++ // Block SIGINT/SIGTERM on this thread to prevent EINTR during
++ // condition_variable waits. Signals are handled by the main thread.
++ sigset_t block_mask;
++ sigemptyset(&block_mask);
++ sigaddset(&block_mask, SIGINT);
++ sigaddset(&block_mask, SIGTERM);
++ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
++
+ // thread to monitor stopping signal OR child crash
+ auto is_stopping = [this, &name]() {
+ return this->stopping_models.find(name) != this->stopping_models.end();
+@@ -632,7 +644,10 @@ void server_models::load(const std::string & name) {
+ };
+ {
+ std::unique_lock<std::mutex> lk(this->mutex);
+- this->cv_stop.wait(lk, should_wake);
++ // Use wait_for() instead of wait() to work around XNU futex timeout bug
++ while (!should_wake()) {
++ this->cv_stop.wait_for(lk, std::chrono::seconds(30));
++ }
+ }
+ // child may have already exited (e.g. crashed) — skip shutdown sequence
+ if (!subprocess_alive(child_proc.get())) {
+@@ -753,13 +768,14 @@ void server_models::update_status(const std::string & name, server_model_status
+
+ void server_models::wait_until_loading_finished(const std::string & name) {
+ std::unique_lock<std::mutex> lk(mutex);
+- cv.wait(lk, [this, &name]() {
++ // Use wait_for() instead of wait() to work around XNU futex timeout bug
++ while (true) {
+ auto it = mapping.find(name);
+- if (it != mapping.end()) {
+- return it->second.meta.status != SERVER_MODEL_STATUS_LOADING;
++ if (it != mapping.end() && it->second.meta.status != SERVER_MODEL_STATUS_LOADING) {
++ break;
+ }
+- return false;
+- });
++ cv.wait_for(lk, std::chrono::seconds(30));
++ }
+ }
+
+ bool server_models::ensure_model_ready(const std::string & name) {
diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patch
index 0431dede..54cabb6e 100644
--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
--- a/llama.cpp/tools/server/server.cpp
+++ b/llama.cpp/tools/server/server.cpp
-@@ -10,6 +10,7 @@
+@@ -12,6 +12,7 @@
#include <atomic>
#include <clocale>
#include <exception>
@@ -9,7 +9,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
-@@ -17,6 +18,11 @@
+@@ -19,6 +20,11 @@
#include <windows.h>
#endif
@@ -21,7 +21,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
-@@ -67,7 +73,13 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
+@@ -69,12 +75,22 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
};
}
@@ -36,9 +36,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
std::setlocale(LC_NUMERIC, "C");
// own arguments required by this example
-@@ -98,6 +110,10 @@ int main(int argc, char ** argv) {
- params.model_alias.insert(params.model.name);
- }
+ common_params params;
+#ifdef COSMOCC
+ llamafile_has_metal(); // triggers Metal backend registration on macOS ARM64
@@ -46,8 +44,8 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
+
common_init();
- // struct that contains llama context and inference
-@@ -267,6 +283,13 @@ int main(int argc, char ** argv) {
+ if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) {
+@@ -293,6 +309,13 @@ int main(int argc, char ** argv) {
// this will unblock start_loop()
ctx_server.terminate();
};
@@ -61,7 +59,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
}
// TODO: refactor in common/console
-@@ -298,6 +321,11 @@ int main(int argc, char ** argv) {
+@@ -324,6 +347,11 @@ int main(int argc, char ** argv) {
LOG_INF("%s: server is listening on %s\n", __func__, ctx_http.listening_address.c_str());
LOG_INF("%s: starting the main loop...\n", __func__);
@@ -71,9 +69,9 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
+ }
+
// optionally, notify router server that this instance is ready
- const char * router_port = std::getenv("LLAMA_SERVER_ROUTER_PORT");
std::thread monitor_thread;
-@@ -322,5 +350,48 @@ int main(int argc, char ** argv) {
+ if (server_models::is_child_server()) {
+@@ -347,5 +375,48 @@ int main(int argc, char ** argv) {
}
}
diff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
index bae095d0..82a8f41e 100644
--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
@@ -1,7 +1,23 @@
diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp
+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp
-@@ -6043,7 +6043,13 @@ void ThreadPool::worker(bool is_dynamic) {
+@@ -6214,6 +6214,15 @@ void ThreadPool::cleanup_finished_threads() {
+ }
+
+ void ThreadPool::worker(bool is_dynamic) {
++#ifndef _WIN32
++ // Block SIGINT/SIGTERM on pool threads to prevent EINTR during
++ // condition_variable waits. Signals are handled by the main thread.
++ sigset_t block_mask;
++ sigemptyset(&block_mask);
++ sigaddset(&block_mask, SIGINT);
++ sigaddset(&block_mask, SIGTERM);
++ pthread_sigmask(SIG_BLOCK, &block_mask, nullptr);
++#endif
+ for (;;) {
+ std::function<void()> fn;
+ {
+@@ -6231,7 +6240,13 @@ void ThreadPool::worker(bool is_dynamic) {
break;
}
} else {
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 05a6d51e..de077711 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -224,7 +224,8 @@ LLAMAFILE_SERVER_SUPPORT_OBJS := \
o/$(MODE)/llama.cpp/tools/server/server-http.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-models.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-queue.cpp.o \
- o/$(MODE)/llama.cpp/tools/server/server-task.cpp.o
+ o/$(MODE)/llama.cpp/tools/server/server-task.cpp.o \
+ o/$(MODE)/llama.cpp/tools/server/server-tools.cpp.o
# Metal source files to embed in the executable (for runtime compilation on macOS)
# These are extracted at runtime and compiled into ggml-metal.dylib
@@ -285,7 +286,7 @@ LLAMAFILE_SERVER_INCS := \
# Compile server.cpp
o/$(MODE)/llamafile/server.cpp.o: llama.cpp/tools/server/server.cpp $(SERVER_ASSETS)
@mkdir -p $(@D)
- $(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) $(LLAMAFILE_SERVER_INCS) -c -o $@ $<
+ $(CXX) $(CXXFLAGS) $(LLAMAFILE_CPPFLAGS) $(LLAMAFILE_SERVER_INCS) -DLLAMA_BUILD_WEBUI -c -o $@ $<
# ==============================================================================
# Main executable
diff --git a/llamafile/chatbot_cli.cpp b/llamafile/chatbot_cli.cpp
index 322b923c..f0beaf86 100644
--- a/llamafile/chatbot_cli.cpp
+++ b/llamafile/chatbot_cli.cpp
@@ -86,7 +86,7 @@ static cli_chat_template_result cli_apply_chat_template_full(llama_model *model,
// Initialize parser params from chat_params
result.parser_params.format = chat_params.format;
- result.parser_params.thinking_forced_open = chat_params.thinking_forced_open;
+ result.parser_params.generation_prompt = chat_params.generation_prompt;
if (!chat_params.parser.empty()) {
result.parser_params.parser.load(chat_params.parser);
}
diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cpp
index 9b44987f..ccf415f6 100644
--- a/llamafile/chatbot_main.cpp
+++ b/llamafile/chatbot_main.cpp
@@ -249,12 +249,14 @@ int main(int argc, char **argv) {
// Check if the template supports enable_thinking (like llama.cpp server does).
// This is needed for models like Qwen3.5 that check enable_thinking in their
// template - without this, the template outputs a closed thinking block.
+ // Honor --reasoning on/off/auto (g_params->enable_reasoning: -1 auto, 0 off, 1 on).
bool supports_thinking = common_chat_templates_support_enable_thinking(g_chat_templates.get());
+ bool enable_thinking = supports_thinking && (g_params->enable_reasoning != 0);
common_chat_templates_inputs inputs;
inputs.messages = {dummy_msg};
inputs.use_jinja = true;
- inputs.enable_thinking = supports_thinking;
+ inputs.enable_thinking = enable_thinking;
// CRITICAL: Set reasoning_format BEFORE applying templates. The PEG parser
// is built during common_chat_templates_apply() and checks this value to
// decide whether to include reasoning extraction in the grammar.
@@ -263,7 +265,7 @@ int main(int argc, char **argv) {
try {
auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);
g_chat_syntax.format = chat_params.format;
- g_chat_syntax.thinking_forced_open = chat_params.thinking_forced_open;
+ g_chat_syntax.generation_prompt = chat_params.generation_prompt;
// Load the PEG parser if one was provided
if (!chat_params.parser.empty()) {
diff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cpp
index ba351ec3..ac693404 100644
--- a/llamafile/chatbot_repl.cpp
+++ b/llamafile/chatbot_repl.cpp
@@ -72,15 +72,17 @@ std::string apply_chat_template_with_thinking(
if (!g_chat_templates)
return "";
- // Check if template supports thinking mode
+ // Check if template supports thinking mode, and honor --reasoning on/off/auto
+ // (g_params->enable_reasoning: -1 auto, 0 disable, 1 enable)
bool supports_thinking = common_chat_templates_support_enable_thinking(g_chat_templates.get());
+ bool enable_thinking = supports_thinking && (g_params->enable_reasoning != 0);
common_chat_templates_inputs inputs;
inputs.messages = past_msgs;
inputs.messages.push_back(new_msg);
inputs.use_jinja = true;
inputs.add_generation_prompt = add_generation_prompt;
- inputs.enable_thinking = supports_thinking;
+ inputs.enable_thinking = enable_thinking;
inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);
diff --git a/llamafile/tinyblas_cpu.h b/llamafile/tinyblas_cpu.h
index a8ab359e..9cbb13bd 100644
--- a/llamafile/tinyblas_cpu.h
+++ b/llamafile/tinyblas_cpu.h
@@ -769,6 +769,33 @@ class tinyBLAS_Q0_ARM {
return vsubq_s8(vreinterpretq_s8_u8(vshrq_n_u8(vld1q_u8(b->qs), 4)), vdupq_n_s8(0x8));
}
+ // Q1_0: expand 2 bytes (16 bits) to 16 signed int8 values (+1/-1)
+ inline int8x16_t load_lo(const block_q1_0 *b) {
+ // Bits 0-15 (bytes 0-1): each bit → +1 or -1
+ const uint8x8_t byte0 = vdup_n_u8(b->qs[0]);
+ const uint8x8_t byte1 = vdup_n_u8(b->qs[1]);
+ const uint8x8_t bitmask = vcreate_u8(0x8040201008040201ULL);
+ // Test each bit, result: 0xFF or 0x00
+ const uint8x8_t t0 = vtst_u8(byte0, bitmask);
+ const uint8x8_t t1 = vtst_u8(byte1, bitmask);
+ // Combine to 16 elements and convert 0xFF→+1, 0x00→-1
+ const int8x16_t mask = vreinterpretq_s8_u8(vcombine_u8(t0, t1));
+ const int8x16_t ones = vdupq_n_s8(1);
+ return vsubq_s8(vandq_s8(mask, vdupq_n_s8(2)), ones); // (mask & 2) - 1
+ }
+
+ inline int8x16_t load_hi(const block_q1_0 *b) {
+ // Bits 16-31 (bytes 2-3): each bit → +1 or -1
+ const uint8x8_t byte2 = vdup_n_u8(b->qs[2]);
+ const uint8x8_t byte3 = vdup_n_u8(b->qs[3]);
+ const uint8x8_t bitmask = vcreate_u8(0x8040201008040201ULL);
+ const uint8x8_t t0 = vtst_u8(byte2, bitmask);
+ const uint8x8_t t1 = vtst_u8(byte3, bitmask);
+ const int8x16_t mask = vreinterpretq_s8_u8(vcombine_u8(t0, t1));
+ const int8x16_t ones = vdupq_n_s8(1);
+ return vsubq_s8(vandq_s8(mask, vdupq_n_s8(2)), ones);
+ }
+
const TA *const A;
const TB *const B;
TC *const C;
@@ -987,6 +1014,29 @@ class tinyBLAS_Q0_AVX2 {
_mm256_set1_epi8(8));
}
+ inline __m256i load(const block_q1_0 *b) {
+ // Expand 4 bytes (32 bits) to 32 signed int8 values: bit=1 → +1, bit=0 → -1
+ const uint32_t bits = *(const uint32_t *)b->qs;
+ const __m128i bits128 = _mm_set1_epi32(bits);
+ const __m256i bits256 = _mm256_broadcastsi128_si256(bits128);
+ const __m256i shuf = _mm256_set_epi8(
+ 3,3,3,3,3,3,3,3, 2,2,2,2,2,2,2,2,
+ 1,1,1,1,1,1,1,1, 0,0,0,0,0,0,0,0);
+ const __m256i shuffled = _mm256_shuffle_epi8(bits256, shuf);
+ const __m256i bitmask = _mm256_set_epi8(
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01);
+ const __m256i test = _mm256_cmpeq_epi8(
+ _mm256_and_si256(shuffled, bitmask), bitmask);
+ // test: 0xFF where bit=1, 0x00 where bit=0
+ // Convert: 0xFF → +1, 0x00 → -1
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i bit_val = _mm256_and_si256(test, ones); // 1 or 0
+ return _mm256_sub_epi8(_mm256_add_epi8(bit_val, bit_val), ones); // 2*v - 1 → +1 or -1
+ }
+
inline __m256 updot(__m256i u, __m256i s) {
__m256i res;
#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
@@ -1007,6 +1057,282 @@ class tinyBLAS_Q0_AVX2 {
const int ith;
const int nth;
};
+// Q1_0_g128: 128-element blocks with 4:1 ratio to Q8_0 blocks
+template <int CONFIG, typename TC>
+class tinyBLAS_Q0_g128_AVX2 {
+ // Typedefs required by the INDEX macro
+ typedef block_q1_0 TA;
+ typedef block_q8_0 TB;
+ public:
+ tinyBLAS_Q0_g128_AVX2(long k, const block_q1_0 *A, long lda, const block_q8_0 *B, long ldb,
+ TC *C, long ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(long m, long n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(long m0, long m, long n0, long n) {
+ long mc, nc, mp, np;
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3;
+ nc = 3;
+ gemm<3, 3, true>(m0, m, n0, n);
+ break;
+ case 0x32:
+ case 0x23:
+ case 0x22:
+ mc = 2;
+ nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x31:
+ case 0x21:
+ mc = 2;
+ nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13:
+ case 0x12:
+ mc = 1;
+ nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1;
+ nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN, int PRECISE>
+ NOINLINE void gemm(long m0, long m, long n0, long n) {
+ long ytiles = RM > 1 ? (m - m0) / RM : 1;
+ long xtiles = RN > 1 ? (n - n0) / RN : 1;
+ long tiles = xtiles * ytiles;
+ long duty = (tiles + nth - 1) / nth;
+ long start = duty * ith;
+ long end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (long job = start; job < end; ++job) {
+ long ii = m0 + job / xtiles * RM;
+ long jj = n0 + job % xtiles * RN;
+ __m256 Cv[RN][RM] = {};
+ __m256 Ce[RN][RM] = {};
+ for (long l = 0; l < k; ++l) {
+#pragma GCC unroll 100
+ for (int j = 0; j < RN; ++j)
+#pragma GCC unroll 100
+ for (int i = 0; i < RM; ++i) {
+ const block_q1_0 *Abl = &INDEX(A, lda, ii + i, l)[0];
+ float scale_a = unhalf(Abl->d);
+ // Process 4 Q8_0 sub-blocks per g128 block
+ for (int sub = 0; sub < 4; ++sub) {
+ const block_q8_0 *Bbl = &INDEX(B, ldb, jj + j, 4*l + sub)[0];
+ __m256 a = _mm256_set1_ps(scale_a * unhalf(Bbl->d));
+ __m256i qa = load_sub(Abl, sub);
+ __m256i qb = _mm256_loadu_si256((const __m256i *)Bbl->qs);
+ __m256 b = updot(_mm256_sign_epi8(qa, qa),
+ _mm256_sign_epi8(qb, qa));
+ if (PRECISE)
+ Cv[j][i] = madder(a, b, Cv[j][i], &Ce[j][i]);
+ else
+ Cv[j][i] = madd(a, b, Cv[j][i]);
+ }
+ }
+ }
+#pragma GCC unroll 100
+ for (int j = 0; j < RN; ++j)
+#pragma GCC unroll 100
+ for (int i = 0; i < RM; ++i)
+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
+ }
+ }
+
+ // Extract 32 bits from sub-block `sub` (0-3) of a g128 block
+ inline __m256i load_sub(const block_q1_0 *b, int sub) {
+ const uint32_t bits = *(const uint32_t *)(b->qs + sub * 4);
+ const __m128i bits128 = _mm_set1_epi32(bits);
+ const __m256i bits256 = _mm256_broadcastsi128_si256(bits128);
+ const __m256i shuf = _mm256_set_epi8(
+ 3,3,3,3,3,3,3,3, 2,2,2,2,2,2,2,2,
+ 1,1,1,1,1,1,1,1, 0,0,0,0,0,0,0,0);
+ const __m256i shuffled = _mm256_shuffle_epi8(bits256, shuf);
+ const __m256i bitmask = _mm256_set_epi8(
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01,
+ (char)0x80,0x40,0x20,0x10,0x08,0x04,0x02,0x01);
+ const __m256i test = _mm256_cmpeq_epi8(
+ _mm256_and_si256(shuffled, bitmask), bitmask);
+ const __m256i ones = _mm256_set1_epi8(1);
+ const __m256i bit_val = _mm256_and_si256(test, ones);
+ return _mm256_sub_epi8(_mm256_add_epi8(bit_val, bit_val), ones);
+ }
+
+ inline __m256 updot(__m256i u, __m256i s) {
+ __m256i res;
+#if defined(__AVXVNNI__) || (defined(__AVX512VNNI__) && defined(__AVX512VL__))
+ res = _mm256_dpbusd_epi32(_mm256_setzero_si256(), u, s);
+#else
+ res = _mm256_madd_epi16(_mm256_set1_epi16(1), _mm256_maddubs_epi16(u, s));
+#endif
+ return _mm256_cvtepi32_ps(res);
+ }
+
+ const block_q1_0 *const A;
+ const block_q8_0 *const B;
+ TC *const C;
+ const long k;
+ const long lda;
+ const long ldb;
+ const long ldc;
+ const int ith;
+ const int nth;
+};
#endif // __AVX2__
+#if defined(__ARM_FEATURE_DOTPROD)
+// Q1_0_g128: 128-element blocks with 4:1 ratio to Q8_0 blocks (ARM)
+template <int CONFIG, typename TC>
+class tinyBLAS_Q0_g128_ARM {
+ // Typedefs required by the INDEX macro
+ typedef block_q1_0 TA;
+ typedef block_q8_0 TB;
+ public:
+ tinyBLAS_Q0_g128_ARM(long k, const block_q1_0 *A, long lda, const block_q8_0 *B, long ldb,
+ TC *C, long ldc, int ith, int nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(long m, long n) {
+ mnpack(0, m, 0, n);
+ }
+
+ private:
+ void mnpack(long m0, long m, long n0, long n) {
+ long mc, nc, mp, np;
+ switch ((MIN(m - m0, 3) << 4) | MIN(n - n0, 3)) {
+ case 0x33:
+ mc = 3; nc = 3;
+ gemm<3, 3, true>(m0, m, n0, n);
+ break;
+ case 0x32: case 0x23: case 0x22:
+ mc = 2; nc = 2;
+ gemm<2, 2, true>(m0, m, n0, n);
+ break;
+ case 0x31: case 0x21:
+ mc = 2; nc = 1;
+ gemm<2, 1, true>(m0, m, n0, n);
+ break;
+ case 0x13: case 0x12:
+ mc = 1; nc = 2;
+ gemm<1, 2, true>(m0, m, n0, n);
+ break;
+ case 0x11:
+ mc = 1; nc = 1;
+ gemm<1, 1, true>(m0, m, n0, n);
+ break;
+ default:
+ return;
+ }
+ mp = m0 + (m - m0) / mc * mc;
+ np = n0 + (n - n0) / nc * nc;
+ mnpack(mp, m, n0, np);
+ mnpack(m0, m, np, n);
+ }
+
+ template <int RM, int RN, int PRECISE>
+ NOINLINE void gemm(long m0, long m, long n0, long n) {
+ long ytiles = RM > 1 ? (m - m0) / RM : 1;
+ long xtiles = RN > 1 ? (n - n0) / RN : 1;
+ long tiles = xtiles * ytiles;
+ long duty = (tiles + nth - 1) / nth;
+ long start = duty * ith;
+ long end = start + duty;
+ if (end > tiles)
+ end = tiles;
+ for (long job = start; job < end; ++job) {
+ long ii = m0 + job / xtiles * RM;
+ long jj = n0 + job % xtiles * RN;
+ float32x4_t Cv[RN][RM] = {};
+ float32x4_t Ce[RN][RM] = {};
+ for (int l = 0; l < k; ++l) {
+#pragma GCC unroll 100
+ for (int j = 0; j < RN; ++j)
+#pragma GCC unroll 100
+ for (int i = 0; i < RM; ++i) {
+ const block_q1_0 *Abl = &INDEX(A, lda, ii + i, l)[0];
+ float scale_a = unhalf(Abl->d);
+ for (int sub = 0; sub < 4; ++sub) {
+ const block_q8_0 *Bbl = &INDEX(B, ldb, jj + j, 4*l + sub)[0];
+ float b = scale_a * unhalf(Bbl->d);
+ int8x16_t qa_lo = load_sub_lo(Abl, sub);
+ int8x16_t qa_hi = load_sub_hi(Abl, sub);
+ int8x16_t qb_lo = vld1q_s8(Bbl->qs);
+ int8x16_t qb_hi = vld1q_s8(Bbl->qs + 16);
+ float32x4_t a = vcvtq_f32_s32(vdotq_s32(
+ vdotq_s32(vdupq_n_s32(0), qa_lo, qb_lo),
+ qa_hi, qb_hi));
+ if (PRECISE)
+ Cv[j][i] = badder(a, b, Cv[j][i], &Ce[j][i]);
+ else
+ Cv[j][i] = vmlaq_n_f32(Cv[j][i], a, b);
+ }
+ }
+ }
+#pragma GCC unroll 100
+ for (int j = 0; j < RN; ++j)
+#pragma GCC unroll 100
+ for (int i = 0; i < RM; ++i)
+ store(INDEX(C, ldc, jj + j, ii + i), hsum(Cv[j][i]));
+ }
+ }
+
+ // Extract 16 bits (2 bytes) from sub-block, expand to 16 signed int8
+ inline int8x16_t load_sub_lo(const block_q1_0 *b, int sub) {
+ const uint8_t *qs = b->qs + sub * 4;
+ const uint8x8_t b0 = vdup_n_u8(qs[0]);
+ const uint8x8_t b1 = vdup_n_u8(qs[1]);
+ const uint8x8_t bitmask = vcreate_u8(0x8040201008040201ULL);
+ const uint8x8_t t0 = vtst_u8(b0, bitmask);
+ const uint8x8_t t1 = vtst_u8(b1, bitmask);
+ const int8x16_t mask = vreinterpretq_s8_u8(vcombine_u8(t0, t1));
+ return vsubq_s8(vandq_s8(mask, vdupq_n_s8(2)), vdupq_n_s8(1));
+ }
+
+ inline int8x16_t load_sub_hi(const block_q1_0 *b, int sub) {
+ const uint8_t *qs = b->qs + sub * 4 + 2;
+ const uint8x8_t b0 = vdup_n_u8(qs[0]);
+ const uint8x8_t b1 = vdup_n_u8(qs[1]);
+ const uint8x8_t bitmask = vcreate_u8(0x8040201008040201ULL);
+ const uint8x8_t t0 = vtst_u8(b0, bitmask);
+ const uint8x8_t t1 = vtst_u8(b1, bitmask);
+ const int8x16_t mask = vreinterpretq_s8_u8(vcombine_u8(t0, t1));
+ return vsubq_s8(vandq_s8(mask, vdupq_n_s8(2)), vdupq_n_s8(1));
+ }
+
+ const block_q1_0 *const A;
+ const block_q8_0 *const B;
+ TC *const C;
+ const long k;
+ const long lda;
+ const long ldb;
+ const long ldc;
+ const int ith;
+ const int nth;
+};
+#endif // __ARM_FEATURE_DOTPROD
+
} // namespace
diff --git a/llamafile/tinyblas_cpu_mixmul.inc b/llamafile/tinyblas_cpu_mixmul.inc
index 831184cd..18b0bcfe 100644
--- a/llamafile/tinyblas_cpu_mixmul.inc
+++ b/llamafile/tinyblas_cpu_mixmul.inc
@@ -250,6 +250,20 @@ class MixMul {
return false;
#endif
+ case GGML_TYPE_Q1_0:
+ // Upstream Q1_0 uses 128-element blocks (QK1_0=128)
+ if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
+ return false;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ return mixmat<32, 32, tinyBLAS_Q0_g128_AVX2<NCB | NCC, TC>,
+ block_q1_0, block_q8_0, TC>();
+#elif defined(__ARM_FEATURE_DOTPROD)
+ return mixmat<32, 32, tinyBLAS_Q0_g128_ARM<NCB | NCC, TC>,
+ block_q1_0, block_q8_0, TC>();
+#else
+ return false;
+#endif
+
case GGML_TYPE_Q8_0:
if (thought->type != GGML_TYPE_F32 && thought->type != GGML_TYPE_Q8_0)
return false;
diff --git a/llamafile/tinyblas_cpu_sgemm.inc b/llamafile/tinyblas_cpu_sgemm.inc
index 99f8378b..c1fc3d2c 100644
--- a/llamafile/tinyblas_cpu_sgemm.inc
+++ b/llamafile/tinyblas_cpu_sgemm.inc
@@ -220,6 +220,27 @@ bool llamafile_sgemm_impl(long m, long n, long k, const void *A, long lda, const
#endif
}
+ case GGML_TYPE_Q1_0: {
+ // Upstream Q1_0 uses 128-element blocks (QK1_0=128)
+ if (Btype == GGML_TYPE_F32)
+ return WANT_QUANTIZATION;
+ if (Btype != GGML_TYPE_Q8_0)
+ return NOT_SUPPORTED;
+#if defined(__AVX2__) || defined(__AVX512F__)
+ tinyBLAS_Q0_g128_AVX2<0, TC> tb{
+ k, (const block_q1_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+#elif defined(__ARM_FEATURE_DOTPROD)
+ tinyBLAS_Q0_g128_ARM<0, TC> tb{
+ k, (const block_q1_0 *)A, lda, (const block_q8_0 *)B, ldb, C, ldc, ith, nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return NOT_SUPPORTED;
+#endif
+ }
+
default:
return NOT_SUPPORTED;
}
diff --git a/tests/BUILD.mk b/tests/BUILD.mk
index fb1fe7ba..46b2730b 100644
--- a/tests/BUILD.mk
+++ b/tests/BUILD.mk
@@ -31,7 +31,13 @@ EXTRACT_DATA_URIS_TEST_DEPS := \
o/$(MODE)/llamafile/string.o \
o/$(MODE)/llamafile/xterm.o \
o/$(MODE)/third_party/stb/stb.a \
- o/$(MODE)/llama.cpp/common/build-info.cpp.o
+ o/$(MODE)/llama.cpp/common/build-info.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/caps.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/lexer.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/parser.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/runtime.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/string.cpp.o \
+ o/$(MODE)/llama.cpp/common/jinja/value.cpp.o
o/$(MODE)/tests/extract_data_uris_test.o: tests/extract_data_uris_test.cpp
@mkdir -p $(@D)
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index a883a85d..5e4e993e 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -127,13 +127,21 @@ def timeouts(timeout_multiplier) -> Timeouts:
return Timeouts(timeout_multiplier)
[email protected](scope="session")
-def llamafile(executable, model, gpu_mode) -> LlamafileRunner:
- """Create a LlamafileRunner instance for tests."""
+def llamafile(request, executable, model, gpu_mode) -> LlamafileRunner:
+ """Create a LlamafileRunner instance for tests.
+
+ Thinking is disabled by default so that small reasoning models don't
+ derail basic assertions. Tests marked ``@pytest.mark.thinking`` get a
+ runner with thinking enabled by default, so thinking-specific tests
+ don't need to opt in on every call.
+ """
+ default_thinking = request.node.get_closest_marker("thinking") is not None
return LlamafileRunner(
executable=executable,
model=model,
gpu=gpu_mode,
+ default_thinking=default_thinking,
)
diff --git a/tests/integration/tests/test_cli.py b/tests/integration/tests/test_cli.py
index cd7cc49f..51113e27 100644
--- a/tests/integration/tests/test_cli.py
+++ b/tests/integration/tests/test_cli.py
@@ -39,32 +39,31 @@ class TestCLIThinking:
"""Test that the model has thinking enabled (otherwise --nothink tests make no sense)."""
prompt = "What is 2+2? Think step by step then give the answer."
- # With thinking
- result_think = llamafile.run_cli(prompt, nothink=False, timeout=timeouts.cli)
+ # Thinking class is marked @pytest.mark.thinking, so default is on.
+ result_think = llamafile.run_cli(prompt, timeout=timeouts.cli)
assert result_think.returncode == 0
-
- # nothink output should not contain think tags
assert "<think>" in result_think.stdout
def test_nothink_removes_thinking(self, llamafile, timeouts):
"""Test that --nothink removes thinking content from output."""
prompt = "What is 2+2? Think step by step then give the answer."
- # Without thinking
- result_nothink = llamafile.run_cli(prompt, nothink=True, timeout=timeouts.cli)
+ result_nothink = llamafile.run_cli(
+ prompt, thinking=False, timeout=timeouts.cli
+ )
assert result_nothink.returncode == 0
-
- # nothink output should not contain think tags
assert "<think>" not in result_nothink.stdout
def test_nothink_shorter_output(self, llamafile, timeouts):
"""Test that --nothink produces shorter output (no thinking tokens)."""
prompt = "Explain briefly why the sky is blue."
- result_think = llamafile.run_cli(prompt, nothink=False, timeout=timeouts.cli)
- result_nothink = llamafile.run_cli(prompt, nothink=True, timeout=timeouts.cli)
+ result_think = llamafile.run_cli(prompt, thinking=True, timeout=timeouts.cli)
+ result_nothink = llamafile.run_cli(
+ prompt, thinking=False, timeout=timeouts.cli
+ )
# nothink should generally be shorter (no thinking block)
# This may not always hold for very short responses
diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.py
index 9536c574..dc8fe2c2 100644
--- a/tests/integration/utils/llamafile.py
+++ b/tests/integration/utils/llamafile.py
@@ -115,6 +115,7 @@ class LlamafileRunner:
executable: str,
model: str | None = None,
gpu: str | None = None,
+ default_thinking: bool = False,
):
"""Initialize the runner.
@@ -122,16 +123,25 @@ class LlamafileRunner:
executable: Path to llamafile binary or pre-built .llamafile
model: Path to model file (None for pre-built llamafiles)
gpu: GPU mode - "auto", "apple", "amd", "nvidia", or None for CPU
+ default_thinking: Default thinking mode when callers don't specify
+ one. Set to True for tests that exercise thinking
+ behavior; False otherwise so small reasoning
+ models don't derail unrelated assertions.
"""
self.executable = os.path.abspath(executable)
self.model = os.path.abspath(model) if model else None
self.gpu = gpu
+ self.default_thinking = default_thinking
if not os.path.exists(self.executable):
raise FileNotFoundError(f"Executable not found: {executable}")
if self.model and not os.path.exists(self.model):
raise FileNotFoundError(f"Model not found: {model}")
+ def _resolve_thinking(self, thinking: bool | None) -> bool:
+ """Resolve the thinking flag, falling back to the runner default."""
+ return self.default_thinking if thinking is None else thinking
+
def _base_args(self) -> list[str]:
"""Build base command arguments.
@@ -151,7 +161,7 @@ class LlamafileRunner:
def run_cli(
self,
prompt: str,
- nothink: bool = False,
+ thinking: bool | None = None,
extra_args: list[str] | None = None,
timeout: float = TIMEOUT_CLI,
log_file: str | None = None,
@@ -160,7 +170,9 @@ class LlamafileRunner:
Args:
prompt: The prompt to send
- nothink: If True, disable thinking output
+ thinking: Whether the model should produce a thinking block. If
+ None, falls back to the runner's default_thinking.
+ When False, adds --nothink (llamafile's CLI-mode flag).
extra_args: Additional command-line arguments
timeout: Timeout in seconds
log_file: If provided, adds --log-file flag and stores log content
@@ -173,7 +185,7 @@ class LlamafileRunner:
args = self._base_args()
args.extend(["--cli", "-p", prompt])
- if nothink:
+ if not self._resolve_thinking(thinking):
args.append("--nothink")
if log_file:
@@ -208,6 +220,7 @@ class LlamafileRunner:
def run_tui(
self,
input_file: str,
+ thinking: bool | None = None,
extra_args: list[str] | None = None,
timeout: float = TIMEOUT_TUI,
log_file: str | None = None,
@@ -216,6 +229,9 @@ class LlamafileRunner:
Args:
input_file: Path to file containing input to pipe to stdin
+ thinking: Whether the model should produce a thinking block. If
+ None, falls back to the runner's default_thinking.
+ When False, adds --reasoning off.
extra_args: Additional command-line arguments
timeout: Timeout in seconds
log_file: If provided, adds --log-file flag and stores log content
@@ -228,6 +244,9 @@ class LlamafileRunner:
args = self._base_args()
args.append("--chat")
+ if not self._resolve_thinking(thinking):
+ args.extend(["--reasoning", "off"])
+
if log_file:
args.extend(["--log-file", log_file])
@@ -265,6 +284,7 @@ class LlamafileRunner:
def start_server(
self,
port: int = 8080,
+ thinking: bool | None = None,
extra_args: list[str] | None = None,
log_file: str | None = None,
) -> subprocess.Popen:
@@ -272,6 +292,9 @@ class LlamafileRunner:
Args:
port: Port to listen on
+ thinking: Whether the model should produce a thinking block. If
+ None, falls back to the runner's default_thinking.
+ When False, adds --reasoning off.
extra_args: Additional command-line arguments
log_file: If provided, adds --log-file flag. Caller should read
the file after terminating the process.
@@ -282,6 +305,9 @@ class LlamafileRunner:
args = self._base_args()
args.extend(["--server", "--port", str(port)])
+ if not self._resolve_thinking(thinking):
+ args.extend(["--reasoning", "off"])
+
if log_file:
args.extend(["--log-file", log_file])
@@ -299,6 +325,7 @@ class LlamafileRunner:
def start_combined(
self,
port: int = 8080,
+ thinking: bool | None = None,
extra_args: list[str] | None = None,
log_file: str | None = None,
) -> subprocess.Popen:
@@ -306,6 +333,9 @@ class LlamafileRunner:
Args:
port: Port for the server component
+ thinking: Whether the model should produce a thinking block. If
+ None, falls back to the runner's default_thinking.
+ When False, adds --reasoning off.
extra_args: Additional command-line arguments
log_file: If provided, adds --log-file flag. Caller should read
the file after terminating the process.
@@ -316,6 +346,9 @@ class LlamafileRunner:
args = self._base_args()
args.extend(["--port", str(port)])
+ if not self._resolve_thinking(thinking):
+ args.extend(["--reasoning", "off"])
+
if log_file:
args.extend(["--log-file", log_file])