Skip to content

Distribute and run LLMs with a single file.

Update llama.cpp to v0.8.0 (#1077)

Davide Eynarde5a6ad951cc7
diff --git a/agent.cpp.patches/patches/src_agent.cpp.patch b/agent.cpp.patches/patches/src_agent.cpp.patch
new file mode 100644
index 00000000..6f07a1af
--- /dev/null
+++ b/agent.cpp.patches/patches/src_agent.cpp.patch
@@ -0,0 +1,12 @@
+diff --git a/src/agent.cpp b/src/agent.cpp
+--- a/agent.cpp/src/agent.cpp
++++ b/agent.cpp/src/agent.cpp
+@@ -171,7 +171,7 @@ Agent::build_prompt_tokens()
+
+ auto params = common_chat_templates_apply(model->get_templates(), inputs);
+
+- return model->tokenize(params.prompt);
++ return model->tokenize(params.prompt.str());
+ }
+
+ bool
diff --git a/agent.cpp.patches/patches/src_model.cpp.patch b/agent.cpp.patches/patches/src_model.cpp.patch
index b2a18110..68178d49 100644
--- a/agent.cpp.patches/patches/src_model.cpp.patch
+++ b/agent.cpp.patches/patches/src_model.cpp.patch
@@ -180,17 +180,45 @@ diff --git a/src/model.cpp b/src/model.cpp
true) < 0) {
return {}; // Return empty vector on failure
}
-@@ -261,6 +276,9 @@ parse_with_parser(const common_peg_arena& parser,
+@@ -232,19 +247,6 @@ Model::tokenize(const std::string& prompt) const
+
+ namespace {
+
+-/// Deserializes the PEG parser derived from the chat template. An empty
+-/// definition yields an empty arena, which common_chat_peg_parse() treats as a
+-/// pure-content parser.
+-common_peg_arena
+-load_parser(const std::string& parser_definition)
+-{
+- common_peg_arena arena;
+- if (!parser_definition.empty()) {
+- arena.load(parser_definition);
+- }
+- return arena;
+-}
+-
+ common_chat_msg
+ parse_with_parser(const common_peg_arena& parser,
+ const common_chat_params& params,
+@@ -259,12 +261,14 @@ parse_with_parser(const common_peg_arena& parser,
+ // The parser is derived from the chat template together with the
+ // generation prompt it was built against, and parsing needs both
// (ggml-org/llama.cpp#18675). Without them tool calls stay as raw text.
- syntax.generation_prompt = params.generation_prompt;
- syntax.parse_tool_calls = true;
-+ // llamafile: extract <think>...</think> into reasoning_content instead
-+ // of leaving it inline in content (callers decide how to show it).
-+ syntax.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
+- syntax.generation_prompt = params.generation_prompt;
+- syntax.parse_tool_calls = true;
++ // llamafile: since llama.cpp v0.8.0 tool calls are always parsed, and
++ // reasoning extraction is built into the parser at apply time
++ // (inputs.reasoning_format), so there are no parse-time switches left.
++ syntax.generation_prompt = common_chat_input(params.generation_prompt);
try {
auto parsed_msg =
-@@ -276,6 +294,62 @@ parse_with_parser(const common_peg_arena& parser,
+- common_chat_peg_parse(parser, response, false, syntax);
++ common_chat_peg_parse(parser, common_chat_input(response), false, syntax);
+ parsed_msg.role = "assistant";
+ return parsed_msg;
+ } catch (const std::exception& e) {
+@@ -276,6 +280,62 @@ parse_with_parser(const common_peg_arena& parser,
}
}
@@ -253,7 +281,7 @@ diff --git a/src/model.cpp b/src/model.cpp
}
common_chat_msg
-@@ -288,11 +362,97 @@ Model::generate(const std::vector<common_chat_msg>& messages,
+@@ -288,28 +348,213 @@ Model::generate(const std::vector<common_chat_msg>& messages,
inputs.tools = tools;
inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;
inputs.add_generation_prompt = true;
@@ -261,7 +289,7 @@ diff --git a/src/model.cpp b/src/model.cpp
+ inputs.enable_thinking = config_.enable_thinking; // llamafile: was false
+ // llamafile: build the template's parser with reasoning rules so any
+ // <think>...</think> the model emits anyway is extracted into
-+ // reasoning_content (matching syntax.reasoning_format in the parse).
++ // reasoning_content.
+ inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
auto params =
@@ -350,19 +378,24 @@ diff --git a/src/model.cpp b/src/model.cpp
+ }
+
// Tokenize the prompt
- std::vector<llama_token> prompt_tokens = tokenize(params.prompt);
+- std::vector<llama_token> prompt_tokens = tokenize(params.prompt);
++ std::vector<llama_token> prompt_tokens = tokenize(params.prompt.str());
if (prompt_tokens.empty()) {
-@@ -308,8 +468,114 @@ Model::generate(const std::vector<common_chat_msg>& messages,
- parser_source_ = params.parser;
+ throw ModelError("failed to tokenize prompt");
}
-- return parse_with_parser(
-- parser_arena_, params, response, config_.chat_format);
+ std::string response = generate_from_tokens(prompt_tokens, callback);
+
+- // The PEG parser only changes when the rendered template changes, so keep
+- // the deserialized arena around instead of re-parsing it every turn.
+- if (parser_source_ != params.parser) {
+- parser_arena_ = load_parser(params.parser);
+- parser_source_ = params.parser;
+ common_chat_msg parsed_msg;
+ std::optional<ModelError> parse_error;
+ try {
+ parsed_msg = parse_with_parser(
-+ parser_arena_, params, response, config_.chat_format);
++ params.parser, params, response, config_.chat_format);
+ } catch (const ModelError& e) {
+ // llamafile: a parse refusal must not kill the agent loop. Fall
+ // back to content-only, with the reasoning split off so it is
@@ -375,7 +408,10 @@ diff --git a/src/model.cpp b/src/model.cpp
+ parse_error = e;
+ parsed_msg.content = response;
+ split_reasoning(params, parsed_msg);
-+ }
+ }
+-
+- return parse_with_parser(
+- parser_arena_, params, response, config_.chat_format);
+ // A reply that ends inside a reasoning block it never closed, with
+ // nothing else, holds the model's answer in that block: small models
+ // do this after a tool result. Take it as the content, so a tool call
@@ -469,7 +505,17 @@ diff --git a/src/model.cpp b/src/model.cpp
}
common_chat_msg
-@@ -338,10 +604,16 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
+@@ -317,8 +562,7 @@ parse_response(const common_chat_params& params,
+ const std::string& response,
+ std::optional<common_chat_format> format_override)
+ {
+- return parse_with_parser(
+- load_parser(params.parser), params, response, format_override);
++ return parse_with_parser(params.parser, params, response, format_override);
+ }
+
+ std::string
+@@ -338,10 +582,16 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
common_prefix++;
}
@@ -488,7 +534,7 @@ diff --git a/src/model.cpp b/src/model.cpp
processed_tokens_.resize(common_prefix);
n_past_ = common_prefix;
}
-@@ -370,15 +642,15 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
+@@ -370,15 +620,15 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
i += batch_size;
}
diff --git a/agent.cpp.patches/patches/src_model.h.patch b/agent.cpp.patches/patches/src_model.h.patch
index efd9bfae..f0b8f5d0 100644
--- a/agent.cpp.patches/patches/src_model.h.patch
+++ b/agent.cpp.patches/patches/src_model.h.patch
@@ -27,3 +27,14 @@ diff --git a/src/model.h b/src/model.h
// Owning handles for this Model's LoRA adapters, in ModelConfig::loras
// order. Set on this Model's context only, so different Model instances
// sharing the same ModelWeights can carry independent adapters.
+@@ -236,10 +240,6 @@ class Model
+ std::vector<llama_token> processed_tokens_; // Track tokens in KV cache
+ int n_past_ = 0; // Track position in KV cache
+ ModelConfig config_;
+- // Chat parser derived from the template, cached across turns along with
+- // the definition it was built from
+- std::string parser_source_;
+- common_peg_arena parser_arena_;
+ };
+
+ } // namespace agent_cpp
diff --git a/llama.cpp b/llama.cpp
index 7ab4ee7b..8642541b 160000
--- a/llama.cpp
+++ b/llama.cpp
@@ -1 +1 @@
-Subproject commit 7ab4ee7baad2d920464cbacfad4f4b07cf111fd2
+Subproject commit 8642541b255bfb5dc39cd351a178be09013a55d6
diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.md
index 96d5e109..788c4196 100644
--- a/llama.cpp.patches/README.md
+++ b/llama.cpp.patches/README.md
@@ -46,18 +46,18 @@ The `GGML_CALL` macro (defined as `__attribute__((__ms_abi__))` when `GGML_MULTI
| `ggml_include_ggml-backend.h.patch` | Defines the `GGML_CALL` macro; adds it to the five `get_proc_address` return typedefs (`ggml_backend_split_buffer_type_t`, `ggml_backend_set_n_threads_t`, `ggml_backend_dev_get_extra_bufts_t`, `ggml_backend_set_abort_callback_t`, `ggml_backend_get_features_t`) |
| `ggml_include_ggml-cpu.h.patch` | Adds `GGML_CALL` to declarations of `ggml_backend_cpu_set_n_threads` and `ggml_backend_cpu_set_abort_callback` (returned via `get_proc_address`) |
| `ggml_include_ggml-cuda.h.patch` | Adds `GGML_CALL` to the declaration of `ggml_backend_cuda_register_host_buffer` (upstream removed the row-split multi-GPU buffer in b10052, so the former `ggml_backend_cuda_split_buffer_type` annotation is gone) |
-| `ggml_src_ggml-backend-impl.h.patch` | Adds `GGML_CALL` to all 49+ function pointers across the five interface structs (`ggml_backend_buffer_type_i`, `ggml_backend_buffer_i`, `ggml_backend_i`, `ggml_backend_device_i`, `ggml_backend_reg_i`); also adds `free_struct` callback (see Cross-Module Memory below) |
-| `ggml_src_ggml-backend.cpp.patch` | Adds `GGML_CALL` to CPU buffer, buffer type, and multi-buffer callback implementations; also adds `free_struct` support (see Cross-Module Memory below) |
+| `ggml_src_ggml-backend-impl.h.patch` | Adds `GGML_CALL` to all 49+ function pointers across the five interface structs (`ggml_backend_buffer_type_i`, `ggml_backend_buffer_i`, `ggml_backend_i`, `ggml_backend_device_i`, `ggml_backend_reg_i`), including the optional `alloc_buffer_n`/`get_alloc_size_n` buffer-type members added in v0.8.0; also adds `free_struct` callback (see Cross-Module Memory below) |
+| `ggml_src_ggml-backend.cpp.patch` | Adds `GGML_CALL` to CPU buffer, buffer type, and multi-buffer callback implementations; also adds `free_struct` support (see Cross-Module Memory below), and carries upstream PR #25584 (see Upstream fixes carried ahead of a release) |
| `ggml_src_ggml-cpu_ggml-cpu.cpp.patch` | Adds `GGML_CALL` to all CPU backend, device, and registry callback implementations, plus `get_proc_address`-returned functions (`set_n_threads`, `set_abort_callback`, `get_extra_buffers_type`, `get_features`) |
| `ggml_src_ggml-cpu_amx_amx.cpp.patch` | Adds `GGML_CALL` to all AMX buffer and buffer type callback implementations (10 functions) |
| `ggml_src_ggml-cpu_repack.cpp.patch` | Adds `GGML_CALL` to CPU repack buffer and buffer type callback implementations (5 functions) |
-| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Adds `GGML_CALL` to all CUDA backend callback implementations (60+ functions); also adds `free_struct` and TinyBLAS BF16 guard (see below) |
+| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Adds `GGML_CALL` to all CUDA backend callback implementations (60+ functions, including the pinned host buffer type's `get_max_size`, new in v0.8.0); also adds `free_struct` and TinyBLAS BF16 guard (see below) |
| `ggml_src_ggml-metal_ggml-metal.cpp.patch` | Adds `GGML_CALL` to all Metal backend callback implementations (62 functions); also adds `free_struct` (see below) |
-| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Adds `GGML_CALL` to all Vulkan backend callback implementations; also adds `free_struct` and a heap memory underflow fix (see below) |
+| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Adds `GGML_CALL` to all Vulkan backend callback implementations; also adds `free_struct` (see below) |
| `ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch` | Adds `GGML_CALL` to the backend callback declarations. b11100 split the Vulkan backend into several translation units and moved the interface structs into `ggml-vulkan-buffers.cpp`, so the callbacks it holds became non-static and are declared here; declaration and definition must agree on the convention |
| `ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch` | Adds the `free_struct` entry to `ggml_backend_vk_buffer_interface`, which b11100 moved into this new file |
| `ggml_src_ggml-metal_ggml-metal-fusion.h.patch` / `.cpp.patch` | Adds `GGML_CALL` to the `add_alloc_dep` function pointer these pass through (see Graph-optimize allocation dependencies below), and includes `ggml-backend.h` for the macro |
-| `ggml_src_ggml-backend-meta.cpp.patch` | Adds `GGML_CALL` to all meta-device, meta-buffer-type, meta-buffer, and meta-backend callback implementations (the meta backend aggregates several simple backends behind one interface, so its callbacks are reached through the same function-pointer structs) |
+| `ggml_src_ggml-backend-meta.cpp.patch` | Adds `GGML_CALL` to all meta-device, meta-buffer-type (including its `alloc_buffer_n`, new in v0.8.0), meta-buffer, and meta-backend callback implementations (the meta backend aggregates several simple backends behind one interface, so its callbacks are reached through the same function-pointer structs) |
### Cross-Module Memory Management
@@ -111,8 +111,9 @@ These patches address compatibility issues when building with Cosmopolitan libc
| Patch | Description |
|-------|-------------|
| `common_arg.cpp.patch` | Adds `COSMOCC` platform detection for `PATH_MAX` (includes `linux/limits.h`) |
-| `common_common.cpp.patch` | Adds platform-aware cache directory detection for Cosmopolitan (`fs_get_cache_directory`: checks `LOCALAPPDATA`, `XDG_CACHE_HOME`, falls back to `~/.cache/`) and the same for the config directory (`fs_get_config_directory`, new in b10441: `APPDATA`, `XDG_CONFIG_HOME`, `~/.config/`) — both `#else` out to `#error Unknown architecture` upstream, since cosmocc defines none of `__linux__`/`__APPLE__`/`_WIN32`; also adds mmproj model size estimation to GPU fit params so the fit algorithm reserves enough VRAM for multimodal projectors |
+| `common_common.cpp.patch` | Adds platform-aware cache directory detection for Cosmopolitan (`fs_get_cache_directory`: checks `LOCALAPPDATA`, `XDG_CACHE_HOME`, falls back to `~/.cache/`) and the same for the config directory (`fs_get_config_directory`: `APPDATA`, `XDG_CONFIG_HOME`, `~/.config/`). cosmocc defines none of `__linux__`/`__APPLE__`/`_WIN32`; until v0.8.0 that hit `#error Unknown architecture`, and since then it falls to upstream's generic XDG branch, which compiles but would move a Windows host's cache out of `%LOCALAPPDATA%` and orphan models already downloaded there, so the `COSMOCC` branch stays. (Until v0.8.0 it also added the mmproj file size to every device's fit target. Upstream's server reserves the projector itself since #21489, per device, and llamafile's `--cli`/`--chat` load models without fit, so that hunk only double-counted: on a 24 GB GPU, Gemma-4-31B + mmproj lost 3 offloaded layers. Dropped.) |
| `common_download.cpp.patch` | Adds `COSMOCC` platform detection for `PATH_MAX` |
+| `common_common.h.patch` | Keeps the server's default `--port` at **8080**, llamafile's documented default (help text, docs, integration tests, users' scripts). Upstream moved it to 9931 in v0.8.0 (#30159); this is a deliberate product choice, not a portability fix |
| `vendor_sheredom_subprocess.h.patch` | Under `__COSMOPOLITAN__`, rewrites `subprocess_error_from_errno()`'s `switch` over `E*` values as an `if`/`else` chain. Cosmopolitan resolves errno at runtime (one binary, every host OS), so those are not constant expressions and cannot be case labels — upstream's version fails with `case value is not a constant expression`. Same mapping, no behavior change. Needed since b10441, which added this function; the header only enters the build when `LLAMA_SUBPROCESS` is set (see Server Integration). Also decodes the child's exit status on Windows, where Cosmopolitan's `waitpid()` returns the NT exit code as the wait status: Cosmopolitan children exit with `code << 8`, native ones (`cmd.exe`) with the plain code, so upstream's `WIFEXITED`/`WEXITSTATUS` turned `exit 3` into 1 and `exit 128` into 0. A status with its low byte set is now taken as a native exit code (#1066) |
| `tools_server_server-tools.cpp.patch` | Under `__COSMOPOLITAN__`, picks the shell for the built-in `exec_shell_command` and `get_info` tools at run time with `IsWindows()`: `cmd.exe /c` on Windows, `sh -c` / `uname -a` elsewhere (and always inside a `--tools-runtime` isolate), the same choice upstream makes at compile time with `_WIN32`, which the cosmocc build never defines. The name is spelled `cmd.exe` because Cosmopolitan's `posix_spawnp()` does not try the `.exe` suffix when it searches `PATH`. Without it both tools fail on Windows with *"failed to spawn process"* (#1066) |
@@ -123,7 +124,8 @@ Cosmopolitan libc has specific behaviors with condition variables and signals th
| Patch | Description |
|-------|-------------|
| `common_log.cpp.patch` | Adds `#include <csignal>`; blocks `SIGINT`/`SIGTERM` on logger thread via `pthread_sigmask` to prevent `EINTR` exceptions; replaces the untimed `cv.wait()` calls with `wait_for(30s)` loops (the worker's `cv_new` wait, plus the queue-full `cv_full` waits in `add()` and in `add_json()`, new upstream in b11100) to work around XNU futex timeout bug (~72 minute expiry). The `add*()` waits run on the logging caller's thread, so the signal mask does not cover them |
-| `tools_server_server-models.cpp.patch` | Adds `#include <csignal>`; blocks signals via `pthread_sigmask` on `server_monitor`'s thread (b11100 replaced the per-model `stopping_thread` this used to cover with one monitor thread watching every child); replaces untimed `cv.wait()` with `wait_for(30s)` loops on every model-lifecycle wait (`unload_lru`, the reload-drain wait, `unload_all`, the `is_reloading` guard in `load`, and the generic `wait()` predicate helper) to work around the XNU futex timeout bug |
+| `tools_server_server-models.cpp.patch` | Adds `#include <csignal>`; blocks signals via `pthread_sigmask` on `server_monitor`'s thread (b11100 replaced the per-model `stopping_thread` this used to cover with one monitor thread watching every child); replaces untimed `cv.wait()` with `wait_for(30s)` loops on every model-lifecycle wait (the reload-drain wait, `unload_all`, the `is_reloading` guard in `load`, and the generic `wait()` predicate helper) to work around the XNU futex timeout bug |
+| `tools_server_server-models-sched.cpp.patch` | Same `wait_for(30s)` treatment for the two waits in `server_models_sched::make_room()` (the memory-measurement wait and the LRU-unload wait). v0.8.0 moved the LRU eviction out of `server_models::unload_lru()` into this new scheduler |
| `tools_server_server-queue.cpp.patch` | Adds missing includes (`<cerrno>`, `<system_error>`, `<csignal>`); blocks `SIGINT`/`SIGTERM` on queue thread (the `yield_to_queue` worker thread is spawned after this, so it inherits the mask); replaces `wait()` with `wait_for(30s)` loops in five locations (`wait_until_no_sleep`, main loop, `recv`, plus `worker_loop` and `yield_to_queue`, both new upstream in b10441); runs `yield_to_queue()` inline when a GPU backend is loaded (see "GPU decode must stay on the main thread" below) |
| `tools_server_server-stream.cpp.patch` | Adds `#include <csignal>`; blocks signals on the stream-session GC thread via `pthread_sigmask`. Without it, Ctrl+C on `llama-server` aborts with *"libc++abi: terminating due to uncaught exception ... condition_variable timed_wait failed: Interrupted system call"* instead of shutting down: `SIGINT` is delivered to that thread while it sits in `gc_wake_cv.wait_for(60s)`, `pthread_cond_timedwait()` returns `EINTR`, and libcxx rethrows it as `std::system_error` that nothing catches. This thread was the last one in the server with a condition-variable wait and no signal mask (the log worker, task queue, model-stopping thread and httplib pool were already covered). Pre-existing, not introduced by b10441; combined TUI mode is unaffected |
| `vendor_cpp-httplib_httplib.cpp.patch` | Fixes httplib thread pool with `wait_for()` instead of `wait()` for XNU futex compatibility; passes the full `sockaddr_un` for a pathname Unix socket, since Cosmopolitan's `bind()`/`connect()` reject addresses shorter than `struct sockaddr` and a `--host` socket path under 14 bytes failed with `EINVAL`; also see HTTPS / TLS Support below |
@@ -186,7 +188,7 @@ These patches restore llamafile's optimized CPU kernels (TinyBLAS matmul, AVX-51
| Patch | Description |
|-------|-------------|
-| `ggml_src_ggml-cpu_ggml-cpu.c.patch` | Routes MoE matmul (`ggml_compute_forward_mul_mat_id`) through `llamafile_mixmul` / `llamafile_mixmul_iqk`, mirroring the dense-matmul `llamafile_sgemm` hook; reserves work-buffer space for the MoE kernel in `ggml_graph_plan` via `llamafile_mixmul_needs` |
+| `ggml_src_ggml-cpu_ggml-cpu.c.patch` | Routes MoE matmul (`ggml_compute_forward_mul_mat_id`) through `llamafile_mixmul` / `llamafile_mixmul_iqk`, mirroring the dense-matmul `llamafile_sgemm` hook; reserves work-buffer space for the MoE kernel in `ggml_graph_plan` via `llamafile_mixmul_needs` (in the `GGML_OP_MUL_MAT_ID` case, after upstream's tiled workspace; on the v0.8.0 bump the hunk's context drifted and `patch` silently placed it in `GGML_OP_OUT_PROD`, so check its position on each bump). v0.8.0 replaced the IQ panel path (`iqp.cpp`) with a tiled k-quant matmul (`ggml-cpu/tiled/`) that runs before these hooks, but it is compiled only under `__AVX__`/`__AVX2__`/`__AVX512VNNI__` and the cosmocc build compiles `ggml-cpu` at baseline ISA, so in llamafile it always declines and the TinyBLAS hooks keep handling those matmuls |
| `ggml_src_ggml-cpu_ops.cpp.patch` | Routes flash-attention inner loops through llamafile's AVX-512 helpers (`llamafile_fa_vec_dot_f16`, `llamafile_fa_fp16_to_fp32_row`, `llamafile_fa_simd_gemm`) in both the one-chunk and tiled FA paths; also accumulates VKQ in f32 on CPUs lacking native f16 FMA (avoiding costly f16↔f32 round-trips per KV step) |
| `src_llama-context.cpp.patch` | Defaults `-fa auto` to **off** on CPU-only setups (no GPU devices), since the CPU flash-attention path is slower than the non-FA path on x86; users can still force `-fa on` for memory savings on long contexts |
@@ -197,15 +199,15 @@ These patches integrate llamafile's file handling APIs for loading models from b
| Patch | Description |
|-------|-------------|
| `src_llama-mmap.h.patch` | Adds `has_premapped_content()`, `premapped_content()`, and `get_llamafile()` methods to `llama_file` class |
-| `src_llama-mmap.cpp.patch` | Under `COSMOCC`, redirects file open/read/seek/tell/close through llamafile API (`llamafile_open_gguf`, `llamafile_read`, etc.); adds premapped content support to `llama_mmap` using llamafile reference counting (`llamafile_ref`/`llamafile_unref`); skips `munmap` for premapped content |
+| `src_llama-mmap.cpp.patch` | Under `COSMOCC`, redirects file open/read/seek/tell/close through llamafile API (`llamafile_open_gguf`, `llamafile_read`, etc.); adds premapped content support to `llama_mmap` using llamafile reference counting (`llamafile_ref`/`llamafile_unref`); skips `munmap` for premapped content; on Windows makes `unmap_fragment()` a no-op, as upstream's `_WIN32` branch does (cosmocc never defines `_WIN32`, so llamafile runs the POSIX branch everywhere, and Windows can only unmap a whole view: cosmo's `munmap` fails with `ENOTSUP`/`EINVAL` on a sub-range, which printed "munmap failed" warnings and left the view mapped at exit). Memory use then matches the official llama.cpp Windows build: the whole file stays mapped after GPU offload in both |
| `ggml_src_gguf.cpp.patch` | Adds `tell()`/`seek()` to `gguf_reader`; under `COSMOCC`, adds `gguf_llamafile_reader` that reads via llamafile API; templatizes `gguf_init_from_reader_impl` so both readers work; redirects `gguf_init_from_file` through `llamafile_open_gguf` (supports `/zip/` paths, `.llamafile` containers) |
### Server Integration
| Patch | Description |
|-------|-------------|
-| `tools_server_server.cpp.patch` | Renames upstream's `llama_server()` to `server_main()` and adds `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode; adds Metal/GPU backend trigger before `common_init()`; installs the sandbox (`llamafile_sandbox_server()`, issue #930) — the mechanism lives in `llamafile/sandbox.c`; the patch fills a `llamafile_sandbox_spec` with the on-disk paths the loader opens after the lock (model, mmproj, media dir, LoRA, draft model, control vectors, public path as read; slot-save and prompt cache as read-write), detects outbound-needing features (`--rpc` via argv/env, MCP proxy, server tools) to relax `anet`→`inet`, passes `FLAG_confine_reads` for opt-in unveil(), binds the listening socket first (`ctx_http.bind_socket()`, so a `--host /path.sock` Unix socket exists before `anet` forbids creating one, issue #971), quiesces the log worker (`common_log_pause`/`resume`, so no thread escapes the per-thread filter) around the call, before the HTTP listener spawns and before model load, and skips it in combined/GPU modes; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, `--unsecure` consumption (`llamafile_consume_flag`), and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |
-| `tools_server_server-http.h.patch` / `tools_server_server-http.cpp.patch` | Splits the bind out of `server_http_context::start()` into `bind_socket()`, which `start()` calls if it hasn't run yet, so the server can bind before installing the sandbox (issue #971) |
+| `tools_server_server.cpp.patch` | Makes `server_main()` the real entry point (upstream's `llama_server(argc, argv)` becomes a thin wrapper around it) and adds `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode. v0.8.0 (#31113) split the server into `server_run_router()` / `server_run_instance()` / `server_run()`; the callbacks (and `argv`, for `--rpc` detection) now travel in upstream's `server_run_opts`, and `on_ready`, `on_shutdown_available`, and the sandbox all live in `server_run_instance()` (the router stays unsandboxed, as before). The sandbox is also skipped under `--connect` (new in v0.8.0), which spawns `llama-connect` after the lock and would otherwise fail to start; adds Metal/GPU backend trigger before `common_init()`; installs the sandbox (`llamafile_sandbox_server()`, issue #930) — the mechanism lives in `llamafile/sandbox.c`; the patch fills a `llamafile_sandbox_spec` with the on-disk paths the loader opens after the lock (model, mmproj, media dir, LoRA, draft model, control vectors, public path as read; slot-save and prompt cache as read-write), detects outbound-needing features (`--rpc` via argv/env, MCP proxy, server tools) to relax `anet`→`inet`, passes `FLAG_confine_reads` for opt-in unveil(), binds the listening sockets first (`ctx_http.bind_socket()`, so a `--host /path.sock` Unix socket exists before `anet` forbids creating one, issue #971), quiesces the log worker (`common_log_pause`/`resume`, so no thread escapes the per-thread filter) around the call, before the HTTP listener spawns and before model load, and skips it in combined/GPU modes; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, `--unsecure` consumption (`llamafile_consume_flag`), and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |
+| `tools_server_server-http.h.patch` / `tools_server_server-http.cpp.patch` | Splits the bind out of `server_http_context::start()` into `bind_socket()`, which `start()` calls if it hasn't run yet, so the server can bind before installing the sandbox (issue #971). Since v0.8.0 `start()` binds every listener (one per `--host`) before spawning the HTTP thread pool, and `bind_socket()` does exactly that first part |
**GPU decode must stay on the main thread** (`tools_server_server-queue.cpp.patch`).
b10441 moved `llama_decode()` off the main thread: `update_slots()` now hands
@@ -333,16 +335,15 @@ once upstream has the same change.
| Patch | Description |
|-------|-------------|
-| `ggml_src_ggml-alloc.c.patch` | [PR #25584](https://github.com/ggml-org/llama.cpp/pull/25584), verbatim minus its test. `ggml_backend_alloc_ctx_tensors_from_buft()` splits a context across buffers when a tensor exceeds the backend's `max_size` (1 GiB on Vulkan); when the tail of the context holds only views, the final `alloc_tensor_range()` is skipped and those views never get `ggml_backend_view_init()`. The persistent KV stream views (`layer.k_stream`/`v_stream`) then keep `data == NULL`, and the server's state save/restore hits `GGML_ASSERT(tensor->data != NULL && "tensor not allocated")` in `ggml_backend_tensor_get()`. The fix allocates only parent tensors per split range and initializes every view in one final pass. **Symptom without it:** `llama-server` on Vulkan dies on the *second* request at the default `--parallel 4` (upstream [#29221](https://github.com/ggml-org/llama.cpp/issues/29221); `--parallel 1` is the workaround). Reproduced on an L40S with vanilla b11100, so it is not llamafile-specific; verified fixed on both vanilla and llamafile. Upstream also files it against #19839, #23737 and #21762. |
+| `ggml_src_ggml-backend.cpp.patch` (part) | [PR #25584](https://github.com/ggml-org/llama.cpp/pull/25584), still open at v0.8.0. When a buffer type has a `max_size` (1 GiB on Vulkan), the allocation of a context's tensors is split across several buffers; when a tensor larger than `max_size` is followed only by views, the zero-sized tail is never planned as a buffer range and those views never get `ggml_backend_view_init()`. The persistent KV stream views (`layer.k_stream`/`v_stream`) then keep `data == NULL`, and the server's state save/restore hits `GGML_ASSERT(tensor->data != NULL && "tensor not allocated")` in `ggml_backend_tensor_get()`. The fix allocates only parent tensors per range and initializes every view in one final pass. Up to b11100 the splitting lived in `ggml-alloc.c` (where the PR applies, as `ggml_src_ggml-alloc.c.patch`); v0.8.0 (#23671) moved it, bug included, into `ggml_backend_buft_alloc_buffer_n_default()` in `ggml-backend.cpp`, so the same fix is now carried there. **Symptom without it:** `llama-server` on Vulkan dies on the *second* request at the default `--parallel 4` (upstream [#29221](https://github.com/ggml-org/llama.cpp/issues/29221); `--parallel 1` is the workaround). Reproduced on an L40S with vanilla b11100, so it is not llamafile-specific; verified fixed on both vanilla and llamafile at b11100. Upstream also files it against #19839, #23737 and #21762. |
### Bug Fixes
| Patch | Description |
|-------|-------------|
-| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach) |
-| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Fixes unsigned integer underflow in `ggml_backend_vk_get_device_memory` where Vulkan's `heapUsage` can exceed `heapBudget` (clamps to zero instead of wrapping) |
+| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach). Since v0.8.0 (#28367) a missing path surfaces as a `directory_iterator` `ENOENT`, so only that error is silenced; other enumeration failures are still logged |
| `src_models_t5.cpp.patch` | Forward-declares the `graph<false>`/`graph<true>` explicit specializations before `build_arch_graph` so clang's `-std=gnu++23` doesn't reject them as specializations after implicit instantiation |
-| `ggml_src_ggml.c.patch` | Makes the `ggml_time_ms()`/`ggml_time_us()` call `ggml_time_init()` lazily. The Windows dylibs link their own copy of `ggml.c`, so the DLL's `timer_freq` is never set by the executable's `ggml_init()` and any `ggml_time_*()` call made from inside the DLL divides by zero (`0xC0000094`, surfaced by Cosmopolitan as SIGFPE). First hit when #1051 enabled `-DGGML_CUDA_USE_GRAPHS`, calling `ggml_time_us()` in `ggml-cuda`. Linux/macOS are unaffected (`clock_gettime`, no static divisor). |
+| `ggml_src_ggml.c.patch` | Makes the `ggml_time_ms()`/`ggml_time_us()` call `ggml_time_init()` lazily. The Windows dylibs link their own copy of `ggml.c`, so the DLL's `timer_freq` is never set by the executable's `ggml_init()` and any `ggml_time_*()` call made from inside the DLL divides by zero (`0xC0000094`, surfaced by Cosmopolitan as SIGFPE). First hit when #1051 enabled `-DGGML_CUDA_USE_GRAPHS`, calling `ggml_time_us()` in `ggml-cuda`. Linux/macOS are unaffected (`clock_gettime`, no static divisor). v0.8.0 (#30969) made the division floating-point, so it no longer traps, but `inf` cast to `int64_t` is undefined (INT64_MIN on x86) and every DLL-side timing would be garbage, so the lazy init is still needed. |
The same clang `-std=gnu++23` problem used to need patches for `eagle3.cpp` and
`dflash.cpp`; b11100 defines `build_arch_graph` at the end of both files, after
diff --git a/llama.cpp.patches/fetch-ui-assets.sh b/llama.cpp.patches/fetch-ui-assets.sh
index c81257e2..9c9dfea6 100755
--- a/llama.cpp.patches/fetch-ui-assets.sh
+++ b/llama.cpp.patches/fetch-ui-assets.sh
@@ -125,7 +125,9 @@ echo "Fetching prebuilt web UI assets from Hugging Face..."
# build number. "latest" is kept only as a last-resort fallback (enumeration
# failed, e.g. offline / API change, or our commit predates every published
# tag) so the build can still get *some* UI rather than none.
-UI_CUR_BUILD="$(cd "$LLAMA_DIR" && git describe --tags --always 2>/dev/null \
+# --match: upstream also tags releases as vX.Y.Z, often on the same commit as a
+# bNNNN tag, and a plain describe then prefers it; we need the build number.
+UI_CUR_BUILD="$(cd "$LLAMA_DIR" && git describe --tags --match 'b[0-9]*' --always 2>/dev/null \
| grep -oE '^b[0-9]+' | grep -oE '[0-9]+' || true)"
UI_CANDIDATES=()
diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mk
index 9e670287..68cf7fe8 100644
--- a/llama.cpp.patches/llamafile-files/BUILD.mk
+++ b/llama.cpp.patches/llamafile-files/BUILD.mk
@@ -34,7 +34,8 @@ GGML_SRCS_CPP := \
llama.cpp/ggml/src/ggml-cpu/binary-ops.cpp \
llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp \
llama.cpp/ggml/src/ggml-cpu/hbm.cpp \
- llama.cpp/ggml/src/ggml-cpu/iqp.cpp \
+ llama.cpp/ggml/src/ggml-cpu/tiled/tiled.cpp \
+ llama.cpp/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp \
llama.cpp/ggml/src/ggml-cpu/ops.cpp \
llama.cpp/ggml/src/ggml-cpu/repack.cpp \
llama.cpp/ggml/src/ggml-cpu/traits.cpp \
@@ -52,6 +53,7 @@ GGML_OBJS := \
# ==============================================================================
LLAMA_SRCS_CPP := \
+ llama.cpp/src/llama-moe-cache.cpp \
llama.cpp/src/llama.cpp \
llama.cpp/src/models/afmoe.cpp \
llama.cpp/src/models/apertus.cpp \
@@ -62,11 +64,13 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/bailingmoe.cpp \
llama.cpp/src/models/bailingmoe2.cpp \
llama.cpp/src/models/bailingmoe3.cpp \
+ llama.cpp/src/models/beam.cpp \
llama.cpp/src/models/bert.cpp \
llama.cpp/src/models/bitnet.cpp \
llama.cpp/src/models/bloom.cpp \
llama.cpp/src/models/chameleon.cpp \
llama.cpp/src/models/chatglm.cpp \
+ llama.cpp/src/models/clef.cpp \
llama.cpp/src/models/clip.cpp \
llama.cpp/src/models/codeshell.cpp \
llama.cpp/src/models/cogvlm.cpp \
@@ -80,6 +84,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/deepseek2ocr.cpp \
llama.cpp/src/models/deepseek32.cpp \
llama.cpp/src/models/deepseek4.cpp \
+ llama.cpp/src/models/deepseek41.cpp \
llama.cpp/src/models/delta-net-base.cpp \
llama.cpp/src/models/dflash.cpp \
llama.cpp/src/models/dots1.cpp \
@@ -94,7 +99,9 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/exaone-moe.cpp \
llama.cpp/src/models/falcon-h1.cpp \
llama.cpp/src/models/falcon.cpp \
+ llama.cpp/src/models/flint.cpp \
llama.cpp/src/models/gemma-embedding.cpp \
+ llama.cpp/src/models/gemma-embedding2.cpp \
llama.cpp/src/models/gemma.cpp \
llama.cpp/src/models/gemma2.cpp \
llama.cpp/src/models/gemma3.cpp \
@@ -104,6 +111,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/glm-dsa.cpp \
llama.cpp/src/models/glm4-moe.cpp \
llama.cpp/src/models/glm4.cpp \
+ llama.cpp/src/models/glm5-next.cpp \
llama.cpp/src/models/gpt2.cpp \
llama.cpp/src/models/gptneox.cpp \
llama.cpp/src/models/granite-hybrid.cpp \
@@ -111,6 +119,11 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/granite-switch.cpp \
llama.cpp/src/models/granite.cpp \
llama.cpp/src/models/granite-swa.cpp \
+ llama.cpp/src/models/k2-horizon.cpp \
+ llama.cpp/src/models/kolibri1.cpp \
+ llama.cpp/src/models/lfm-base.cpp \
+ llama.cpp/src/models/lfm3.cpp \
+ llama.cpp/src/models/maion-coder.cpp \
llama.cpp/src/models/mamba-base.cpp \
llama.cpp/src/models/grok.cpp \
llama.cpp/src/models/grovemoe.cpp \
@@ -152,6 +165,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/modern-bert.cpp \
llama.cpp/src/models/mpt.cpp \
llama.cpp/src/models/muse-glimmer.cpp \
+ llama.cpp/src/models/muse-spark.cpp \
llama.cpp/src/models/nanbeige.cpp \
llama.cpp/src/models/nemotron-h-moe.cpp \
llama.cpp/src/models/nemotron-h.cpp \
@@ -196,6 +210,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/rwkv7-base.cpp \
llama.cpp/src/models/rwkv7.cpp \
llama.cpp/src/models/seed-oss.cpp \
+ llama.cpp/src/models/sensenova-u1.cpp \
llama.cpp/src/models/smallthinker.cpp \
llama.cpp/src/models/smollm3.cpp \
llama.cpp/src/models/spark2-5.cpp \
@@ -206,6 +221,7 @@ LLAMA_SRCS_CPP := \
llama.cpp/src/models/t5.cpp \
llama.cpp/src/models/t5encoder.cpp \
llama.cpp/src/models/talkie.cpp \
+ llama.cpp/src/models/vllm-decision2.cpp \
llama.cpp/src/models/wavtokenizer-dec.cpp \
llama.cpp/src/models/xverse.cpp \
llama.cpp/src/llama-adapter.cpp \
@@ -257,6 +273,7 @@ COMMON_SRCS_CPP := \
llama.cpp/common/console.cpp \
llama.cpp/common/debug.cpp \
llama.cpp/common/download.cpp \
+ llama.cpp/common/file-lock.cpp \
llama.cpp/common/fit.cpp \
llama.cpp/common/hf-cache.cpp \
llama.cpp/common/imatrix-loader.cpp \
@@ -275,6 +292,10 @@ COMMON_SRCS_CPP := \
llama.cpp/common/ngram-cache.cpp \
llama.cpp/common/ngram-map.cpp \
llama.cpp/common/ngram-mod.cpp \
+ llama.cpp/common/parsers/harmony-xml.cpp \
+ llama.cpp/common/parsers/k2-horizon.cpp \
+ llama.cpp/common/parsers/llm-jp-harmony.cpp \
+ llama.cpp/common/parsers/muse-spark.cpp \
llama.cpp/common/parsers/parsers.cpp \
llama.cpp/common/parsers/cohere2moe.cpp \
llama.cpp/common/parsers/deepseek.cpp \
@@ -291,6 +312,7 @@ COMMON_SRCS_CPP := \
llama.cpp/common/parsers/ministral3.cpp \
llama.cpp/common/parsers/muse-glimmer.cpp \
llama.cpp/common/parsers/qwen3-coder.cpp \
+ llama.cpp/common/parsers/translate-gemma.cpp \
llama.cpp/common/peg-parser.cpp \
llama.cpp/common/preset.cpp \
llama.cpp/common/reasoning-budget.cpp \
@@ -371,6 +393,10 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/internvl.cpp \
llama.cpp/tools/mtmd/models/kimik25.cpp \
llama.cpp/tools/mtmd/models/kimivl.cpp \
+ llama.cpp/tools/mtmd/models/lfm3a-gen.cpp \
+ llama.cpp/tools/mtmd/models/lfm3a.cpp \
+ llama.cpp/tools/mtmd/models/lfm3vl.cpp \
+ llama.cpp/tools/mtmd/models/ling3vl.cpp \
llama.cpp/tools/mtmd/models/llama4.cpp \
llama.cpp/tools/mtmd/models/llava.cpp \
llama.cpp/tools/mtmd/models/mimo-audio.cpp \
@@ -379,6 +405,8 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/minimax-m3.cpp \
llama.cpp/tools/mtmd/models/mobilenetv5.cpp \
llama.cpp/tools/mtmd/models/muse-glimmer.cpp \
+ llama.cpp/tools/mtmd/models/muse-spark-audio.cpp \
+ llama.cpp/tools/mtmd/models/nemotron-omni.cpp \
llama.cpp/tools/mtmd/models/nemotron-v2-vl.cpp \
llama.cpp/tools/mtmd/models/paddleocr.cpp \
llama.cpp/tools/mtmd/models/parakeet.cpp \
@@ -391,8 +419,11 @@ MTMD_SRCS_CPP := \
llama.cpp/tools/mtmd/models/qwen3tts-gen.cpp \
llama.cpp/tools/mtmd/models/qwen3tts-spkenc.cpp \
llama.cpp/tools/mtmd/models/qwen3vl.cpp \
+ llama.cpp/tools/mtmd/models/sensenova-u1.cpp \
llama.cpp/tools/mtmd/models/siglip.cpp \
+ llama.cpp/tools/mtmd/models/soprano.cpp \
llama.cpp/tools/mtmd/models/step3vl.cpp \
+ llama.cpp/tools/mtmd/models/voxtral-rt.cpp \
llama.cpp/tools/mtmd/models/whisper-enc.cpp \
llama.cpp/tools/mtmd/models/yasa2.cpp \
llama.cpp/tools/mtmd/models/youtuvl.cpp
@@ -497,6 +528,10 @@ TOOL_BENCH_SRCS := \
llama.cpp/tools/llama-bench/main.cpp
TOOL_SERVER_SRCS := \
+ llama.cpp/tools/server/server-connect.cpp \
+ llama.cpp/tools/server/server-decision.cpp \
+ llama.cpp/tools/server/server-models-sched.cpp \
+ llama.cpp/tools/server/server-realtime.cpp \
llama.cpp/tools/server/server.cpp \
llama.cpp/tools/server/server-chat.cpp \
llama.cpp/tools/server/server-common.cpp \
diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patch
index 1b0565e0..b775f70a 100644
--- a/llama.cpp.patches/patches/common_common.cpp.patch
+++ b/llama.cpp.patches/patches/common_common.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/common.cpp b/common/common.cpp
--- a/llama.cpp/common/common.cpp
+++ b/llama.cpp/common/common.cpp
-@@ -34,6 +34,10 @@
+@@ -39,6 +39,10 @@
#include <sys/sysctl.h>
#endif
@@ -12,7 +12,7 @@ diff --git a/common/common.cpp b/common/common.cpp
#if defined(_WIN32)
#define WIN32_LEAN_AND_MEAN
#ifndef NOMINMAX
-@@ -76,7 +80,38 @@ common_time_meas::~common_time_meas() {
+@@ -80,7 +84,38 @@ common_time_meas::~common_time_meas() {
//
int32_t common_cpu_get_num_physical_cores() {
@@ -52,81 +52,37 @@ diff --git a/common/common.cpp b/common/common.cpp
int32_t logical_cpus = _system_configuration.ncpus;
int32_t smt_threads = _system_configuration.smt_threads;
if (smt_threads > 0) {
-@@ -1059,6 +1094,20 @@ std::string fs_get_cache_directory() {
- if (cache_directory.empty()) {
- throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
- }
+@@ -1007,6 +1042,16 @@ std::filesystem::path fs_get_cache_directory() {
+ if (cache_directory.empty()) {
+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");
+ }
+#elif defined(COSMOCC)
-+ // We don't know what OS we are running on at compile time, just CPU
-+ // architecture. Try various environment variables, fall back to ~/.cache.
-+ cache_directory = common_get_env("LOCALAPPDATA");
-+ if (cache_directory.empty()) {
-+ cache_directory = common_get_env("XDG_CACHE_HOME");
-+ }
-+ if (cache_directory.empty()) {
-+ const std::string home = common_get_env("HOME");
-+ if (home.empty()) {
-+ throw std::runtime_error("Failed to find $HOME directory");
-+ }
-+ cache_directory = home + "/.cache/";
-+ }
- #elif defined(__EMSCRIPTEN__)
- GGML_ABORT("not implemented on this platform");
++ // We don't know what OS we are running on at compile time, just CPU
++ // architecture. Try various environment variables, fall back to ~/.cache.
++ cache_directory = common_get_path_from_env("LOCALAPPDATA");
++ if (cache_directory.empty()) {
++ cache_directory = common_get_path_from_env("XDG_CACHE_HOME");
++ }
++ if (cache_directory.empty()) {
++ cache_directory = get_home_directory() / ".cache";
++ }
+ #elif defined(__APPLE__)
+ cache_directory = get_home_directory() / "Library/Caches";
#else
-@@ -1104,6 +1153,20 @@ std::string fs_get_config_directory() {
+@@ -1025,6 +1070,16 @@ std::filesystem::path fs_get_config_directory() {
if (config_directory.empty()) {
throw std::runtime_error("Failed to find %APPDATA% directory");
}
+#elif defined(COSMOCC)
+ // We don't know what OS we are running on at compile time, just CPU
+ // architecture. Try various environment variables, fall back to ~/.config.
-+ config_directory = common_get_env("APPDATA");
++ config_directory = common_get_path_from_env("APPDATA");
+ if (config_directory.empty()) {
-+ config_directory = common_get_env("XDG_CONFIG_HOME");
++ config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
+ }
+ if (config_directory.empty()) {
-+ const std::string home = common_get_env("HOME");
-+ if (home.empty()) {
-+ throw std::runtime_error("Failed to find $HOME directory");
-+ }
-+ config_directory = home + "/.config/";
++ config_directory = get_home_directory() / ".config";
+ }
- #elif defined(__EMSCRIPTEN__)
- // caller decides what to do when there is no config directory
- throw std::runtime_error("not implemented on this platform");
-@@ -1296,6 +1359,26 @@ common_init_result::common_init_result(common_params & params, bool model_only)
- COM_TRC("%s", "fitting params to device memory ...\n");
- COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");
-
-+ // if a multimodal projection model is specified and will be loaded on GPU,
-+ // add its estimated size to the fit margins so the fit leaves enough room for it
-+ std::vector<size_t> fit_params_target = params.fit_params_target;
-+ if (!params.mmproj.path.empty() && params.mmproj_use_gpu) {
-+ struct gguf_init_params gparams = { /*.no_alloc =*/ true, /*.ctx =*/ nullptr };
-+ struct gguf_context * gctx = gguf_init_from_file(params.mmproj.path.c_str(), gparams);
-+ if (gctx != nullptr) {
-+ size_t mmproj_size = 0;
-+ for (int64_t i = 0; i < gguf_get_n_tensors(gctx); i++) {
-+ mmproj_size += gguf_get_tensor_size(gctx, i);
-+ }
-+ gguf_free(gctx);
-+ LOG_INF("%s: reserving %.2f MiB extra for mmproj model '%s'\n", __func__,
-+ mmproj_size / (1024.0 * 1024.0), params.mmproj.path.c_str());
-+ for (size_t & target : fit_params_target) {
-+ target += mmproj_size;
-+ }
-+ }
-+ }
-+
- // the draft context is created from the same base params and follows the main context, fit both together
- const bool has_draft = params.speculative.has_dft();
- const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),
-@@ -1320,7 +1403,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
- common_fit_params(params.model.path.c_str(), &mparams, &cparams,
- params.tensor_split,
- params.tensor_buft_overrides.data(),
-- params.fit_params_target.data(),
-+ fit_params_target.data(),
- params.fit_params_min_ctx,
- has_draft || spec_mtp ? &extra : nullptr,
- params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);
+ #else
+ config_directory = common_get_path_from_env("XDG_CONFIG_HOME");
+ if (config_directory.empty()) {
diff --git a/llama.cpp.patches/patches/common_common.h.patch b/llama.cpp.patches/patches/common_common.h.patch
new file mode 100644
index 00000000..739457a6
--- /dev/null
+++ b/llama.cpp.patches/patches/common_common.h.patch
@@ -0,0 +1,12 @@
+diff --git a/common/common.h b/common/common.h
+--- a/llama.cpp/common/common.h
++++ b/llama.cpp/common/common.h
+@@ -629,7 +629,7 @@ struct common_params {
+ std::string cls_sep = "\t"; // separator of classification sequences
+
+ // server params
+- int32_t port = 9931; // server listens on this network port
++ int32_t port = 8080; // server listens on this network port (llamafile: keeps its documented 8080, upstream moved to 9931 in #30159)
+ bool reuse_port = false; // allow multiple sockets to bind to the same port
+ int32_t timeout_read = 3600; // http read timeout in seconds
+ int32_t timeout_write = timeout_read; // http write timeout in seconds
diff --git a/llama.cpp.patches/patches/common_download.cpp.patch b/llama.cpp.patches/patches/common_download.cpp.patch
index a731fcb1..ec9a78e8 100644
--- a/llama.cpp.patches/patches/common_download.cpp.patch
+++ b/llama.cpp.patches/patches/common_download.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/common/download.cpp b/common/download.cpp
--- a/llama.cpp/common/download.cpp
+++ b/llama.cpp/common/download.cpp
-@@ -24,6 +24,8 @@
+@@ -25,6 +25,8 @@
#ifndef __EMSCRIPTEN__
#ifdef __linux__
#include <linux/limits.h>
diff --git a/llama.cpp.patches/patches/common_http.h.patch b/llama.cpp.patches/patches/common_http.h.patch
index a598abda..3f628d16 100644
--- a/llama.cpp.patches/patches/common_http.h.patch
+++ b/llama.cpp.patches/patches/common_http.h.patch
@@ -1,7 +1,7 @@
diff --git a/common/http.h b/common/http.h
--- a/llama.cpp/common/http.h
+++ b/llama.cpp/common/http.h
-@@ -40,10 +40,15 @@ static common_http_url common_http_parse_url(const std::string & url) {
+@@ -42,10 +42,15 @@ static common_http_url common_http_parse_url(const std::string & url) {
}
auto rest = url.substr(scheme_end + 3);
@@ -19,7 +19,7 @@ diff --git a/common/http.h b/common/http.h
auto colon_pos = auth.find(':');
if (colon_pos != std::string::npos) {
parts.user = auth.substr(0, colon_pos);
-@@ -51,18 +56,14 @@ static common_http_url common_http_parse_url(const std::string & url) {
+@@ -53,18 +58,14 @@ static common_http_url common_http_parse_url(const std::string & url) {
} else {
parts.user = auth;
}
@@ -43,7 +43,7 @@ diff --git a/common/http.h b/common/http.h
// split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986)
std::string port_str;
-@@ -74,6 +75,8 @@ static common_http_url common_http_parse_url(const std::string & url) {
+@@ -76,6 +77,8 @@ static common_http_url common_http_parse_url(const std::string & url) {
auto after = parts.host.substr(close + 1);
if (!after.empty() && after.front() == ':') {
port_str = after.substr(1);
@@ -52,7 +52,7 @@ diff --git a/common/http.h b/common/http.h
}
parts.host = parts.host.substr(1, close - 1);
} else {
-@@ -85,7 +88,14 @@ static common_http_url common_http_parse_url(const std::string & url) {
+@@ -87,7 +90,14 @@ static common_http_url common_http_parse_url(const std::string & url) {
}
if (!port_str.empty()) {
@@ -67,7 +67,7 @@ diff --git a/common/http.h b/common/http.h
} else if (parts.scheme == "http") {
parts.port = 80;
} else if (parts.scheme == "https") {
-@@ -104,7 +114,7 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:
+@@ -218,7 +228,7 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:
throw std::runtime_error("error: invalid URL format");
}
diff --git a/llama.cpp.patches/patches/common_log.cpp.patch b/llama.cpp.patches/patches/common_log.cpp.patch
index e80ea789..99ebfa41 100644
--- a/llama.cpp.patches/patches/common_log.cpp.patch
+++ b/llama.cpp.patches/patches/common_log.cpp.patch
@@ -1,15 +1,18 @@
diff --git a/common/log.cpp b/common/log.cpp
--- a/llama.cpp/common/log.cpp
+++ b/llama.cpp/common/log.cpp
-@@ -25,6 +25,7 @@
- # define fileno _fileno
- #else
- # include <unistd.h>
-+# include <csignal>
- #endif // defined(_WIN32)
+@@ -14,6 +14,10 @@
+ #include <vector>
+ #include <algorithm>
++#if !defined(_WIN32)
++# include <csignal>
++#endif
++
int common_log_verbosity_thold = LOG_DEFAULT_LLAMA;
-@@ -243,7 +244,13 @@ public:
+
+ int common_log_get_verbosity_thold(void) {
+@@ -240,7 +244,13 @@ public:
std::unique_lock<std::mutex> lock(mtx);
// block if the queue is full
@@ -24,7 +27,7 @@ diff --git a/common/log.cpp b/common/log.cpp
if (!running) {
// discard messages while the worker thread is paused
-@@ -310,7 +317,13 @@ public:
+@@ -307,7 +317,13 @@ public:
std::unique_lock<std::mutex> lock(mtx);
// block if the queue is full
@@ -39,7 +42,7 @@ diff --git a/common/log.cpp b/common/log.cpp
if (!running) {
// discard messages while the worker thread is paused
-@@ -345,9 +358,32 @@ public:
+@@ -342,9 +358,32 @@ public:
running = true;
thrd = std::thread([this]() {
diff --git a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
index 013b0f23..5271f6ca 100644
--- a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
+++ b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch
@@ -18,7 +18,7 @@ diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
#ifdef __cplusplus
extern "C" {
#endif
-@@ -210,19 +220,19 @@ extern "C" {
+@@ -212,19 +222,19 @@ extern "C" {
typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);
// Split buffer type for tensor parallelism (old)
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patch b/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patch
deleted file mode 100644
index 05ca1a25..00000000
--- a/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patch
+++ /dev/null
@@ -1,67 +0,0 @@
-diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c
---- a/llama.cpp/ggml/src/ggml-alloc.c
-+++ b/llama.cpp/ggml/src/ggml-alloc.c
-@@ -1124,6 +1124,20 @@ static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buff
- free(*buffers);
- }
-
-+static bool alloc_ctx_tensors_finalize_views(struct ggml_context * ctx) {
-+ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {
-+ if (t->view_src != NULL && t->buffer == NULL) {
-+ enum ggml_status status = ggml_backend_view_init(t);
-+ if (status != GGML_STATUS_SUCCESS) {
-+ GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);
-+ return false;
-+ }
-+ }
-+ }
-+
-+ return true;
-+}
-+
- static bool alloc_tensor_range(struct ggml_context * ctx,
- struct ggml_tensor * first, struct ggml_tensor * last,
- ggml_backend_buffer_type_t buft, size_t size,
-@@ -1142,24 +1156,14 @@ static bool alloc_tensor_range(struct ggml_context * ctx,
- struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
-
- for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {
-- enum ggml_status status = GGML_STATUS_SUCCESS;
-- if (t->data == NULL) {
-- if (t->view_src == NULL) {
-- status = ggml_tallocr_alloc(&tallocr, t);
-- } else if (t->buffer == NULL) {
-- status = ggml_backend_view_init(t);
-- }
-- } else {
-- if (t->view_src != NULL && t->buffer == NULL) {
-- // view of a pre-allocated tensor
-- status = ggml_backend_view_init(t);
-+ if (t->view_src == NULL && t->data == NULL) {
-+ enum ggml_status status = ggml_tallocr_alloc(&tallocr, t);
-+ if (status != GGML_STATUS_SUCCESS) {
-+ GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t->name);
-+ free_buffers(buffers, n_buffers);
-+ return false;
- }
- }
-- if (status != GGML_STATUS_SUCCESS) {
-- GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name);
-- free_buffers(buffers, n_buffers);
-- return false;
-- }
- }
-
- return true;
-@@ -1217,6 +1221,11 @@ static ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft_impl(
- return NULL;
- }
-
-+ if (!alloc_ctx_tensors_finalize_views(ctx)) {
-+ free_buffers(&buffers, &n_buffers);
-+ return NULL;
-+ }
-+
- ggml_backend_buffer_t buffer;
- if (n_buffers == 1) {
- buffer = buffers[0];
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
index 138340bf..4210fa5a 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch
@@ -1,31 +1,37 @@
diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
--- a/llama.cpp/ggml/src/ggml-backend-impl.h
+++ b/llama.cpp/ggml/src/ggml-backend-impl.h
-@@ -15,17 +15,17 @@ extern "C" {
+@@ -15,21 +15,21 @@ extern "C" {
//
struct ggml_backend_buffer_type_i {
-- const char * (*get_name) (ggml_backend_buffer_type_t buft);
-+ const char * (GGML_CALL *get_name) (ggml_backend_buffer_type_t buft);
+- const char * (*get_name) (ggml_backend_buffer_type_t buft);
++ const char * (GGML_CALL *get_name) (ggml_backend_buffer_type_t buft);
// allocate a buffer of this type
-- ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
-+ ggml_backend_buffer_t (GGML_CALL *alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
+- ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
++ ggml_backend_buffer_t (GGML_CALL *alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
+ // (optional) allocate tensors from a list into a buffer of this type (defaults to alloc_buffer + linear allocator)
+- ggml_backend_buffer_t (*alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
++ ggml_backend_buffer_t (GGML_CALL *alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
// tensor alignment
-- size_t (*get_alignment) (ggml_backend_buffer_type_t buft);
-+ size_t (GGML_CALL *get_alignment) (ggml_backend_buffer_type_t buft);
+- size_t (*get_alignment) (ggml_backend_buffer_type_t buft);
++ size_t (GGML_CALL *get_alignment) (ggml_backend_buffer_type_t buft);
// (optional) max buffer size that can be allocated (defaults to SIZE_MAX)
-- size_t (*get_max_size) (ggml_backend_buffer_type_t buft);
-+ size_t (GGML_CALL *get_max_size) (ggml_backend_buffer_type_t buft);
+- size_t (*get_max_size) (ggml_backend_buffer_type_t buft);
++ size_t (GGML_CALL *get_max_size) (ggml_backend_buffer_type_t buft);
// (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes)
-- size_t (*get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
-+ size_t (GGML_CALL *get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
+- size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
++ size_t (GGML_CALL *get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
+ // (optional) total data size needed to allocate the given tensors, including padding and splitting (defaults to per-tensor get_alloc_size)
+- size_t (*get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
++ size_t (GGML_CALL *get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);
// (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)
-- bool (*is_host) (ggml_backend_buffer_type_t buft);
-+ bool (GGML_CALL *is_host) (ggml_backend_buffer_type_t buft);
+- bool (*is_host) (ggml_backend_buffer_type_t buft);
++ bool (GGML_CALL *is_host) (ggml_backend_buffer_type_t buft);
};
struct ggml_backend_buffer_type {
-@@ -44,26 +44,30 @@ extern "C" {
+@@ -48,26 +48,30 @@ extern "C" {
//
struct ggml_backend_buffer_i {
@@ -68,7 +74,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend_buffer {
-@@ -114,45 +118,45 @@ extern "C" {
+@@ -115,45 +119,45 @@ extern "C" {
struct ggml_backend_graph_optimize_params {
// keep `tensor` allocated at least until `until` (a node of the same graph) has been computed
// can be called multiple times for the same tensor: the longest lifetime applies
@@ -131,7 +137,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend {
-@@ -175,46 +179,46 @@ extern "C" {
+@@ -176,46 +180,46 @@ extern "C" {
// the current functions to obtain the properties can remain, since they are more convenient for often used properties
struct ggml_backend_device_i {
// device name: short identifier for this device, such as "CPU" or "CUDA0"
@@ -193,7 +199,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h
};
struct ggml_backend_device {
-@@ -228,15 +232,15 @@ extern "C" {
+@@ -229,15 +233,15 @@ extern "C" {
//
struct ggml_backend_reg_i {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
index df78814f..5e53e54e 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch
@@ -75,19 +75,22 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_buft_is_meta(buft));
const ggml_backend_meta_buffer_type_context * meta_buft_ctx = (const ggml_backend_meta_buffer_type_context *) buft->context;
return meta_buft_ctx->name.c_str();
-@@ -288,9 +288,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen
+@@ -288,11 +288,11 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen
return meta_buft_ctx->simple_bufts[index];
}
-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
+static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);
+-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors);
++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors);
+
-static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
+static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_alignment = 1;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -301,7 +301,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty
+@@ -303,7 +303,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty
return max_alignment;
}
@@ -96,7 +99,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_size = SIZE_MAX;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -310,7 +310,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ
+@@ -312,7 +312,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ
return max_size;
}
@@ -105,7 +108,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
size_t max_alloc_size = 0;
for (size_t i = 0; i < n_simple_bufts; i++) {
-@@ -320,7 +320,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t
+@@ -322,7 +322,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t
return max_alloc_size;
}
@@ -114,7 +117,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
for (size_t i = 0; i < n_simple_bufts; i++) {
if (!ggml_backend_buft_is_host(ggml_backend_meta_buft_simple_buft(buft, i))) {
-@@ -343,7 +343,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {
+@@ -347,7 +347,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {
return buft != nullptr && buft->iface.get_name == ggml_backend_meta_buffer_type_iface.get_name;
}
@@ -123,7 +126,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
static std::map<ggml_backend_dev_t, struct ggml_backend_buffer_type> meta_bufts;
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
{
-@@ -370,7 +370,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
+@@ -374,7 +374,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_
return &result.first->second;
}
@@ -132,7 +135,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_dev_is_meta(dev));
const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
-@@ -453,7 +453,7 @@ struct ggml_backend_meta_buffer_context {
+@@ -457,7 +457,7 @@ struct ggml_backend_meta_buffer_context {
}
};
@@ -141,7 +144,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
delete buf_ctx;
-@@ -1172,7 +1172,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
+@@ -1197,7 +1197,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co
return ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync);
}
@@ -150,7 +153,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_UNUSED(buffer);
return (void *) 0x1000000000000000; // FIXME
}
-@@ -1304,14 +1304,14 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m
+@@ -1335,14 +1335,14 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m
return GGML_STATUS_SUCCESS;
}
@@ -167,7 +170,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
const ggml_backend_meta_split_state split_state =
-@@ -1413,7 +1413,7 @@ static void ggml_backend_meta_buffer_memset_tensor(
+@@ -1444,7 +1444,7 @@ static void ggml_backend_meta_buffer_memset_tensor(
}
}
@@ -176,7 +179,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);
GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
-@@ -1541,7 +1541,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -1572,7 +1572,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
}
}
@@ -185,7 +188,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);
const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);
GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);
-@@ -1641,14 +1641,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -1672,14 +1672,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co
}
}
@@ -202,7 +205,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(ggml_backend_buffer_is_meta(buffer));
ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;
for (size_t i = 0; i < buf_ctx->bufs.size(); i++) {
-@@ -1684,7 +1684,7 @@ void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_
+@@ -1715,7 +1715,7 @@ void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_
}
}
@@ -211,7 +214,16 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
const ggml_init_params params = {
-@@ -1851,20 +1851,20 @@ struct ggml_backend_meta_context {
+@@ -1740,7 +1740,7 @@ static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_bac
+ return ggml_backend_buffer_init(buft, ggml_backend_meta_buffer_iface, buf_ctx, max_size);
+ }
+
+-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors) {
++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors) {
+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);
+
+ constexpr size_t compute_headroom = 16; // Maximum number of views per statically allocated tensor that can be created between evals.
+@@ -1883,20 +1883,20 @@ struct ggml_backend_meta_context {
}
};
@@ -235,7 +247,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_backends = ggml_backend_meta_n_backends(backend);
GGML_ASSERT(offset == 0);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1909,7 +1909,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens
+@@ -1941,7 +1941,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens
}
}
@@ -244,7 +256,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
const size_t n_backends = ggml_backend_meta_n_backends(backend);
GGML_ASSERT(offset == 0);
GGML_ASSERT(ggml_is_contiguous(tensor));
-@@ -1954,14 +1954,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm
+@@ -1986,14 +1986,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm
}
}
@@ -261,7 +273,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp
GGML_ASSERT(cgraph->grads == nullptr);
const size_t n_backends = ggml_backend_meta_n_backends(backend);
ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;
-@@ -2487,7 +2487,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {
+@@ -2523,7 +2523,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {
return backend != nullptr && backend->iface.get_name == ggml_backend_meta_i.get_name;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
index 63af2d06..468fbf80 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch
@@ -1,12 +1,13 @@
diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp
--- a/llama.cpp/ggml/src/ggml-backend-reg.cpp
+++ b/llama.cpp/ggml/src/ggml-backend-reg.cpp
-@@ -510,7 +510,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
- if (ec) {
- GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());
- } else {
-- GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
-+ // GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
+@@ -540,7 +540,8 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
}
- continue;
}
+
+- if (dir_ec) {
++ // llamafile: a search path that does not exist is expected (backends are loaded as DSOs elsewhere); don't log it
++ if (dir_ec && dir_ec != std::errc::no_such_file_or_directory) {
+ GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str());
+ }
+ }
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
index 4337f9fa..3f68a78c 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch
@@ -1,7 +1,71 @@
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
--- a/llama.cpp/ggml/src/ggml-backend.cpp
+++ b/llama.cpp/ggml/src/ggml-backend.cpp
-@@ -121,7 +121,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
+@@ -108,34 +108,22 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
+
+ struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
+
+- // allocate tensors in the current buffer
++ // allocate the parent tensors in the current buffer; views are initialized
++ // in a single pass below, once every buffer exists (llamafile: carries
++ // upstream PR #25584 — a range made only of views is never planned when it
++ // follows a tensor larger than max_size, so its views kept data == NULL)
+ struct ggml_tensor * t_failed = NULL;
+ for (int j = item.first; j < item.last; j++) {
+ struct ggml_tensor * t = tensors[j];
+- if (t->data == NULL) {
+- if (t->view_src == NULL) {
+- if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
+- t_failed = t;
+- break;
+- }
+- } else if (t->buffer == NULL) {
+- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
+- t_failed = t;
+- break;
+- }
+- }
+- } else {
+- if (t->view_src != NULL && t->buffer == NULL) {
+- // view of a pre-allocated tensor
+- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
+- t_failed = t;
+- break;
+- }
++ if (t->view_src == NULL && t->data == NULL) {
++ if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
++ t_failed = t;
++ break;
+ }
+ }
+ }
+ if (t_failed != NULL) {
+- GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
++ GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t_failed->name);
+ for (ggml_backend_buffer_t b : buffers) {
+ ggml_backend_buffer_free(b);
+ }
+@@ -150,6 +138,19 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe
+ return NULL;
+ }
+
++ for (int j = 0; j < n_tensors; j++) {
++ struct ggml_tensor * t = tensors[j];
++ if (t->view_src != NULL && t->buffer == NULL) {
++ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
++ GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);
++ for (ggml_backend_buffer_t b : buffers) {
++ ggml_backend_buffer_free(b);
++ }
++ return NULL;
++ }
++ }
++ }
++
+ if (buffers.size() == 1) {
+ return buffers[0];
+ }
+@@ -261,7 +262,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
if (buffer->iface.free_buffer != NULL) {
buffer->iface.free_buffer(buffer);
}
@@ -17,7 +81,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
}
size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {
-@@ -682,7 +689,7 @@ struct ggml_backend_multi_buffer_context {
+@@ -822,7 +830,7 @@ struct ggml_backend_multi_buffer_context {
size_t n_buffers;
};
@@ -26,7 +90,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
-@@ -693,7 +700,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
+@@ -833,7 +841,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)
free(ctx);
}
@@ -35,7 +99,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;
for (size_t i = 0; i < ctx->n_buffers; i++) {
-@@ -713,6 +720,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
+@@ -853,6 +861,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {
/* .cpy_tensor = */ NULL,
/* .clear = */ ggml_backend_multi_buffer_clear,
/* .reset = */ NULL,
@@ -43,7 +107,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
};
ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {
-@@ -1063,6 +1071,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
+@@ -1211,6 +1220,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru
}
// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend
@@ -62,7 +126,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {
// reset splits
sched->n_splits = 0;
-@@ -1445,13 +1465,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
+@@ -1609,13 +1630,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
std::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;
struct ggml_backend_graph_optimize_params opt_params = {
@@ -77,7 +141,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
/* .user_data = */ &alloc_deps,
};
-@@ -2352,7 +2366,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
+@@ -2466,7 +2481,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
// CPU backend - buffer
@@ -86,7 +150,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
uintptr_t data = (uintptr_t)buffer->context;
-@@ -2364,33 +2378,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
+@@ -2478,33 +2493,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {
return (void *)data;
}
@@ -125,7 +189,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(src);
if (ggml_backend_buffer_is_host(src->buffer)) {
memcpy(dst->data, src->data, ggml_nbytes(src));
-@@ -2401,7 +2415,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
+@@ -2515,7 +2530,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
GGML_UNUSED(buffer);
}
@@ -134,7 +198,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
GGML_ASSERT(buffer);
memset(buffer->context, value, buffer->size);
}
-@@ -2418,6 +2432,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
+@@ -2532,6 +2547,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
@@ -142,7 +206,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
};
static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
-@@ -2432,19 +2447,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
+@@ -2546,19 +2562,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {
/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cpu_buffer_clear,
/* .reset = */ NULL,
@@ -165,7 +229,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
void * data = ggml_aligned_malloc(size);
if (data == NULL) {
-@@ -2455,13 +2471,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
+@@ -2569,13 +2586,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back
return ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);
}
@@ -181,7 +245,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
return true;
GGML_UNUSED(buft);
-@@ -2484,7 +2500,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
+@@ -2600,7 +2617,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
return &ggml_backend_cpu_buffer_type;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
index d388f878..1789da20 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c
-@@ -1555,6 +1555,26 @@ static void ggml_compute_forward_mul_mat_id(
+@@ -1559,6 +1559,26 @@ static void ggml_compute_forward_mul_mat_id(
const struct ggml_tensor * src1 = dst->src[1];
const struct ggml_tensor * ids = dst->src[2];
@@ -28,7 +28,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
GGML_TENSOR_BINARY_OP_LOCALS
const int ith = params->ith;
-@@ -1691,6 +1711,24 @@ static void ggml_compute_forward_mul_mat_id(
+@@ -1692,6 +1712,24 @@ static void ggml_compute_forward_mul_mat_id(
const int64_t nr0 = ne01;
const int64_t nr1 = cne1;
@@ -53,10 +53,10 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
int chunk_size = 16;
if (nr0 == 1 || nr1 == 1) {
chunk_size = 64;
-@@ -2919,6 +2957,19 @@ struct ggml_cplan ggml_graph_plan(
- if (ggml_cpu_iqp_supports_mul_mat_id(node)) {
- cur += n_tasks * ggml_cpu_iqp_scratch_size(node) + 64;
- }
+@@ -2921,6 +2959,19 @@ struct ggml_cplan ggml_graph_plan(
+ // Workspace for tiled (see tiled.h)
+ cur = GGML_PAD(cur, 64);
+ cur += ggml_tiled_wdata_size(n_tasks, node);
+#if GGML_USE_LLAMAFILE
+ // [jart] reserve work-buffer space for llamafile_mixmul.
+ // Without this, MixMul::allocate_shared_memory() in
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
index fc7397ea..edc8d77a 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch
@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
-@@ -483,7 +483,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
+@@ -486,7 +486,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
}
}
@@ -160,7 +160,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
return ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);
GGML_UNUSED(dev);
}
-@@ -508,19 +508,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
+@@ -511,19 +511,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {
// CPU backend - backend (reg)
@@ -183,7 +183,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
GGML_ASSERT(index == 0);
static ggml_backend_cpu_device_context ctx;
-@@ -535,7 +535,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg
+@@ -538,7 +538,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg
// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,
// and additionally to allow other backends to expose their own list of features that applications can query using the same API
@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
static std::vector<ggml_backend_feature> features = []() {
ggml_cpu_init();
-@@ -652,7 +652,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
+@@ -655,7 +655,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
index b9208666..39584e02 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch
@@ -20,7 +20,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
// ggml_compute_forward_dup
static void ggml_compute_forward_dup_same_cont(
-@@ -8693,6 +8705,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8897,6 +8909,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type");
GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type");
@@ -36,7 +36,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
int ith = params->ith;
for (int ir = ir0; ir < ir1; ++ir) {
-@@ -8712,10 +8733,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8916,10 +8937,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulator
ggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16
@@ -62,7 +62,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
}
const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;
-@@ -8744,7 +8777,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8948,7 +8981,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
float s; // KQ value
const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);
@@ -76,7 +76,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
s = s*scale; // scale KQ value
-@@ -8761,21 +8800,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8965,21 +9004,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));
@@ -116,7 +116,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
} else {
if (s > M) {
// s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f
-@@ -8783,26 +8837,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
+@@ -8987,26 +9041,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
ms = expf(Mold - M);
// V = V*expf(Mold - M)
@@ -146,7 +146,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
for (int64_t d = 0; d < DV; ++d) {
VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);
}
-@@ -9023,8 +9071,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -9227,8 +9275,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;
if (kv_type == GGML_TYPE_F16) {
const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;
@@ -173,7 +173,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
}
} else {
const float * k_f32_src = (const float *)k_data;
-@@ -9034,7 +9098,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -9238,7 +9302,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
}
}
memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));
@@ -191,8 +191,8 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
+#endif
ggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);
- // Set padded KQ entries to -inf so softmax gives them zero weight
-@@ -9087,7 +9162,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+ if (logit_softcap != 0.0f) {
+@@ -9291,7 +9366,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
for (int tk = 0; tk < kv_tile; tk++) {
const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;
if (kv_type == GGML_TYPE_F16) {
@@ -208,7 +208,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
} else {
memcpy(V32 + tk * DV, v_data, DV * sizeof(float));
}
-@@ -9097,7 +9180,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
+@@ -9301,7 +9384,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(
memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));
}
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
index c067ae03..1df7c54e 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
--- a/llama.cpp/ggml/src/ggml-cpu/repack.cpp
+++ b/llama.cpp/ggml/src/ggml-cpu/repack.cpp
-@@ -5140,14 +5140,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
+@@ -5145,14 +5145,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
return nullptr;
}
@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
const void * data, size_t offset, size_t size) {
GGML_ASSERT(offset == 0);
GGML_ASSERT(size == ggml_nbytes(tensor));
-@@ -5159,13 +5159,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buff
+@@ -5164,13 +5164,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buff
GGML_UNUSED(buffer);
}
@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);
if (buffer == nullptr) {
-@@ -5180,7 +5180,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(gg
+@@ -5185,7 +5185,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(gg
return buffer;
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
index 8a2e8a33..b5168512 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh
-@@ -336,9 +336,15 @@ static bool fast_bf16_hardware_available(const int cc) {
+@@ -337,9 +337,15 @@ static bool fast_bf16_hardware_available(const int cc) {
}
static bool bf16_mma_hardware_available(const int cc) {
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
index 0a13aeef..ad35547e 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu
-@@ -513,6 +513,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
+@@ -519,6 +519,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
return dequantize_row_q5_K_cuda;
case GGML_TYPE_Q6_K:
return dequantize_row_q6_K_cuda;
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_IQ2_XXS:
return dequantize_row_iq2_xxs_cuda;
case GGML_TYPE_IQ2_XS:
-@@ -531,6 +532,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
+@@ -537,6 +538,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
return dequantize_row_iq4_xs_cuda;
case GGML_TYPE_IQ3_S:
return dequantize_row_iq3_s_cuda;
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_MXFP4:
return dequantize_row_mxfp4_cuda;
case GGML_TYPE_NVFP4:
-@@ -573,6 +575,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+@@ -579,6 +581,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
return dequantize_row_q5_K_cuda;
case GGML_TYPE_Q6_K:
return dequantize_row_q6_K_cuda;
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
case GGML_TYPE_IQ2_XXS:
return dequantize_row_iq2_xxs_cuda;
case GGML_TYPE_IQ2_XS:
-@@ -591,6 +594,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
+@@ -597,6 +600,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
return dequantize_row_iq4_xs_cuda;
case GGML_TYPE_IQ3_S:
return dequantize_row_iq3_s_cuda;
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
index 01dd1f39..47e4086a 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch
@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
// check if a same-type copy reduces to a 2D strided copy (height rows of width
// contiguous bytes), so it can use cudaMemcpy2DAsync instead of the scalar kernel
-@@ -524,9 +526,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
+@@ -525,9 +527,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
} else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {
ggml_cpy_q5_0_f32_cuda
(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
index 72ec44fb..6f5dfd83 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu
-@@ -738,7 +738,7 @@ struct ggml_backend_cuda_buffer_context {
+@@ -781,7 +781,7 @@ struct ggml_backend_cuda_buffer_context {
}
};
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
delete ctx;
}
-@@ -747,12 +747,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
+@@ -790,12 +790,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {
return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;
}
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
if (tensor->view_src != NULL) {
-@@ -773,7 +773,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
+@@ -817,7 +817,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer
return GGML_STATUS_SUCCESS;
}
@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -781,7 +781,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
+@@ -825,7 +825,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -789,7 +789,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
+@@ -833,7 +833,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -797,7 +797,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
+@@ -841,7 +841,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;
-@@ -807,7 +807,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
+@@ -851,7 +851,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
-@@ -817,7 +817,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
+@@ -861,7 +861,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -79,7 +79,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (ggml_backend_buffer_is_cuda(src->buffer)) {
ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;
ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;
-@@ -842,7 +842,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
+@@ -886,7 +886,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co
GGML_UNUSED(buffer);
}
@@ -88,7 +88,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;
ggml_cuda_set_device(ctx->device);
-@@ -850,6 +850,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
+@@ -894,6 +894,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
}
@@ -103,7 +103,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,
/* .get_base = */ ggml_backend_cuda_buffer_get_base,
-@@ -862,6 +870,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
+@@ -906,6 +914,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,
/* .clear = */ ggml_backend_cuda_buffer_clear,
/* .reset = */ NULL,
@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
};
// cuda buffer type
-@@ -870,7 +879,7 @@ struct ggml_backend_cuda_buffer_type_context {
+@@ -914,7 +923,7 @@ struct ggml_backend_cuda_buffer_type_context {
std::string name;
};
@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
return ctx->name.c_str();
-@@ -880,7 +889,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
+@@ -924,7 +933,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;
}
@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;
ggml_cuda_set_device(buft_ctx->device);
-@@ -899,13 +908,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
+@@ -943,13 +952,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac
return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);
}
@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *) buft->context;
size_t size = tensor->op == GGML_OP_FLASH_ATTN_EXT
-@@ -1259,7 +1268,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg
+@@ -1305,7 +1314,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg
// host buffer type
@@ -154,7 +154,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return GGML_CUDA_NAME "_Host";
GGML_UNUSED(buft);
-@@ -1269,7 +1278,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
+@@ -1315,7 +1324,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {
return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;
}
@@ -163,7 +163,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
CUDA_CHECK(cudaFreeHost(buffer->context));
}
-@@ -1291,17 +1300,25 @@ static void * ggml_cuda_host_malloc(size_t size) {
+@@ -1337,22 +1346,30 @@ static void * ggml_cuda_host_malloc(size_t size) {
return ptr;
}
@@ -191,7 +191,13 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
return buffer;
}
-@@ -1655,6 +1672,14 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
+
+-static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
++static size_t GGML_CALL ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
+ GGML_UNUSED(buft);
+ // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed
+ return 4ull*1024*1024*1024;
+@@ -1739,6 +1756,14 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
}
}
@@ -206,7 +212,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
switch (compute_type) {
case GGML_TYPE_F32:
ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F32>(ctx, src0, src1, dst);
-@@ -2429,20 +2454,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+@@ -2630,20 +2655,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
// backend
@@ -230,7 +236,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2451,7 +2476,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
+@@ -2652,7 +2677,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens
CUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
@@ -239,7 +245,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2460,7 +2485,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
+@@ -2661,7 +2686,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm
CUDA_CHECK(cudaMemcpyAsync(data, (const char *) tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
@@ -248,7 +254,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2471,7 +2496,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct
+@@ -2672,7 +2697,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct
(char *) tensor->data + offset, stride_tensor, data, stride_data, size, n_copies, cudaMemcpyHostToDevice, cuda_ctx->stream()));
}
@@ -257,7 +263,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;
-@@ -2482,7 +2507,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const
+@@ -2683,7 +2708,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const
data, stride_data, (const char *) tensor->data + offset, stride_tensor, size, n_copies, cudaMemcpyDeviceToHost, cuda_ctx->stream()));
}
@@ -266,7 +272,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;
ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;
-@@ -2541,7 +2566,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
+@@ -2742,7 +2767,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_
return true;
}
@@ -275,7 +281,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));
-@@ -4417,7 +4442,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
+@@ -4899,7 +4924,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co
}
#endif // USE_CUDA_GRAPH
@@ -284,7 +290,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
ggml_cuda_set_device(cuda_ctx->device);
-@@ -4476,13 +4501,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
+@@ -4962,13 +4987,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,
return GGML_STATUS_SUCCESS;
}
@@ -300,7 +306,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;
if (ggml_backend_is_cuda(backend)) {
-@@ -4501,7 +4526,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
+@@ -4987,7 +5012,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev
}
}
@@ -308,26 +314,26 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
+static void GGML_CALL ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {
ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;
- static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));
-@@ -4835,7 +4860,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
+ const bool disable_fusion = ggml_cuda_fusion_disabled();
+@@ -5422,7 +5447,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot
*total /= share_count;
}
-bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+bool GGML_CALL ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ if (!ggml_cuda_register_host_enabled()) {
return false;
}
-@@ -4858,7 +4883,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
+@@ -5445,7 +5470,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {
#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)
}
-void ggml_backend_cuda_unregister_host_buffer(void * buffer) {
+void GGML_CALL ggml_backend_cuda_unregister_host_buffer(void * buffer) {
- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {
+ if (!ggml_cuda_register_host_enabled()) {
return;
}
-@@ -4881,12 +4906,12 @@ struct ggml_backend_cuda_device_context {
+@@ -5468,12 +5493,12 @@ struct ggml_backend_cuda_device_context {
int op_offload_min_batch_size;
};
@@ -342,7 +348,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
return ctx->description.c_str();
}
-@@ -4967,7 +4992,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
+@@ -5554,7 +5579,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k
}
#endif // defined(__linux__)
@@ -351,7 +357,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
ggml_cuda_set_device(ctx->device);
cudaError_t err = cudaMemGetInfo(free, total);
-@@ -5008,7 +5033,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+@@ -5594,7 +5619,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
*total /= share_count;
}
@@ -360,7 +366,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context;
cudaDeviceProp prop;
-@@ -5019,7 +5044,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend
+@@ -5605,7 +5630,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend
: GGML_BACKEND_DEVICE_TYPE_GPU;
}
@@ -369,7 +375,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
props->name = ggml_backend_cuda_device_get_name(dev);
-@@ -5044,24 +5069,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
+@@ -5630,24 +5655,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
};
}
@@ -398,7 +404,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
// check if all the sources are allocated on this device
-@@ -5162,6 +5187,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5757,6 +5782,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
case GGML_TYPE_Q8_K:
@@ -412,7 +418,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
case GGML_TYPE_IQ1_M:
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ2_S:
-@@ -5171,6 +5203,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5766,6 +5798,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_TYPE_IQ3_XXS:
case GGML_TYPE_IQ4_NL:
case GGML_TYPE_IQ4_XS:
@@ -420,7 +426,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
case GGML_TYPE_BF16:
return true;
default:
-@@ -5280,9 +5313,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -5873,9 +5906,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
if (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) {
return true;
}
@@ -435,7 +441,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) {
return true;
}
-@@ -5519,7 +5557,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+@@ -6121,7 +6159,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
}
}
@@ -444,7 +450,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;
const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;
return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft));
-@@ -5540,13 +5578,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -6142,13 +6180,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -460,7 +466,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
#ifdef GGML_CUDA_NO_PEER_COPY
GGML_UNUSED(dev);
return nullptr;
-@@ -5565,14 +5603,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
+@@ -6167,14 +6205,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_
#endif
}
@@ -477,7 +483,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
GGML_UNUSED(dev);
CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));
}
-@@ -5601,23 +5639,23 @@ struct ggml_backend_cuda_reg_context {
+@@ -6203,23 +6241,23 @@ struct ggml_backend_cuda_reg_context {
std::vector<ggml_backend_dev_t> devices;
};
@@ -505,7 +511,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
static std::vector<ggml_backend_feature> features = []() {
std::vector<ggml_backend_feature> features;
#define _STRINGIFY(...) #__VA_ARGS__
-@@ -5674,7 +5712,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
+@@ -6276,7 +6314,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
GGML_UNUSED(reg);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
index 673e761f..67bedc19 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch
@@ -2,22 +2,22 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cu
+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cu
@@ -45,6 +45,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
- mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
+ mul_mat_q_case<GGML_TYPE_Q6_K, GGML_PREC_Q8>(ctx, args, stream);
break;
// -----------------------------------------------------------------------
+#ifndef GGML_CUDA_NO_IQ_QUANTS
case GGML_TYPE_IQ1_S:
- mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream);
+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);
break;
@@ -69,6 +70,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con
case GGML_TYPE_IQ4_NL:
- mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);
+ mul_mat_q_case<GGML_TYPE_IQ4_NL, GGML_PREC_Q8>(ctx, args, stream);
break;
+#endif // GGML_CUDA_NO_IQ_QUANTS
// -----------------------------------------------------------------------
case GGML_TYPE_MXFP4:
- mul_mat_q_case<GGML_TYPE_MXFP4>(ctx, args, stream);
-@@ -285,6 +287,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+ // src1 at Q4 uses the native FP4 instructions, which are Blackwell-only
+@@ -405,6 +407,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
// -------------------------------------------------
@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
case GGML_TYPE_IQ1_S:
case GGML_TYPE_IQ2_XXS:
case GGML_TYPE_IQ2_XS:
-@@ -293,6 +296,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -413,6 +416,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
case GGML_TYPE_IQ3_S:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ4_NL:
@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu
// -------------------------------------------------
case GGML_TYPE_MXFP4:
case GGML_TYPE_NVFP4:
-@@ -369,9 +373,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
+@@ -489,9 +493,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
return ne11 <= 128;
case GGML_TYPE_Q6_K:
return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
index e8b3f216..e75ec5e1 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh
-@@ -1585,6 +1585,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
+@@ -1634,6 +1634,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);
extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);
// -----------------------------------------
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh
extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);
-@@ -1593,6 +1594,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
+@@ -1642,6 +1643,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);
extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);
extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch
index 6f17a51e..20f3858b 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp
--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.cpp
+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.cpp
-@@ -717,7 +717,7 @@ static bool ggml_metal_fusion_match_raw_pattern(
+@@ -811,7 +811,7 @@ static bool ggml_metal_fusion_match_raw_pattern(
static void ggml_metal_fusion_add_pattern_alloc_deps(
void * user_data,
@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggm
ggml_cgraph * gf,
const ggml_metal_fusion * fusion,
int node_idx) {
-@@ -739,7 +739,7 @@ static void ggml_metal_fusion_add_pattern_alloc_deps(
+@@ -833,7 +833,7 @@ static void ggml_metal_fusion_add_pattern_alloc_deps(
void ggml_metal_fusion_add_alloc_deps(
void * user_data,
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch
index 5dbbf36a..6aa4979d 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h
--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.h
+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.h
-@@ -8,6 +8,7 @@
+@@ -9,6 +9,7 @@
#pragma once
#include "ggml-impl.h"
@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-
#include <stdbool.h>
-@@ -51,7 +52,7 @@ ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * f
+@@ -55,7 +56,7 @@ ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * f
// apply any alloc-dependencies required by the fused kernels during graph optimize
void ggml_metal_fusion_add_alloc_deps(
void * user_data,
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
index d5b1b90c..4e943d42 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch
@@ -155,7 +155,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
};
static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {
-@@ -256,33 +266,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
+@@ -259,33 +269,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
// default (shared) buffer type
@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -332,33 +342,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
+@@ -337,33 +347,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi
// default (private) buffer type
@@ -235,7 +235,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -407,34 +417,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
+@@ -414,34 +424,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev
// mapped buffer type
@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return false;
GGML_UNUSED(buft);
-@@ -485,13 +495,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
+@@ -494,13 +504,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi
// backend
@@ -292,7 +292,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_t ctx = (ggml_metal_t)backend->context;
// wait for any ongoing async operations to finish
-@@ -502,25 +512,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
+@@ -511,25 +521,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {
free(backend);
}
@@ -322,7 +322,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
if (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {
return false;
}
-@@ -541,27 +551,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
+@@ -550,27 +560,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml
return ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);
}
@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
GGML_ASSERT(params && params->add_alloc_dep);
// keep the MoE weighted-reduction inputs alive until the fused output so the
-@@ -659,7 +669,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
+@@ -668,7 +678,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {
// backend device
@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -667,25 +677,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
+@@ -676,25 +686,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {
return props_dev->name;
}
@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
props->name = ggml_backend_metal_device_get_name(dev);
props->description = ggml_backend_metal_device_get_description(dev);
props->type = ggml_backend_metal_device_get_type(dev);
-@@ -701,7 +711,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
+@@ -710,7 +720,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
};
}
@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_t ctx = ggml_metal_init(ctx_dev);
-@@ -726,7 +736,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
+@@ -735,7 +745,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t
GGML_UNUSED(params);
}
@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);
-@@ -734,7 +744,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
+@@ -743,7 +753,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml
return props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);
}
@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);
-@@ -744,13 +754,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
+@@ -753,13 +763,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen
return ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);
}
@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return
buft->device == dev && (
buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||
-@@ -771,7 +781,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
+@@ -780,7 +790,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {
}
}
@@ -445,7 +445,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
return (op->op == GGML_OP_MUL_MAT ||
-@@ -779,7 +789,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
+@@ -788,7 +798,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g
get_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;
}
@@ -454,7 +454,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);
-@@ -793,7 +803,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
+@@ -802,7 +812,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev
return ev;
}
@@ -463,7 +463,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t ev = (ggml_metal_event_t)event->context;
-@@ -803,7 +813,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
+@@ -812,7 +822,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba
delete event;
}
@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;
ggml_metal_event_t evt = (ggml_metal_event_t)event->context;
-@@ -855,18 +865,18 @@ struct ggml_backend_metal_reg_deleter {
+@@ -864,18 +874,18 @@ struct ggml_backend_metal_reg_deleter {
typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;
@@ -494,7 +494,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
ggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;
GGML_ASSERT(index < ctx->devices.size());
return ctx->devices[index];
-@@ -879,7 +889,7 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
+@@ -888,7 +898,7 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {
{ NULL, NULL },
};
@@ -503,7 +503,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal
return g_ggml_backend_metal_features;
GGML_UNUSED(reg);
-@@ -936,7 +946,7 @@ static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, b
+@@ -945,7 +955,7 @@ static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, b
ggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled);
}
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch
index 15445f90..b35ecf46 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp
-@@ -754,6 +754,7 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
+@@ -837,6 +837,7 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {
/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,
/* .clear = */ ggml_backend_vk_buffer_clear,
/* .reset = */ NULL,
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch
index 779fdcf5..cab214ba 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-common.h
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-common.h
-@@ -209,26 +209,27 @@ bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgr
+@@ -213,26 +213,27 @@ bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgr
bool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise);
bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);
diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
index 36b4784f..62b24289 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp
-@@ -12492,20 +12492,28 @@ bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
+@@ -13096,20 +13096,28 @@ bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {
return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;
}
@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");
if (tensor->view_src != nullptr) {
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
-@@ -12513,7 +12521,7 @@ enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer
+@@ -13117,7 +13125,7 @@ enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer
return GGML_STATUS_SUCCESS;
}
@@ -42,7 +42,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -12526,7 +12534,7 @@ void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_ten
+@@ -13130,7 +13138,7 @@ void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_ten
ggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);
}
@@ -51,7 +51,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
vk_buffer buf = buf_ctx->dev_buffer;
-@@ -12538,7 +12546,7 @@ void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor
+@@ -13142,7 +13150,7 @@ void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
@@ -60,7 +60,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
n_copies << ", " << stride_tensor << ", " << stride_data << ")");
-@@ -12552,7 +12560,7 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten
+@@ -13156,7 +13164,7 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten
ggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);
}
@@ -69,7 +69,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");
ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;
-@@ -12565,7 +12573,7 @@ void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_
+@@ -13169,7 +13177,7 @@ void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);
}
@@ -78,7 +78,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<
n_copies << ", " << stride_tensor << ", " << stride_data << ")");
-@@ -12580,7 +12588,7 @@ void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const gg
+@@ -13184,7 +13192,7 @@ void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const gg
ggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);
}
@@ -87,7 +87,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (ggml_nbytes(src) == 0) {
return true;
}
-@@ -12601,19 +12609,19 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_
+@@ -13205,19 +13213,19 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_
UNUSED(buffer);
}
@@ -110,7 +110,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;
-@@ -12629,17 +12637,17 @@ ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buff
+@@ -13233,17 +13241,17 @@ ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buff
return ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);
}
@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return ggml_nbytes(tensor);
UNUSED(buft);
-@@ -12655,18 +12663,18 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
+@@ -13259,18 +13267,18 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {
return &dev->buffer_type;
}
@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
-static void ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {
+static void GGML_CALL ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {
VK_LOG_MEMORY("ggml_backend_vk_host_buffer_free_buffer()");
- ggml_vk_host_free(vk_instance.devices[0], buffer->context);
+ ggml_vk_host_free(buffer->context);
}
-static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {
@@ -153,7 +153,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");
size += 32; // Behave like the CPU buffer type
-@@ -12682,19 +12690,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
+@@ -13284,19 +13292,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
buffer->buft = buft;
buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;
@@ -176,7 +176,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
return vk_instance.devices[0]->suballocation_block_size;
UNUSED(buft);
-@@ -12721,13 +12730,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
+@@ -13325,13 +13334,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {
return &ggml_backend_vk_buffer_type_host;
}
@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");
-@@ -12743,7 +12752,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
+@@ -13347,7 +13356,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b
return &ctx->device->buffer_type;
}
@@ -201,7 +201,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_set_tensor_2d_async(" << size << ", " << n_copies << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -12801,12 +12810,12 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten
+@@ -13405,12 +13414,12 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten
}
}
@@ -216,7 +216,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {
VK_LOG_DEBUG("ggml_backend_vk_get_tensor_2d_async(" << size << ", " << n_copies << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -12857,12 +12866,12 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg
+@@ -13461,12 +13470,12 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg
}
}
@@ -231,7 +231,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;
-@@ -13004,7 +13013,7 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
+@@ -13608,7 +13617,7 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {
}
}
@@ -240,7 +240,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_synchronize()");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -13741,7 +13750,7 @@ static int32_t find_first_set(uint32_t x) {
+@@ -14358,7 +14367,7 @@ static int32_t find_first_set(uint32_t x) {
return ret;
}
@@ -249,7 +249,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14236,7 +14245,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+@@ -14862,7 +14871,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
UNUSED(backend);
}
@@ -258,7 +258,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
{
VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
-@@ -14617,7 +14626,7 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph,
+@@ -15249,7 +15258,7 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph,
}
}
@@ -267,7 +267,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -14655,7 +14664,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
+@@ -15287,7 +15296,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev
ctx->compute_ctx.reset();
}
@@ -276,19 +276,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");
ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;
vk_event *vkev = (vk_event *)event->context;
-@@ -14756,7 +14765,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total
- *total += heap.size;
-
- if (membudget_supported && i < budgetprops.heapUsage.size()) {
-- *free += budgetprops.heapBudget[i] - budgetprops.heapUsage[i];
-+ // heapUsage can exceed heapBudget (permitted by spec); clamp to 0 to avoid underflow
-+ if (budgetprops.heapBudget[i] > budgetprops.heapUsage[i]) {
-+ *free += budgetprops.heapBudget[i] - budgetprops.heapUsage[i];
-+ }
- } else {
- *free += heap.size;
- }
-@@ -14813,38 +14825,38 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
+@@ -15448,38 +15457,38 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
return std::string(pci_bus_id);
}
@@ -334,7 +322,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
props->name = ggml_backend_vk_device_get_name(dev);
-@@ -14861,13 +14873,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
+@@ -15496,13 +15505,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
};
}
@@ -350,7 +338,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
const vk_device& device = ggml_vk_get_device(ctx->device);
-@@ -15522,7 +15534,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
+@@ -16164,7 +16173,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
UNUSED(dev);
}
@@ -359,7 +347,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
if (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {
return false;
}
-@@ -15533,13 +15545,13 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_ba
+@@ -16175,13 +16184,13 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_ba
return buft_ctx->device->idx == ctx->device;
}
@@ -375,7 +363,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15562,7 +15574,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
+@@ -16204,7 +16213,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t
};
}
@@ -384,7 +372,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15582,7 +15594,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
+@@ -16224,7 +16233,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe
delete event;
}
@@ -393,7 +381,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
auto device = ggml_vk_get_device(ctx->device);
-@@ -15614,7 +15626,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
+@@ -16256,7 +16265,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm
}
}
@@ -402,7 +390,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu
VK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");
GGML_UNUSED(max_tensor_size);
-@@ -15652,17 +15664,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
+@@ -16294,17 +16303,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {
/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,
};
diff --git a/llama.cpp.patches/patches/ggml_src_ggml.c.patch b/llama.cpp.patches/patches/ggml_src_ggml.c.patch
index d74076b8..43dff796 100644
--- a/llama.cpp.patches/patches/ggml_src_ggml.c.patch
+++ b/llama.cpp.patches/patches/ggml_src_ggml.c.patch
@@ -1,17 +1,17 @@
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
--- a/llama.cpp/ggml/src/ggml.c
+++ b/llama.cpp/ggml/src/ggml.c
-@@ -548,11 +548,13 @@ void ggml_time_init(void) {
+@@ -556,11 +556,13 @@ void ggml_time_init(void) {
InitOnceExecuteOnce(&once, ggml_time_init_once, NULL, NULL);
}
int64_t ggml_time_ms(void) {
+ ggml_time_init(); // llamafile: GPU DLLs carry their own ggml.c; init lazily so timer_freq is never 0
LARGE_INTEGER t;
QueryPerformanceCounter(&t);
- return ((t.QuadPart-timer_start) * 1000) / timer_freq;
+ return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000);
}
int64_t ggml_time_us(void) {
+ ggml_time_init(); // llamafile: see ggml_time_ms
LARGE_INTEGER t;
QueryPerformanceCounter(&t);
- return ((t.QuadPart-timer_start) * 1000000) / timer_freq;
+ return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000000);
diff --git a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
index 0036961e..c7e66409 100644
--- a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
+++ b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch
@@ -12,7 +12,7 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
#include <cinttypes>
#include <cstddef>
#include <cstdint>
-@@ -997,7 +1001,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru
+@@ -1014,7 +1018,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru
return gguf_init_from_reader(gr, params);
}
@@ -67,7 +67,7 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp
FILE * file = ggml_fopen(fname, "rb");
if (!file) {
-@@ -1008,6 +1059,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
+@@ -1025,6 +1076,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p
struct gguf_context * result = gguf_init_from_file_ptr(file, params);
fclose(file);
return result;
diff --git a/llama.cpp.patches/patches/src_llama-context.cpp.patch b/llama.cpp.patches/patches/src_llama-context.cpp.patch
index f99ba6a1..b9e48a5c 100644
--- a/llama.cpp.patches/patches/src_llama-context.cpp.patch
+++ b/llama.cpp.patches/patches/src_llama-context.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
--- a/llama.cpp/src/llama-context.cpp
+++ b/llama.cpp/src/llama-context.cpp
-@@ -553,6 +553,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
+@@ -657,6 +657,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3
}
};
diff --git a/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patch b/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patch
new file mode 100644
index 00000000..109b0c6d
--- /dev/null
+++ b/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patch
@@ -0,0 +1,53 @@
+diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp
+--- a/llama.cpp/src/llama-memory-hybrid-idx.cpp
++++ b/llama.cpp/src/llama-memory-hybrid-idx.cpp
+@@ -16,6 +16,25 @@
+ // llama_memory_hybrid_idx
+ //
+
++// llamafile: defined before the constructor, which instantiates ~unique_ptr<kpool_layout>;
++// with -std=gnu++23 that destructor is constexpr and clang instantiates it on the spot.
++// Which cells of a sequence make up which pool, for the whole cache.
++struct llama_memory_hybrid_idx::kpool_layout {
++ struct seq {
++ llama_pos pos_min = 0;
++ uint32_t strm = 0; // Stream holding this sequence's cells
++ std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position.
++ std::vector<uint32_t> pools;
++
++ // Where the pool scan stopped, so an append resumes instead of starting over.
++ size_t j_next = 0;
++ };
++
++ std::array<seq, LLAMA_MAX_SEQ> seqs;
++
++ uint32_t n_pool_real = 0;
++};
++
+ llama_memory_hybrid_idx::llama_memory_hybrid_idx(
+ const llama_model & model,
+ bool no_alloc,
+@@ -329,23 +348,6 @@ static std::vector<uint32_t> llama_memory_hybrid_idx_ns(const llama_kv_cache::sl
+ return res;
+ }
+
+-// Which cells of a sequence make up which pool, for the whole cache.
+-struct llama_memory_hybrid_idx::kpool_layout {
+- struct seq {
+- llama_pos pos_min = 0;
+- uint32_t strm = 0; // Stream holding this sequence's cells
+- std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position.
+- std::vector<uint32_t> pools;
+-
+- // Where the pool scan stopped, so an append resumes instead of starting over.
+- size_t j_next = 0;
+- };
+-
+- std::array<seq, LLAMA_MAX_SEQ> seqs;
+-
+- uint32_t n_pool_real = 0;
+-};
+-
+ // Which pools of the layout the current ubatch must re-pool, in the layout's pool order.
+ struct llama_memory_hybrid_idx_context::kpool_state {
+ std::vector<uint32_t> is_new;
diff --git a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
index f0208df5..4e7080dc 100644
--- a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
+++ b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch
@@ -1,18 +1,19 @@
diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
--- a/llama.cpp/src/llama-mmap.cpp
+++ b/llama.cpp/src/llama-mmap.cpp
-@@ -4,6 +4,10 @@
+@@ -4,6 +4,11 @@
#include "ggml.h"
+#ifdef COSMOCC
+#include "llamafile/llamafile.h"
++#include <cosmo.h>
+#endif
+
#include <cstring>
#include <climits>
#include <cstdlib>
-@@ -182,6 +186,15 @@ struct llama_file::impl {
+@@ -182,6 +187,15 @@ struct llama_file::impl {
}
#else
impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {
@@ -28,7 +29,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#ifdef __linux__
// Try unbuffered I/O for read only
if (use_direct_io && std::strcmp(mode, "rb") == 0) {
-@@ -234,6 +247,9 @@ struct llama_file::impl {
+@@ -234,6 +248,9 @@ struct llama_file::impl {
}
size_t tell() const {
@@ -38,7 +39,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
if (fd == -1) {
off_t ret = llama_mmap_ftell(fp);
if (ret == -1) {
-@@ -251,6 +267,11 @@ struct llama_file::impl {
+@@ -251,6 +268,11 @@ struct llama_file::impl {
}
void seek(size_t offset, int whence) const {
@@ -50,7 +51,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
off_t ret = 0;
if (fd == -1) {
ret = llama_mmap_fseek(fp, offset, whence);
-@@ -260,12 +281,22 @@ struct llama_file::impl {
+@@ -260,12 +282,22 @@ struct llama_file::impl {
if (ret == -1) {
throw std::runtime_error(format("seek error: %s", strerror(errno)));
}
@@ -73,7 +74,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
errno = 0;
if (fd == -1) {
const size_t curr_off = tell();
-@@ -315,6 +346,7 @@ struct llama_file::impl {
+@@ -315,6 +347,7 @@ struct llama_file::impl {
bytes_read += (size_t) ret;
}
}
@@ -81,7 +82,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
}
void read_aligned_chunk(void * dest, size_t size) {
-@@ -375,13 +407,20 @@ struct llama_file::impl {
+@@ -388,13 +421,20 @@ struct llama_file::impl {
}
~impl() {
@@ -102,7 +103,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
std::string fname;
#endif
-@@ -410,6 +449,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
+@@ -423,6 +463,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }
bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }
int llama_file::file_id() const {
@@ -113,7 +114,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#ifdef _WIN32
return _fileno(pimpl->fp);
#else
-@@ -417,9 +460,10 @@ int llama_file::file_id() const {
+@@ -430,9 +474,10 @@ int llama_file::file_id() const {
return pimpl->fd;
}
#if defined(fileno)
@@ -126,7 +127,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
#endif
#endif
}
-@@ -437,6 +481,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
+@@ -450,6 +495,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }
void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }
void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }
@@ -158,7 +159,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
// llama_mmap
#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)
-@@ -465,9 +534,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t li
+@@ -478,9 +548,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t li
struct llama_mmap::impl {
#ifdef _POSIX_MAPPED_FILES
std::vector<std::pair<size_t, size_t>> mapped_fragments;
@@ -185,18 +186,26 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp
int fd = file->file_id();
int flags = MAP_SHARED;
if (numa) { prefetch = 0; }
-@@ -528,6 +614,10 @@ struct llama_mmap::impl {
+@@ -541,6 +628,18 @@ struct llama_mmap::impl {
}
void unmap_fragment(size_t first, size_t last) {
+ if (!is_owned) {
+ return;
+ }
++#ifdef COSMOCC
++ // Windows can only unmap a whole view (cosmo's munmap fails with ENOTSUP or
++ // EINVAL on a sub-range), so do what the _WIN32 branch below does: keep the
++ // fragments, and let the destructor unmap the whole file in one call.
++ if (IsWindows()) {
++ return;
++ }
++#endif
+
int page_size = sysconf(_SC_PAGESIZE);
align_range(&first, &last, page_size);
size_t len = last - first;
-@@ -564,6 +654,14 @@ struct llama_mmap::impl {
+@@ -577,6 +676,14 @@ struct llama_mmap::impl {
}
~impl() {
diff --git a/llama.cpp.patches/patches/src_llama-mmap.h.patch b/llama.cpp.patches/patches/src_llama-mmap.h.patch
index 6b6ba5eb..ef31f21e 100644
--- a/llama.cpp.patches/patches/src_llama-mmap.h.patch
+++ b/llama.cpp.patches/patches/src_llama-mmap.h.patch
@@ -1,7 +1,7 @@
diff --git a/src/llama-mmap.h b/src/llama-mmap.h
--- a/llama.cpp/src/llama-mmap.h
+++ b/llama.cpp/src/llama-mmap.h
-@@ -36,6 +36,11 @@ struct llama_file {
+@@ -39,6 +39,11 @@ struct llama_file {
size_t read_alignment() const;
bool has_direct_io() const;
diff --git a/llama.cpp.patches/patches/src_models_t5.cpp.patch b/llama.cpp.patches/patches/src_models_t5.cpp.patch
index 6bb5ae7d..37389c22 100644
--- a/llama.cpp.patches/patches/src_models_t5.cpp.patch
+++ b/llama.cpp.patches/patches/src_models_t5.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/src/models/t5.cpp b/src/models/t5.cpp
--- a/llama.cpp/src/models/t5.cpp
+++ b/llama.cpp/src/models/t5.cpp
-@@ -357,6 +357,12 @@ llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_pa
+@@ -355,6 +355,12 @@ llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_pa
ggml_build_forward_expand(gf, cur);
}
diff --git a/llama.cpp.patches/patches/tools_server_server-http.cpp.patch b/llama.cpp.patches/patches/tools_server_server-http.cpp.patch
index c3319387..21ea880a 100644
--- a/llama.cpp.patches/patches/tools_server_server-http.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server-http.cpp.patch
@@ -1,16 +1,16 @@
diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp
--- a/llama.cpp/tools/server/server-http.cpp
+++ b/llama.cpp/tools/server/server-http.cpp
-@@ -104,7 +104,7 @@ bool server_http_context::init(const common_params & params) {
-
- auto & srv = pimpl->srv;
+@@ -159,7 +159,7 @@ bool server_http_context::init(const common_params & params) {
+ bool server_http_context::init_listener(const common_params & params) {
+ auto & srv = pimpl->servers.back();
-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT
+#ifdef CPPHTTPLIB_SSL_ENABLED
if (!params.ssl_file_key.empty() && !params.ssl_file_cert.empty()) {
SRV_TRC("running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());
srv = std::make_unique<httplib::SSLServer>(
-@@ -429,12 +429,15 @@ bool server_http_context::init(const common_params & params) {
+@@ -491,7 +491,11 @@ self.addEventListener('activate', (e) => e.waitUntil((async () => {
return true;
}
@@ -22,14 +22,9 @@ diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp
+
// Bind and listen
- const auto & srv = pimpl->srv;
- auto was_bound = false;
-- auto is_sock = false;
- if (string_ends_with(std::string(hostname), ".sock")) {
- is_sock = true;
- SRV_TRC("%s", "setting address family to AF_UNIX\n");
-@@ -461,6 +464,17 @@ bool server_http_context::start() {
- return false;
+ listening_addresses.clear();
+@@ -524,6 +528,15 @@ bool server_http_context::start() {
+ : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(host).c_str(), port));
}
+ is_bound = true;
@@ -41,8 +36,6 @@ diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp
+ return false;
+ }
+
-+ const auto & srv = pimpl->srv;
-+
- // run the HTTP server in a thread
- thread = std::thread([this] { pimpl->srv->listen_after_bind(); });
- srv->wait_until_ready();
+ // n_threads_http fixed threads (always alive), plus up to 1024 dynamic threads destroyed after each request
+ // ref: https://github.com/yhirose/cpp-httplib/pull/2368
+ pimpl->pool = std::make_unique<httplib::ThreadPool>(pimpl->n_threads_http, pimpl->n_threads_http + 1024);
diff --git a/llama.cpp.patches/patches/tools_server_server-http.h.patch b/llama.cpp.patches/patches/tools_server_server-http.h.patch
index 39c7a976..07e3dce5 100644
--- a/llama.cpp.patches/patches/tools_server_server-http.h.patch
+++ b/llama.cpp.patches/patches/tools_server_server-http.h.patch
@@ -1,21 +1,20 @@
diff --git a/tools/server/server-http.h b/tools/server/server-http.h
--- a/llama.cpp/tools/server/server-http.h
+++ b/llama.cpp/tools/server/server-http.h
-@@ -79,11 +79,17 @@ struct server_http_context {
- std::string hostname;
- int port = 8080;
+@@ -91,11 +91,16 @@ struct server_http_context {
+ std::string path_prefix;
+ int port = 9931;
bool is_ssl = false;
-+ bool is_sock = false; // listening on a Unix domain socket
+ bool is_bound = false;
server_http_context();
~server_http_context();
bool init(const common_params & params);
-+ // creates and binds the listening socket; start() calls it if needed.
-+ // llamafile binds early so the socket exists before the sandbox is
++ // creates and binds the listening sockets; start() calls it if needed.
++ // llamafile binds early so the sockets exist before the sandbox is
+ // installed (a Unix domain socket can't be created under pledge "anet")
+ bool bind_socket();
bool start();
void stop() const;
-
+ void join();
diff --git a/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patch
new file mode 100644
index 00000000..6ad203e9
--- /dev/null
+++ b/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patch
@@ -0,0 +1,33 @@
+diff --git a/tools/server/server-models-sched.cpp b/tools/server/server-models-sched.cpp
+--- a/llama.cpp/tools/server/server-models-sched.cpp
++++ b/llama.cpp/tools/server/server-models-sched.cpp
+@@ -277,7 +277,12 @@ void server_models_sched::make_room(const std::string & name) {
+ };
+ if (!measured()) {
+ SRV_INF("waiting for the memory measurement of model name=%s\n", name.c_str());
+- models.cv.wait(lk, measured);
++ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
++ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
++ // causing condition_variable::wait() to throw ETIMEDOUT.
++ while (!measured()) {
++ models.cv.wait_for(lk, std::chrono::seconds(30));
++ }
+ }
+ }
+
+@@ -298,9 +303,12 @@ void server_models_sched::make_room(const std::string & name) {
+ // wait for unload to complete
+ {
+ std::unique_lock<std::mutex> lk(models.mutex);
+- models.cv.wait(lk, [this, &lru_model_name]() {
+- return models.mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED;
+- });
++ // Use wait_for() instead of wait() to work around a Cosmopolitan libc
++ // bug where untimed futex waits on XNU (macOS) expire after ~72 minutes,
++ // causing condition_variable::wait() to throw ETIMEDOUT.
++ while (models.mapping[lru_model_name].meta.status != SERVER_MODEL_STATUS_UNLOADED) {
++ models.cv.wait_for(lk, std::chrono::seconds(30));
++ }
+ }
+ }
+ }
diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
index ab633263..3ee67458 100644
--- a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
--- a/llama.cpp/tools/server/server-models.cpp
+++ b/llama.cpp/tools/server/server-models.cpp
-@@ -31,6 +31,7 @@
+@@ -34,6 +34,7 @@
#include <cstring>
#ifndef _WIN32
@@ -9,7 +9,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
extern char **environ;
#endif
-@@ -61,7 +62,25 @@ static void request_child_exit(server_subproc & proc) {
+@@ -51,7 +52,25 @@ static void request_child_exit(server_subproc & proc) {
// single-threaded, watching all child processes at once
struct server_monitor {
server_monitor(server_models & models) : models(models) {
@@ -36,7 +36,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
~server_monitor() {
-@@ -893,13 +912,19 @@ void server_models::load_models() {
+@@ -947,13 +966,19 @@ void server_models::load_models() {
}
// wait for all targeted models to reach UNLOADED; cv.wait handles unlock/relock
@@ -58,23 +58,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
// erase models no longer in any source
for (auto it = mapping.begin(); it != mapping.end(); ) {
-@@ -1096,9 +1121,12 @@ void server_models::unload_lru() {
- // wait for unload to complete
- {
- std::unique_lock<std::mutex> lk(mutex);
-- cv.wait(lk, [this, &lru_model_name]() {
-- return mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED;
-- });
-+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc
-+ // bug where untimed futex waits on XNU (macOS) expire after ~72 minutes,
-+ // causing condition_variable::wait() to throw ETIMEDOUT.
-+ while (mapping[lru_model_name].meta.status != SERVER_MODEL_STATUS_UNLOADED) {
-+ cv.wait_for(lk, std::chrono::seconds(30));
-+ }
- }
- }
- }
-@@ -1123,7 +1151,12 @@ void server_models::load(const std::string & name, const load_options & opts) {
+@@ -1158,7 +1183,12 @@ void server_models::load(const std::string & name, const load_options & opts) {
std::unique_lock<std::mutex> lk(mutex);
// edge case: block until any in-progress reload has finished so we always load
// against the freshest preset and a consistent mapping state
@@ -86,20 +70,21 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
+ cv.wait_for(lk, std::chrono::seconds(30));
+ }
- auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;
- if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {
-@@ -1283,14 +1316,20 @@ void server_models::unload_all() {
+ // a download over an existing entry is decided under the lock, so a load of the
+ // same model cannot start in between: a paused download or an idle cached model
+@@ -1381,7 +1411,10 @@ void server_models::unload_all() {
}
}
// wait for every child to exit, the monitor force-kills the ones that ignore the exit command
-- cv.wait(lk, [this]() {
+- cv.wait(lk, [this, &lk]() {
+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug
+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,
+ // causing condition_variable::wait() to throw ETIMEDOUT.
-+ auto all_stopped = [this]() {
- for (const auto & [name, inst] : mapping) {
- if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {
- return false;
++ auto all_stopped = [this, &lk]() {
+ if (sched->is_measuring(lk)) {
+ return false;
+ }
+@@ -1391,7 +1424,10 @@ void server_models::unload_all() {
}
}
return true;
@@ -111,7 +96,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
void server_models::update_status(const std::string & name, const update_status_args & args) {
-@@ -1422,16 +1461,22 @@ void server_models::wait(const std::string & name, std::function<bool(const serv
+@@ -1552,16 +1588,22 @@ void server_models::wait(const std::string & name, std::function<bool(const serv
}
void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate) {
@@ -141,7 +126,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
}
bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {
-@@ -2448,10 +2493,10 @@ server_http_proxy::server_http_proxy(
+@@ -2631,10 +2673,10 @@ server_http_proxy::server_http_proxy(
auto pipe = std::make_shared<server_pipe<msg_t>>();
if (scheme == "https") {
diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patch
index 21e6da5d..c7fe27e9 100644
--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch
+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/tools/server/server.cpp b/tools/server/server.cpp
--- a/llama.cpp/tools/server/server.cpp
+++ b/llama.cpp/tools/server/server.cpp
-@@ -15,6 +15,7 @@
+@@ -16,6 +16,7 @@
#include <atomic>
#include <clocale>
#include <exception>
@@ -9,7 +9,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
#include <signal.h>
#include <thread> // for std::thread::hardware_concurrency
-@@ -22,6 +23,13 @@
+@@ -23,6 +24,13 @@
#include <windows.h>
#endif
@@ -23,15 +23,23 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
static std::function<void(int)> shutdown_handler;
static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;
-@@ -40,7 +48,19 @@ static inline void signal_handler(int signal) {
- int llama_server(int argc, char ** argv);
+@@ -41,6 +49,12 @@ static inline void signal_handler(int signal) {
+ struct server_run_opts {
+ bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself
+ bool resolve_model = true; // download / resolve -hf and docker models; off when the caller did it already
++
++ // llamafile combined mode (TUI + server in one process), see server_main()
++ std::function<void(const std::string &)> on_ready; // the server is loaded and accepting requests
++ std::function<void(std::function<void()>)> on_shutdown_available; // receives a function that stops the server
++ char ** argv = nullptr; // the sandbox detects --rpc on argv, it has no params field
++ bool combined_mode() const { return on_ready || on_shutdown_available; }
+ };
+
+ // satisfies -Wmissing-declarations (used by llama command)
+@@ -48,6 +62,31 @@ int llama_server(int argc, char ** argv);
// to be used via CLI (argc / argv are used by router mode only)
--int llama_server(common_params & params, int argc, char ** argv);
-+// on_ready / on_shutdown_available are llamafile combined-mode (TUI) callbacks.
-+int llama_server(common_params & params, int argc, char ** argv,
-+ std::function<void(const std::string &)> on_ready = nullptr,
-+ std::function<void(std::function<void()>)> on_shutdown_available = nullptr);
+ int llama_server(common_params & params, int argc, char ** argv);
+
+// Core server logic - can be called from llamafile main.cpp or standalone main()
+// Optional callbacks for combined mode:
@@ -40,11 +48,181 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
+int server_main(int argc, char ** argv,
+ std::function<void(const std::string &)> on_ready,
+ std::function<void(std::function<void()>)> on_shutdown_available);
++
++#ifdef LLAMAFILE_TUI
++// By now the program can safely exit. In combined mode return normally instead,
++// so the caller can handle cleanup and join the TUI thread.
++static void server_tui_exit(const server_run_opts & opts) {
++ if (opts.combined_mode()) {
++ return;
++ }
++ common_log_flush(common_log_main());
++ // Metal backend has async logging that llama_synchronize() doesn't wait for.
++ // Without this delay, _exit() might truncate llama_memory_breakdown_print's output
++ sleep(1);
++ // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)
++ _exit(0);
++}
++#endif
+
void llama_server_terminate();
void llama_server_terminate() {
if (shutdown_handler) {
-@@ -86,6 +106,14 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
+@@ -602,6 +641,9 @@ static int server_run_router(common_params & params, int argc, char ** argv, con
+ // when the HTTP server stops, clean up and exit
+ clean_up();
+
++#ifdef LLAMAFILE_TUI
++ server_tui_exit(opts);
++#endif
+ return 0;
+ }
+
+@@ -655,6 +697,107 @@ static int server_run_instance(common_params & params, server_role & role, commo
+ llama_backend_free();
+ };
+
++#ifdef COSMOCC
++ // Sandbox (pledge()/unveil(), llamafile issue #930). The whole
++ // derivation lives in llamafile/sandbox.c so it stays unit-testable
++ // and off the vendored diff. Placement is constrained on every side:
++ // - after the -hf / docker model resolve above: downloads need connect()
++ // - after ctx_http.init(): the TLS certificate/key are read there
++ // - before ctx_http.start(): filters only cover threads created
++ // after installation, so the HTTP listener/pool must not exist
++ // - before load_model(): untrusted GGUF data is parsed sandboxed
++ //
++ // Combined mode hosts an in-process HTTP client (the TUI) that needs
++ // connect(), which the accept-only sandbox forbids, so it's skipped.
++ //
++ // --connect spawns llama-connect once the server listens, i.e. after the
++ // lock; the sandbox forbids exec, so it is skipped there too.
++ if (opts.combined_mode()) {
++ SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");
++ } else if (params.server_connect) {
++ SRV_WRN("%s", "sandbox: disabled with --connect (it spawns llama-connect, which the sandbox forbids)\n");
++ } else {
++ // pledge() (no outbound network / writes / exec) is applied by
++ // default. unveil() path confinement is opt-in (--confine-reads),
++ // because a server opens files by path at request time (multimodal
++ // media, etc.) that can't be known when the ruleset locks.
++ //
++ // cosmo installs the SECCOMP/Landlock filters on the calling thread
++ // only (no TSYNC), so quiesce every pre-existing background thread
++ // across the call: the llama.cpp log worker (pause() joins it,
++ // resume() respawns it under the filter) and the stream-session GC
++ // started at the top of server_main() (stop joins it, start
++ // respawns it; no sessions exist yet, so stopping is a no-op
++ // beyond the join).
++ //
++ // When --confine-reads is on, unveil() confines reads to these
++ // directories, so collect every on-disk path the loader opens
++ // after the lock. Add to this list when llama.cpp gains a new file
++ // parameter (else it'd be denied under confinement). Paths read
++ // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf
++ // downloads in the model resolve above) need no entry.
++ std::vector<const char *> read_paths, rw_paths;
++ read_paths.push_back(params.model.path.c_str());
++ read_paths.push_back(params.mmproj.path.c_str());
++ read_paths.push_back(params.media_path.c_str()); // request-time media
++ read_paths.push_back(params.speculative.draft.mparams.path.c_str());
++ read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());
++ read_paths.push_back(params.public_path.c_str());
++ for (const auto & la : params.lora_adapters)
++ read_paths.push_back(la.path.c_str());
++ for (const auto & cv : params.control_vectors)
++ read_paths.push_back(cv.fname.c_str());
++ rw_paths.push_back(params.slot_save_path.c_str());
++ rw_paths.push_back(params.path_prompt_cache.c_str());
++ rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());
++
++ // Features that dial out at runtime need connect(), which the
++ // accept-only default forbids; relax to full sockets for them
++ // (still no writes/exec). --rpc has no params field -- its handler
++ // registers backends directly -- so detect it on argv/env.
++ const bool needs_outbound =
++ params.ui_mcp_proxy ||
++ !params.server_tools.empty() ||
++ (opts.argv && llamafile_has(opts.argv, "--rpc")) ||
++ (std::getenv("LLAMA_ARG_RPC") != nullptr);
++
++ llamafile_sandbox_spec spec = {
++ read_paths.data(), (int) read_paths.size(),
++ rw_paths.data(), (int) rw_paths.size(),
++ FLAG_confine_reads, needs_outbound,
++ };
++ // Bind the listening sockets now: pledge "anet" only covers
++ // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be
++ // created once the sandbox is on, while accept() on one that is
++ // already bound stays allowed (llamafile issue #971). Binding
++ // spawns no threads; ctx_http.start() reuses the sockets.
++ if (!ctx_http.bind_socket()) {
++ clean_up();
++ SRV_ERR("%s", "exiting due to HTTP server error\n");
++ return 1;
++ }
++
++ char promises[64];
++ common_log_pause(common_log_main());
++ server_stream_session_manager_stop();
++ int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));
++ int sandbox_errno = errno; // capture before the respawns spawn threads
++ server_stream_session_manager_start();
++ common_log_resume(common_log_main());
++ if (sandbox == LLAMAFILE_SANDBOX_FAILED) {
++ SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",
++ strerror(sandbox_errno));
++ clean_up();
++ return 1;
++ } else if (llamafile_sandbox_is_active(sandbox)) {
++ SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,
++ llamafile_sandbox_describe(sandbox));
++ } else {
++ SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));
++ }
++ }
++#endif
++
+ // start the HTTP server before loading the model to be able to serve /health requests
+ if (!ctx_http.start()) {
+ clean_up();
+@@ -687,6 +830,13 @@ static int server_run_instance(common_params & params, server_role & role, commo
+ ctx_server.terminate();
+ };
+
++ // Provide shutdown function to caller (for combined mode)
++ if (opts.on_shutdown_available) {
++ opts.on_shutdown_available([&ctx_server]() {
++ ctx_server.terminate();
++ });
++ }
++
+ if (opts.install_signal_handlers) {
+ server_install_signal_handlers();
+ }
+@@ -699,6 +849,11 @@ static int server_run_instance(common_params & params, server_role & role, commo
+ return 1;
+ }
+
++ // Notify caller that server is ready (for combined mode TUI startup)
++ if (opts.on_ready) {
++ opts.on_ready(ctx_http.listening_addresses.front());
++ }
++
+ // optionally, notify router server that this instance is ready
+ std::thread monitor_thread;
+ if (role.is_managed_by_router()) {
+@@ -725,6 +880,9 @@ static int server_run_instance(common_params & params, server_role & role, commo
+ common_memory_breakdown_print(ll_ctx);
+ }
+
++#ifdef LLAMAFILE_TUI
++ server_tui_exit(opts);
++#endif
+ return 0;
+ }
+
+@@ -770,6 +928,14 @@ static int server_run(common_params & params, int argc, char ** argv, server_rol
}
int llama_server(int argc, char ** argv) {
@@ -56,10 +234,10 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
+int server_main(int argc, char ** argv,
+ std::function<void(const std::string &)> on_ready,
+ std::function<void(std::function<void()>)> on_shutdown_available) {
- std::setlocale(LC_NUMERIC, "C");
+ server_role role;
- #ifndef _WIN32
-@@ -96,6 +124,10 @@ int llama_server(int argc, char ** argv) {
+ std::setlocale(LC_NUMERIC, "C");
+@@ -782,6 +948,10 @@ int llama_server(int argc, char ** argv) {
// own arguments required by this example
common_params params;
@@ -70,170 +248,23 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp
common_init();
// start the stream session manager GC right after common init, before any HTTP route can
-@@ -111,10 +143,12 @@ int llama_server(int argc, char ** argv) {
+@@ -797,7 +967,12 @@ int llama_server(int argc, char ** argv) {
llama_backend_init();
llama_numa_init(params.numa);
-- return llama_server(params, argc, argv);
-+ return llama_server(params, argc, argv, on_ready, on_shutdown_available);
- }
-
--int llama_server(common_params & params, int argc, char ** argv) {
-+int llama_server(common_params & params, int argc, char ** argv,
-+ std::function<void(const std::string &)> on_ready,
-+ std::function<void(std::function<void()>)> on_shutdown_available) {
- bool is_run_by_cli = (argv == nullptr);
-
- common_models_handler models_handler;
-@@ -457,6 +491,103 @@ int llama_server(common_params & params, int argc, char ** argv) {
- llama_backend_free();
- };
-
-+#ifdef COSMOCC
-+ // Sandbox (pledge()/unveil(), llamafile issue #930). The whole
-+ // derivation lives in llamafile/sandbox.c so it stays unit-testable
-+ // and off the vendored diff. Placement is constrained on every side:
-+ // - after common_params_parse(): -hf downloads need connect()
-+ // - after ctx_http.init(): the TLS certificate/key are read there
-+ // - before ctx_http.start(): filters only cover threads created
-+ // after installation, so the HTTP listener/pool must not exist
-+ // - before load_model(): untrusted GGUF data is parsed sandboxed
-+ //
-+ // Combined mode hosts an in-process HTTP client (the TUI) that needs
-+ // connect(), which the accept-only sandbox forbids, so it's skipped.
-+ const bool combined_mode = on_ready || on_shutdown_available;
-+ if (combined_mode) {
-+ SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");
-+ } else {
-+ // pledge() (no outbound network / writes / exec) is applied by
-+ // default. unveil() path confinement is opt-in (--confine-reads),
-+ // because a server opens files by path at request time (multimodal
-+ // media, etc.) that can't be known when the ruleset locks.
-+ //
-+ // cosmo installs the SECCOMP/Landlock filters on the calling thread
-+ // only (no TSYNC), so quiesce every pre-existing background thread
-+ // across the call: the llama.cpp log worker (pause() joins it,
-+ // resume() respawns it under the filter) and the stream-session GC
-+ // started at the top of server_main() (stop joins it, start
-+ // respawns it; no sessions exist yet, so stopping is a no-op
-+ // beyond the join).
-+ //
-+ // When --confine-reads is on, unveil() confines reads to these
-+ // directories, so collect every on-disk path the loader opens
-+ // after the lock. Add to this list when llama.cpp gains a new file
-+ // parameter (else it'd be denied under confinement). Paths read
-+ // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf
-+ // downloads in parse) need no entry.
-+ std::vector<const char *> read_paths, rw_paths;
-+ read_paths.push_back(params.model.path.c_str());
-+ read_paths.push_back(params.mmproj.path.c_str());
-+ read_paths.push_back(params.media_path.c_str()); // request-time media
-+ read_paths.push_back(params.speculative.draft.mparams.path.c_str());
-+ read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());
-+ read_paths.push_back(params.public_path.c_str());
-+ for (const auto & la : params.lora_adapters)
-+ read_paths.push_back(la.path.c_str());
-+ for (const auto & cv : params.control_vectors)
-+ read_paths.push_back(cv.fname.c_str());
-+ rw_paths.push_back(params.slot_save_path.c_str());
-+ rw_paths.push_back(params.path_prompt_cache.c_str());
-+ rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());
-+
-+ // Features that dial out at runtime need connect(), which the
-+ // accept-only default forbids; relax to full sockets for them
-+ // (still no writes/exec). --rpc has no params field -- its handler
-+ // registers backends directly -- so detect it on argv/env.
-+ const bool needs_outbound =
-+ params.ui_mcp_proxy ||
-+ !params.server_tools.empty() ||
-+ llamafile_has(argv, "--rpc") ||
-+ (std::getenv("LLAMA_ARG_RPC") != nullptr);
-+
-+ llamafile_sandbox_spec spec = {
-+ read_paths.data(), (int) read_paths.size(),
-+ rw_paths.data(), (int) rw_paths.size(),
-+ FLAG_confine_reads, needs_outbound,
-+ };
-+ // Bind the listening socket now: pledge "anet" only covers
-+ // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be
-+ // created once the sandbox is on, while accept() on one that is
-+ // already bound stays allowed (llamafile issue #971). Binding
-+ // spawns no threads; ctx_http.start() reuses the socket.
-+ if (!ctx_http.bind_socket()) {
-+ clean_up();
-+ SRV_ERR("%s", "exiting due to HTTP server error\n");
-+ return 1;
-+ }
-+
-+ char promises[64];
-+ common_log_pause(common_log_main());
-+ server_stream_session_manager_stop();
-+ int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));
-+ int sandbox_errno = errno; // capture before the respawns spawn threads
-+ server_stream_session_manager_start();
-+ common_log_resume(common_log_main());
-+ if (sandbox == LLAMAFILE_SANDBOX_FAILED) {
-+ SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",
-+ strerror(sandbox_errno));
-+ clean_up();
-+ return 1;
-+ } else if (llamafile_sandbox_is_active(sandbox)) {
-+ SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,
-+ llamafile_sandbox_describe(sandbox));
-+ } else {
-+ SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));
-+ }
-+ }
-+#endif
-+
- // start the HTTP server before loading the model to be able to serve /health requests
- if (!ctx_http.start()) {
- clean_up();
-@@ -490,6 +621,13 @@ int llama_server(common_params & params, int argc, char ** argv) {
- // this will unblock start_loop()
- ctx_server.terminate();
- };
-+
-+ // Provide shutdown function to caller (for combined mode)
-+ if (on_shutdown_available) {
-+ on_shutdown_available([&ctx_server]() {
-+ ctx_server.terminate();
-+ });
-+ }
- }
-
- // register signal handler if not running by CLI
-@@ -530,6 +668,11 @@ int llama_server(common_params & params, int argc, char ** argv) {
- // when the HTTP server stops, clean up and exit
- clean_up();
- } else {
-+ // Notify caller that server is ready (for combined mode TUI startup)
-+ if (on_ready) {
-+ on_ready(ctx_http.listening_address);
-+ }
+- const int result = server_run(params, argc, argv, role, server_run_opts{});
++ server_run_opts opts;
++ opts.on_ready = std::move(on_ready);
++ opts.on_shutdown_available = std::move(on_shutdown_available);
++ opts.argv = argv;
+
- // optionally, notify router server that this instance is ready
- std::thread monitor_thread;
- if (child.is_child()) {
-@@ -554,5 +697,56 @@ int llama_server(common_params & params, int argc, char ** argv) {
- }
- }
-
-+#ifdef LLAMAFILE_TUI
-+ // In combined mode (callbacks provided), return normally so the caller
-+ // can handle cleanup and TUI thread joining
-+ if (on_ready || on_shutdown_available) {
-+ return 0;
-+ }
-+ // By now the program can safely exit:
-+ // Metal backend has async logging that llama_synchronize() doesn't wait for.
-+ // Without this delay, _exit() might truncate llama_memory_breakdown_print's output
-+ sleep(1);
-+ // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)
-+ _exit(0);
-+#else
- return 0;
-+#endif
++ const int result = server_run(params, argc, argv, role, opts);
+ common_log_flush(common_log_main());
+ return result;
+ }
+@@ -811,3 +986,40 @@ int llama_server(common_params & params, int argc, char ** argv) {
+ server_role role;
+ return server_run(params, argc, argv, role, opts);
}
+
+// Standalone entry point for llama-server executable
diff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
index 16c52d3e..3868571f 100644
--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch
@@ -1,7 +1,7 @@
diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp
+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp
-@@ -2382,6 +2382,16 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
+@@ -2551,6 +2551,16 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,
hints.ai_addr = reinterpret_cast<sockaddr *>(&addr);
hints.ai_addrlen = static_cast<socklen_t>(
sizeof(addr) - sizeof(addr.sun_path) + addrlen);
@@ -18,7 +18,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
#ifndef SOCK_CLOEXEC
#ifndef _WIN32
-@@ -7244,6 +7254,22 @@ void ThreadPool::cleanup_finished_threads() {
+@@ -7723,6 +7733,22 @@ void ThreadPool::cleanup_finished_threads() {
}
void ThreadPool::worker(bool is_dynamic) {
@@ -41,7 +41,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
for (;;) {
std::function<void()> fn;
{
-@@ -7261,7 +7287,13 @@ void ThreadPool::worker(bool is_dynamic) {
+@@ -7740,7 +7766,13 @@ void ThreadPool::worker(bool is_dynamic) {
break;
}
} else {
@@ -56,7 +56,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
}
idle_thread_count_--;
-@@ -14003,6 +14035,13 @@ bool enumerate_macos_keychain_certs(Callback cb) {
+@@ -14684,6 +14716,13 @@ bool enumerate_macos_keychain_certs(Callback cb) {
#if !defined(_WIN32) && !(defined(__APPLE__) && \
defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN))
@@ -70,7 +70,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp
// Common CA certificate file paths on Linux/Unix
const char **system_ca_paths() {
static const char *paths[] = {
-@@ -14020,6 +14059,10 @@ const char **system_ca_dirs() {
+@@ -14701,6 +14740,10 @@ const char **system_ca_dirs() {
static const char *dirs[] = {"/etc/ssl/certs", // Debian/Ubuntu
"/etc/pki/tls/certs", // RHEL/CentOS
"/usr/share/ca-certificates", // Other
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 3a0a2ef1..be5091ff 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -245,11 +245,15 @@ LLAMAFILE_HIGHLIGHT_KEYWORDS := $(LLAMAFILE_HIGHLIGHT_GPERF_FILES:%.gperf=o/$(MO
LLAMAFILE_SERVER_SUPPORT_OBJS := \
o/$(MODE)/llama.cpp/tools/server/server-chat.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-common.cpp.o \
+ o/$(MODE)/llama.cpp/tools/server/server-connect.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-context.cpp.o \
+ o/$(MODE)/llama.cpp/tools/server/server-decision.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-http.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-mcp.cpp.o \
+ o/$(MODE)/llama.cpp/tools/server/server-models-sched.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-models.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-queue.cpp.o \
+ o/$(MODE)/llama.cpp/tools/server/server-realtime.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-schema.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-stream.cpp.o \
o/$(MODE)/llama.cpp/tools/server/server-task.cpp.o \
@@ -301,11 +305,29 @@ LLAMAFILE_METAL_SOURCES := \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/conv.metal.zip.o \
- o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/misc.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal.zip.o \
+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/norm.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/pool.metal.zip.o \
o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal.zip.o \
diff --git a/llamafile/chatbot_cli.cpp b/llamafile/chatbot_cli.cpp
index 44ec6bfc..fad2dab4 100644
--- a/llamafile/chatbot_cli.cpp
+++ b/llamafile/chatbot_cli.cpp
@@ -82,17 +82,12 @@ static cli_chat_template_result cli_apply_chat_template_full(llama_model *model,
inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
auto chat_params = common_chat_templates_apply(templates, inputs);
- result.prompt = chat_params.prompt;
+ result.prompt = chat_params.prompt.str();
- // Initialize parser params from chat_params
- result.parser_params.format = chat_params.format;
- result.parser_params.generation_prompt = chat_params.generation_prompt;
- if (!chat_params.parser.empty()) {
- result.parser_params.parser.load(chat_params.parser);
- }
- // Enable reasoning parsing for thinking models
- result.parser_params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
- result.parser_params.reasoning_in_content = false;
+ // Initialize parser params from chat_params. Reasoning extraction is
+ // built into the parser from inputs.reasoning_format above.
+ result.parser_params = common_chat_parser_params(chat_params);
+ result.parser_params.parser = std::move(chat_params.parser);
return result;
}
@@ -408,6 +403,8 @@ int cli_main(int argc, char **argv) {
common_chat_msg prev_msg; // Previous parse result for diff computation
bool think_tag_opened = false; // Track if we've printed <think>
bool think_tag_closed = false; // Track if we've printed </think>
+ char last_reasoning_char = '\n'; // Last character of the printed reasoning
+ int n_generated = 0; // New tokens so far, for -n / --n-predict
while (n_cur < params.n_ctx) {
if (g_got_sigint) {
@@ -415,6 +412,12 @@ int cli_main(int argc, char **argv) {
break;
}
+ // -n N stops after N new tokens; -1 (default) means no limit and -2
+ // means "until the context is full", which the loop bound covers.
+ if (params.n_predict >= 0 && n_generated >= params.n_predict) {
+ break;
+ }
+
llama_token id = common_sampler_sample(sampler, ctx, -1);
common_sampler_accept(sampler, id, true);
@@ -429,7 +432,7 @@ int cli_main(int argc, char **argv) {
raw_output += token_str;
// Parse incrementally
- auto msg = common_chat_parse(raw_output, /*is_partial=*/true, parser_params);
+ auto msg = common_chat_parse(common_chat_input(raw_output), /*is_partial=*/true, parser_params);
// Compute diffs to find new content
auto diffs = common_chat_msg_diff::compute_diffs(prev_msg, msg);
@@ -442,11 +445,17 @@ int cli_main(int argc, char **argv) {
think_tag_opened = true;
}
fputs(diff.reasoning_content_delta.c_str(), stdout);
+ last_reasoning_char = diff.reasoning_content_delta.back();
fflush(stdout);
}
// Output final content (close think tag first if needed)
if (!diff.content_delta.empty()) {
if (think_tag_opened && !think_tag_closed) {
+ // The parser trims the newline that ends the reasoning
+ // (since llama.cpp v0.8.0), so put </think> on its own line
+ if (last_reasoning_char != '\n') {
+ fputs("\n", stdout);
+ }
fputs("</think>\n", stdout);
think_tag_closed = true;
}
@@ -468,6 +477,7 @@ int cli_main(int argc, char **argv) {
break;
}
n_cur++;
+ n_generated++;
}
// Ensure output ends with newline
diff --git a/llamafile/chatbot_direct.cpp b/llamafile/chatbot_direct.cpp
index e337898f..6d21f89d 100644
--- a/llamafile/chatbot_direct.cpp
+++ b/llamafile/chatbot_direct.cpp
@@ -61,7 +61,7 @@ public:
if (use_chat_parser) {
std::string token_str = token_to_piece(g_ctx, id, true);
raw_output += token_str;
- auto msg = common_chat_parse(raw_output, true, g_chat_syntax);
+ auto msg = common_chat_parse(common_chat_input(raw_output), true, g_chat_syntax);
auto diffs = common_chat_msg_diff::compute_diffs(prev_msg, msg);
for (const auto &diff : diffs) {
diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cpp
index 080b0cad..b6b6c28e 100644
--- a/llamafile/chatbot_main.cpp
+++ b/llamafile/chatbot_main.cpp
@@ -276,17 +276,11 @@ int main(int argc, char **argv) {
try {
auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);
- g_chat_syntax.format = chat_params.format;
- g_chat_syntax.generation_prompt = chat_params.generation_prompt;
- // Load the PEG parser if one was provided
- if (!chat_params.parser.empty()) {
- g_chat_syntax.parser.load(chat_params.parser);
- }
-
- // Copy reasoning format to chat syntax for use by the parser at runtime
- g_chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
- g_chat_syntax.reasoning_in_content = false;
+ // The PEG parser (with reasoning extraction, from
+ // inputs.reasoning_format above) comes built in chat_params.
+ g_chat_syntax = common_chat_parser_params(chat_params);
+ g_chat_syntax.parser = std::move(chat_params.parser);
// Print detected format
if (!FLAG_nologo && g_chat_syntax.format != COMMON_CHAT_FORMAT_CONTENT_ONLY) {
diff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cpp
index 6c676517..bd0a32bc 100644
--- a/llamafile/chatbot_repl.cpp
+++ b/llamafile/chatbot_repl.cpp
@@ -86,7 +86,7 @@ std::string apply_chat_template_with_thinking(
inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);
- return chat_params.prompt;
+ return chat_params.prompt.str();
}
void on_sigint(int sig) {
diff --git a/llamafile/metal.c b/llamafile/metal.c
index e09800aa..550fc3cf 100644
--- a/llamafile/metal.c
+++ b/llamafile/metal.c
@@ -107,11 +107,29 @@ __static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/quantize.h");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/conv.metal");
-__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/misc.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal");
+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/norm.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/pool.metal");
__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal");
@@ -181,11 +199,29 @@ static const struct MetalSource {
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal", "kernels/argsort.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal", "kernels/binbcast.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/conv.metal", "kernels/conv.metal"},
- {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa.metal", "kernels/fa.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal", "kernels/fa_aux.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal", "kernels/fa_common.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal", "kernels/fa_f16.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal", "kernels/fa_f32.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal", "kernels/fa_q4_0.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal", "kernels/fa_q4_1.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal", "kernels/fa_q5_0.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal", "kernels/fa_q5_1.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal", "kernels/fa_q8_0.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal", "kernels/fa_tensor.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal", "kernels/fa_vec_common.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal", "kernels/fa_vec_f16.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal", "kernels/fa_vec_f32.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal", "kernels/fa_vec_q4_0.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal", "kernels/fa_vec_q4_1.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal", "kernels/fa_vec_q5_0.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal", "kernels/fa_vec_q5_1.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal", "kernels/fa_vec_q8_0.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal", "kernels/gated_delta_net.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/misc.metal", "kernels/misc.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal", "kernels/mul_mm.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal", "kernels/mul_mv.metal"},
+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal", "kernels/mul_mv_mma.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/norm.metal", "kernels/norm.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/pool.metal", "kernels/pool.metal"},
{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal", "kernels/quantize.metal"},
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 5e4e993e..d41a3a28 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -39,7 +39,7 @@ def pytest_addoption(parser):
"--gpu",
action="store",
default=None,
- choices=["auto", "apple", "amd", "nvidia", "disable"],
+ choices=["auto", "apple", "amd", "nvidia", "vulkan", "disable"],
help="GPU mode to use (disable for CPU-only)",
)
parser.addoption(
diff --git a/tests/integration/tests/test_cli.py b/tests/integration/tests/test_cli.py
index d636e5fc..0026245f 100644
--- a/tests/integration/tests/test_cli.py
+++ b/tests/integration/tests/test_cli.py
@@ -31,6 +31,25 @@ class TestCLIBasic:
assert result.returncode == 0
+ def test_cli_n_predict_limits_output(self, llamafile, timeouts):
+ """Test that -n caps the number of generated tokens.
+
+ Counting to 200 takes far more than 64 tokens, so without the cap
+ both runs would print the same long answer.
+ """
+ prompt = "Count from 1 to 200, separated by commas."
+ short = llamafile.run_cli(prompt, thinking=False, timeout=timeouts.cli,
+ extra_args=["-n", "8", "--temp", "0"])
+ longer = llamafile.run_cli(prompt, thinking=False, timeout=timeouts.cli,
+ extra_args=["-n", "64", "--temp", "0"])
+
+ assert short.returncode == 0, f"CLI failed: {short.stderr}"
+ assert longer.returncode == 0, f"CLI failed: {longer.stderr}"
+ assert 0 < len(short.stdout.strip()) < len(longer.stdout.strip()), (
+ f"-n 8 output ({len(short.stdout)} chars) is not shorter than "
+ f"-n 64 output ({len(longer.stdout)} chars)"
+ )
+
@pytest.mark.cli
@pytest.mark.thinking
diff --git a/tests/integration/tests/test_gpu.py b/tests/integration/tests/test_gpu.py
index 2749fa7e..a6c0885c 100644
--- a/tests/integration/tests/test_gpu.py
+++ b/tests/integration/tests/test_gpu.py
@@ -219,7 +219,8 @@ class TestGPUAcceleration:
complete``, i.e. graphs actually engaged for this model. ggml may
legitimately refuse them (pre-Volta GPU, unsupported node type); it
then logs ``disabling CUDA graphs due to ...`` and the runtime check
- is skipped rather than failed.
+ is skipped rather than failed. Since llama.cpp v0.8.0 (#30327) ggml
+ prints both lines only with ``GGML_CUDA_GRAPH_DEBUG`` set.
The module's own log lines go to stderr, not to --log-file (they are
written by the ggml logger inside the DSO), so stderr is captured too.
@@ -243,7 +244,8 @@ class TestGPUAcceleration:
proc = runner.start_server(port=server_port, log_file=log_path,
stderr_file=stderr_path,
- extra_args=["--verbose"])
+ extra_args=["--verbose"],
+ env={"GGML_CUDA_GRAPH_DEBUG": "1"})
try:
ready = LlamafileRunner.wait_for_server(
server_port, timeout=timeouts.server_ready, proc=proc
diff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.py
index 3e71fd50..128c12b1 100644
--- a/tests/integration/utils/llamafile.py
+++ b/tests/integration/utils/llamafile.py
@@ -238,7 +238,9 @@ class LlamafileRunner:
Args:
executable: Path to llamafile binary or pre-built .llamafile
model: Path to model file (None for pre-built llamafiles)
- gpu: GPU mode - "auto", "apple", "amd", "nvidia", or None for CPU
+ gpu: GPU mode passed as --gpu - "auto", "apple", "amd", "nvidia",
+ "vulkan", or "disable" for CPU only; None passes no flag,
+ so llamafile picks a GPU itself
default_thinking: Default thinking mode when callers don't specify
one. Set to True for tests that exercise thinking
behavior; False otherwise so small reasoning
@@ -409,6 +411,7 @@ class LlamafileRunner:
extra_args: list[str] | None = None,
log_file: str | None = None,
stderr_file: str | None = None,
+ env: dict[str, str] | None = None,
) -> subprocess.Popen:
"""Start llamafile in server mode.
@@ -424,6 +427,8 @@ class LlamafileRunner:
file. Messages logged by a GPU backend module
(ggml-cuda.so, ...) are emitted by the module's own ggml
logger and reach stderr, not --log-file.
+ env: Extra environment variables for the server process, on top
+ of the inherited environment.
Returns:
Popen process handle (caller must terminate)
@@ -451,6 +456,7 @@ class LlamafileRunner:
stdout=subprocess.DEVNULL,
stderr=stderr,
text=True,
+ env={**os.environ, **env} if env else None,
)
finally:
if stderr_file:
diff --git a/transcribefile/main.cpp b/transcribefile/main.cpp
index 0429472f..7942008a 100644
--- a/transcribefile/main.cpp
+++ b/transcribefile/main.cpp
@@ -65,9 +65,8 @@ static void print_wrapper_help(void) {
" --verbose show GPU loader and backend logging\n"
" (suppressed by default)\n"
"transcribefile notes:\n"
- " GPU backends wired up in this build: metal (macOS on Apple\n"
- " Silicon; used by --backend auto/metal, first run compiles\n"
- " ggml-metal.dylib and caches it under ~/.transcribefile).\n"
+ " No GPU backend in this build: runs on CPU. Metal is disabled\n"
+ " until transcribe.cpp's ggml matches llamafile's again;\n"
" vulkan and cuda are not available yet.\n"
" Default arguments are read from .args in the executable's zip\n"
" store (one per line), so models can be bundled with zipalign.\n");
@@ -86,7 +85,21 @@ static void print_wrapper_help(void) {
// FLAG_gpu stays AUTO (not APPLE) even for --backend metal so a failed
// load degrades silently here; transcribe.cpp reports the missing
// backend itself, with wording that matches its own CLI.
+// Metal is disabled for now. The dylib is built from llamafile's bundled
+// llama.cpp ggml-metal sources and registered with transcribe.cpp's own
+// ggml (0.15.2), which only works while both trees agree on the backend
+// interface structs. llama.cpp v0.8.0 (ggml 0.28) added alloc_buffer_n
+// and get_alloc_size_n to ggml_backend_buffer_type_i, so transcribe.cpp
+// called a NULL slot in ggml_backend_buft_get_alignment and crashed on
+// model load. Re-enable together with a transcribe.cpp update whose ggml
+// matches llama.cpp's.
+static constexpr bool kMetalEnabled = false;
+
static void load_gpu_backends(char ** argv) {
+ if (!kMetalEnabled) {
+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;
+ return;
+ }
const char * backend = "auto";
for (char ** p = argv + 1; p && *p; ++p) {
if (!strcmp(*p, "--backend") && p[1]) {