Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Update llama.cpp to v0.8.0 (#1077)
Davide Eynarde5a6ad951cc7
diff --git a/agent.cpp.patches/patches/src_agent.cpp.patch b/agent.cpp.patches/patches/src_agent.cpp.patchnew file mode 100644index 00000000..6f07a1af--- /dev/null+++ b/agent.cpp.patches/patches/src_agent.cpp.patch@@ -0,0 +1,12 @@+diff --git a/src/agent.cpp b/src/agent.cpp+--- a/agent.cpp/src/agent.cpp++++ b/agent.cpp/src/agent.cpp+@@ -171,7 +171,7 @@ Agent::build_prompt_tokens()++ auto params = common_chat_templates_apply(model->get_templates(), inputs);++- return model->tokenize(params.prompt);++ return model->tokenize(params.prompt.str());+ }++ booldiff --git a/agent.cpp.patches/patches/src_model.cpp.patch b/agent.cpp.patches/patches/src_model.cpp.patchindex b2a18110..68178d49 100644--- a/agent.cpp.patches/patches/src_model.cpp.patch+++ b/agent.cpp.patches/patches/src_model.cpp.patch@@ -180,17 +180,45 @@ diff --git a/src/model.cpp b/src/model.cpptrue) < 0) {return {}; // Return empty vector on failure}-@@ -261,6 +276,9 @@ parse_with_parser(const common_peg_arena& parser,+@@ -232,19 +247,6 @@ Model::tokenize(const std::string& prompt) const++ namespace {++-/// Deserializes the PEG parser derived from the chat template. An empty+-/// definition yields an empty arena, which common_chat_peg_parse() treats as a+-/// pure-content parser.+-common_peg_arena+-load_parser(const std::string& parser_definition)+-{+- common_peg_arena arena;+- if (!parser_definition.empty()) {+- arena.load(parser_definition);+- }+- return arena;+-}+-+ common_chat_msg+ parse_with_parser(const common_peg_arena& parser,+ const common_chat_params& params,+@@ -259,12 +261,14 @@ parse_with_parser(const common_peg_arena& parser,+ // The parser is derived from the chat template together with the+ // generation prompt it was built against, and parsing needs both// (ggml-org/llama.cpp#18675). Without them tool calls stay as raw text.- syntax.generation_prompt = params.generation_prompt;- syntax.parse_tool_calls = true;-+ // llamafile: extract <think>...</think> into reasoning_content instead-+ // of leaving it inline in content (callers decide how to show it).-+ syntax.reasoning_format = COMMON_REASONING_FORMAT_AUTO;+- syntax.generation_prompt = params.generation_prompt;+- syntax.parse_tool_calls = true;++ // llamafile: since llama.cpp v0.8.0 tool calls are always parsed, and++ // reasoning extraction is built into the parser at apply time++ // (inputs.reasoning_format), so there are no parse-time switches left.++ syntax.generation_prompt = common_chat_input(params.generation_prompt);try {auto parsed_msg =-@@ -276,6 +294,62 @@ parse_with_parser(const common_peg_arena& parser,+- common_chat_peg_parse(parser, response, false, syntax);++ common_chat_peg_parse(parser, common_chat_input(response), false, syntax);+ parsed_msg.role = "assistant";+ return parsed_msg;+ } catch (const std::exception& e) {+@@ -276,6 +280,62 @@ parse_with_parser(const common_peg_arena& parser,}}@@ -253,7 +281,7 @@ diff --git a/src/model.cpp b/src/model.cpp}common_chat_msg-@@ -288,11 +362,97 @@ Model::generate(const std::vector<common_chat_msg>& messages,+@@ -288,28 +348,213 @@ Model::generate(const std::vector<common_chat_msg>& messages,inputs.tools = tools;inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;inputs.add_generation_prompt = true;@@ -261,7 +289,7 @@ diff --git a/src/model.cpp b/src/model.cpp+ inputs.enable_thinking = config_.enable_thinking; // llamafile: was false+ // llamafile: build the template's parser with reasoning rules so any+ // <think>...</think> the model emits anyway is extracted into-+ // reasoning_content (matching syntax.reasoning_format in the parse).++ // reasoning_content.+ inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;auto params =@@ -350,19 +378,24 @@ diff --git a/src/model.cpp b/src/model.cpp+ }+// Tokenize the prompt- std::vector<llama_token> prompt_tokens = tokenize(params.prompt);+- std::vector<llama_token> prompt_tokens = tokenize(params.prompt);++ std::vector<llama_token> prompt_tokens = tokenize(params.prompt.str());if (prompt_tokens.empty()) {-@@ -308,8 +468,114 @@ Model::generate(const std::vector<common_chat_msg>& messages,- parser_source_ = params.parser;+ throw ModelError("failed to tokenize prompt");}-- return parse_with_parser(-- parser_arena_, params, response, config_.chat_format);+ std::string response = generate_from_tokens(prompt_tokens, callback);++- // The PEG parser only changes when the rendered template changes, so keep+- // the deserialized arena around instead of re-parsing it every turn.+- if (parser_source_ != params.parser) {+- parser_arena_ = load_parser(params.parser);+- parser_source_ = params.parser;+ common_chat_msg parsed_msg;+ std::optional<ModelError> parse_error;+ try {+ parsed_msg = parse_with_parser(-+ parser_arena_, params, response, config_.chat_format);++ params.parser, params, response, config_.chat_format);+ } catch (const ModelError& e) {+ // llamafile: a parse refusal must not kill the agent loop. Fall+ // back to content-only, with the reasoning split off so it is@@ -375,7 +408,10 @@ diff --git a/src/model.cpp b/src/model.cpp+ parse_error = e;+ parsed_msg.content = response;+ split_reasoning(params, parsed_msg);-+ }+ }+-+- return parse_with_parser(+- parser_arena_, params, response, config_.chat_format);+ // A reply that ends inside a reasoning block it never closed, with+ // nothing else, holds the model's answer in that block: small models+ // do this after a tool result. Take it as the content, so a tool call@@ -469,7 +505,17 @@ diff --git a/src/model.cpp b/src/model.cpp}common_chat_msg-@@ -338,10 +604,16 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,+@@ -317,8 +562,7 @@ parse_response(const common_chat_params& params,+ const std::string& response,+ std::optional<common_chat_format> format_override)+ {+- return parse_with_parser(+- load_parser(params.parser), params, response, format_override);++ return parse_with_parser(params.parser, params, response, format_override);+ }++ std::string+@@ -338,10 +582,16 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,common_prefix++;}@@ -488,7 +534,7 @@ diff --git a/src/model.cpp b/src/model.cppprocessed_tokens_.resize(common_prefix);n_past_ = common_prefix;}-@@ -370,15 +642,15 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,+@@ -370,15 +620,15 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,i += batch_size;}diff --git a/agent.cpp.patches/patches/src_model.h.patch b/agent.cpp.patches/patches/src_model.h.patchindex efd9bfae..f0b8f5d0 100644--- a/agent.cpp.patches/patches/src_model.h.patch+++ b/agent.cpp.patches/patches/src_model.h.patch@@ -27,3 +27,14 @@ diff --git a/src/model.h b/src/model.h// Owning handles for this Model's LoRA adapters, in ModelConfig::loras// order. Set on this Model's context only, so different Model instances// sharing the same ModelWeights can carry independent adapters.+@@ -236,10 +240,6 @@ class Model+ std::vector<llama_token> processed_tokens_; // Track tokens in KV cache+ int n_past_ = 0; // Track position in KV cache+ ModelConfig config_;+- // Chat parser derived from the template, cached across turns along with+- // the definition it was built from+- std::string parser_source_;+- common_peg_arena parser_arena_;+ };++ } // namespace agent_cppdiff --git a/llama.cpp b/llama.cppindex 7ab4ee7b..8642541b 160000--- a/llama.cpp+++ b/llama.cpp@@ -1 +1 @@-Subproject commit 7ab4ee7baad2d920464cbacfad4f4b07cf111fd2+Subproject commit 8642541b255bfb5dc39cd351a178be09013a55d6diff --git a/llama.cpp.patches/README.md b/llama.cpp.patches/README.mdindex 96d5e109..788c4196 100644--- a/llama.cpp.patches/README.md+++ b/llama.cpp.patches/README.md@@ -46,18 +46,18 @@ The `GGML_CALL` macro (defined as `__attribute__((__ms_abi__))` when `GGML_MULTI| `ggml_include_ggml-backend.h.patch` | Defines the `GGML_CALL` macro; adds it to the five `get_proc_address` return typedefs (`ggml_backend_split_buffer_type_t`, `ggml_backend_set_n_threads_t`, `ggml_backend_dev_get_extra_bufts_t`, `ggml_backend_set_abort_callback_t`, `ggml_backend_get_features_t`) || `ggml_include_ggml-cpu.h.patch` | Adds `GGML_CALL` to declarations of `ggml_backend_cpu_set_n_threads` and `ggml_backend_cpu_set_abort_callback` (returned via `get_proc_address`) || `ggml_include_ggml-cuda.h.patch` | Adds `GGML_CALL` to the declaration of `ggml_backend_cuda_register_host_buffer` (upstream removed the row-split multi-GPU buffer in b10052, so the former `ggml_backend_cuda_split_buffer_type` annotation is gone) |-| `ggml_src_ggml-backend-impl.h.patch` | Adds `GGML_CALL` to all 49+ function pointers across the five interface structs (`ggml_backend_buffer_type_i`, `ggml_backend_buffer_i`, `ggml_backend_i`, `ggml_backend_device_i`, `ggml_backend_reg_i`); also adds `free_struct` callback (see Cross-Module Memory below) |-| `ggml_src_ggml-backend.cpp.patch` | Adds `GGML_CALL` to CPU buffer, buffer type, and multi-buffer callback implementations; also adds `free_struct` support (see Cross-Module Memory below) |+| `ggml_src_ggml-backend-impl.h.patch` | Adds `GGML_CALL` to all 49+ function pointers across the five interface structs (`ggml_backend_buffer_type_i`, `ggml_backend_buffer_i`, `ggml_backend_i`, `ggml_backend_device_i`, `ggml_backend_reg_i`), including the optional `alloc_buffer_n`/`get_alloc_size_n` buffer-type members added in v0.8.0; also adds `free_struct` callback (see Cross-Module Memory below) |+| `ggml_src_ggml-backend.cpp.patch` | Adds `GGML_CALL` to CPU buffer, buffer type, and multi-buffer callback implementations; also adds `free_struct` support (see Cross-Module Memory below), and carries upstream PR #25584 (see Upstream fixes carried ahead of a release) || `ggml_src_ggml-cpu_ggml-cpu.cpp.patch` | Adds `GGML_CALL` to all CPU backend, device, and registry callback implementations, plus `get_proc_address`-returned functions (`set_n_threads`, `set_abort_callback`, `get_extra_buffers_type`, `get_features`) || `ggml_src_ggml-cpu_amx_amx.cpp.patch` | Adds `GGML_CALL` to all AMX buffer and buffer type callback implementations (10 functions) || `ggml_src_ggml-cpu_repack.cpp.patch` | Adds `GGML_CALL` to CPU repack buffer and buffer type callback implementations (5 functions) |-| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Adds `GGML_CALL` to all CUDA backend callback implementations (60+ functions); also adds `free_struct` and TinyBLAS BF16 guard (see below) |+| `ggml_src_ggml-cuda_ggml-cuda.cu.patch` | Adds `GGML_CALL` to all CUDA backend callback implementations (60+ functions, including the pinned host buffer type's `get_max_size`, new in v0.8.0); also adds `free_struct` and TinyBLAS BF16 guard (see below) || `ggml_src_ggml-metal_ggml-metal.cpp.patch` | Adds `GGML_CALL` to all Metal backend callback implementations (62 functions); also adds `free_struct` (see below) |-| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Adds `GGML_CALL` to all Vulkan backend callback implementations; also adds `free_struct` and a heap memory underflow fix (see below) |+| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Adds `GGML_CALL` to all Vulkan backend callback implementations; also adds `free_struct` (see below) || `ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch` | Adds `GGML_CALL` to the backend callback declarations. b11100 split the Vulkan backend into several translation units and moved the interface structs into `ggml-vulkan-buffers.cpp`, so the callbacks it holds became non-static and are declared here; declaration and definition must agree on the convention || `ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch` | Adds the `free_struct` entry to `ggml_backend_vk_buffer_interface`, which b11100 moved into this new file || `ggml_src_ggml-metal_ggml-metal-fusion.h.patch` / `.cpp.patch` | Adds `GGML_CALL` to the `add_alloc_dep` function pointer these pass through (see Graph-optimize allocation dependencies below), and includes `ggml-backend.h` for the macro |-| `ggml_src_ggml-backend-meta.cpp.patch` | Adds `GGML_CALL` to all meta-device, meta-buffer-type, meta-buffer, and meta-backend callback implementations (the meta backend aggregates several simple backends behind one interface, so its callbacks are reached through the same function-pointer structs) |+| `ggml_src_ggml-backend-meta.cpp.patch` | Adds `GGML_CALL` to all meta-device, meta-buffer-type (including its `alloc_buffer_n`, new in v0.8.0), meta-buffer, and meta-backend callback implementations (the meta backend aggregates several simple backends behind one interface, so its callbacks are reached through the same function-pointer structs) |### Cross-Module Memory Management@@ -111,8 +111,9 @@ These patches address compatibility issues when building with Cosmopolitan libc| Patch | Description ||-------|-------------|| `common_arg.cpp.patch` | Adds `COSMOCC` platform detection for `PATH_MAX` (includes `linux/limits.h`) |-| `common_common.cpp.patch` | Adds platform-aware cache directory detection for Cosmopolitan (`fs_get_cache_directory`: checks `LOCALAPPDATA`, `XDG_CACHE_HOME`, falls back to `~/.cache/`) and the same for the config directory (`fs_get_config_directory`, new in b10441: `APPDATA`, `XDG_CONFIG_HOME`, `~/.config/`) — both `#else` out to `#error Unknown architecture` upstream, since cosmocc defines none of `__linux__`/`__APPLE__`/`_WIN32`; also adds mmproj model size estimation to GPU fit params so the fit algorithm reserves enough VRAM for multimodal projectors |+| `common_common.cpp.patch` | Adds platform-aware cache directory detection for Cosmopolitan (`fs_get_cache_directory`: checks `LOCALAPPDATA`, `XDG_CACHE_HOME`, falls back to `~/.cache/`) and the same for the config directory (`fs_get_config_directory`: `APPDATA`, `XDG_CONFIG_HOME`, `~/.config/`). cosmocc defines none of `__linux__`/`__APPLE__`/`_WIN32`; until v0.8.0 that hit `#error Unknown architecture`, and since then it falls to upstream's generic XDG branch, which compiles but would move a Windows host's cache out of `%LOCALAPPDATA%` and orphan models already downloaded there, so the `COSMOCC` branch stays. (Until v0.8.0 it also added the mmproj file size to every device's fit target. Upstream's server reserves the projector itself since #21489, per device, and llamafile's `--cli`/`--chat` load models without fit, so that hunk only double-counted: on a 24 GB GPU, Gemma-4-31B + mmproj lost 3 offloaded layers. Dropped.) || `common_download.cpp.patch` | Adds `COSMOCC` platform detection for `PATH_MAX` |+| `common_common.h.patch` | Keeps the server's default `--port` at **8080**, llamafile's documented default (help text, docs, integration tests, users' scripts). Upstream moved it to 9931 in v0.8.0 (#30159); this is a deliberate product choice, not a portability fix || `vendor_sheredom_subprocess.h.patch` | Under `__COSMOPOLITAN__`, rewrites `subprocess_error_from_errno()`'s `switch` over `E*` values as an `if`/`else` chain. Cosmopolitan resolves errno at runtime (one binary, every host OS), so those are not constant expressions and cannot be case labels — upstream's version fails with `case value is not a constant expression`. Same mapping, no behavior change. Needed since b10441, which added this function; the header only enters the build when `LLAMA_SUBPROCESS` is set (see Server Integration). Also decodes the child's exit status on Windows, where Cosmopolitan's `waitpid()` returns the NT exit code as the wait status: Cosmopolitan children exit with `code << 8`, native ones (`cmd.exe`) with the plain code, so upstream's `WIFEXITED`/`WEXITSTATUS` turned `exit 3` into 1 and `exit 128` into 0. A status with its low byte set is now taken as a native exit code (#1066) || `tools_server_server-tools.cpp.patch` | Under `__COSMOPOLITAN__`, picks the shell for the built-in `exec_shell_command` and `get_info` tools at run time with `IsWindows()`: `cmd.exe /c` on Windows, `sh -c` / `uname -a` elsewhere (and always inside a `--tools-runtime` isolate), the same choice upstream makes at compile time with `_WIN32`, which the cosmocc build never defines. The name is spelled `cmd.exe` because Cosmopolitan's `posix_spawnp()` does not try the `.exe` suffix when it searches `PATH`. Without it both tools fail on Windows with *"failed to spawn process"* (#1066) |@@ -123,7 +124,8 @@ Cosmopolitan libc has specific behaviors with condition variables and signals th| Patch | Description ||-------|-------------|| `common_log.cpp.patch` | Adds `#include <csignal>`; blocks `SIGINT`/`SIGTERM` on logger thread via `pthread_sigmask` to prevent `EINTR` exceptions; replaces the untimed `cv.wait()` calls with `wait_for(30s)` loops (the worker's `cv_new` wait, plus the queue-full `cv_full` waits in `add()` and in `add_json()`, new upstream in b11100) to work around XNU futex timeout bug (~72 minute expiry). The `add*()` waits run on the logging caller's thread, so the signal mask does not cover them |-| `tools_server_server-models.cpp.patch` | Adds `#include <csignal>`; blocks signals via `pthread_sigmask` on `server_monitor`'s thread (b11100 replaced the per-model `stopping_thread` this used to cover with one monitor thread watching every child); replaces untimed `cv.wait()` with `wait_for(30s)` loops on every model-lifecycle wait (`unload_lru`, the reload-drain wait, `unload_all`, the `is_reloading` guard in `load`, and the generic `wait()` predicate helper) to work around the XNU futex timeout bug |+| `tools_server_server-models.cpp.patch` | Adds `#include <csignal>`; blocks signals via `pthread_sigmask` on `server_monitor`'s thread (b11100 replaced the per-model `stopping_thread` this used to cover with one monitor thread watching every child); replaces untimed `cv.wait()` with `wait_for(30s)` loops on every model-lifecycle wait (the reload-drain wait, `unload_all`, the `is_reloading` guard in `load`, and the generic `wait()` predicate helper) to work around the XNU futex timeout bug |+| `tools_server_server-models-sched.cpp.patch` | Same `wait_for(30s)` treatment for the two waits in `server_models_sched::make_room()` (the memory-measurement wait and the LRU-unload wait). v0.8.0 moved the LRU eviction out of `server_models::unload_lru()` into this new scheduler || `tools_server_server-queue.cpp.patch` | Adds missing includes (`<cerrno>`, `<system_error>`, `<csignal>`); blocks `SIGINT`/`SIGTERM` on queue thread (the `yield_to_queue` worker thread is spawned after this, so it inherits the mask); replaces `wait()` with `wait_for(30s)` loops in five locations (`wait_until_no_sleep`, main loop, `recv`, plus `worker_loop` and `yield_to_queue`, both new upstream in b10441); runs `yield_to_queue()` inline when a GPU backend is loaded (see "GPU decode must stay on the main thread" below) || `tools_server_server-stream.cpp.patch` | Adds `#include <csignal>`; blocks signals on the stream-session GC thread via `pthread_sigmask`. Without it, Ctrl+C on `llama-server` aborts with *"libc++abi: terminating due to uncaught exception ... condition_variable timed_wait failed: Interrupted system call"* instead of shutting down: `SIGINT` is delivered to that thread while it sits in `gc_wake_cv.wait_for(60s)`, `pthread_cond_timedwait()` returns `EINTR`, and libcxx rethrows it as `std::system_error` that nothing catches. This thread was the last one in the server with a condition-variable wait and no signal mask (the log worker, task queue, model-stopping thread and httplib pool were already covered). Pre-existing, not introduced by b10441; combined TUI mode is unaffected || `vendor_cpp-httplib_httplib.cpp.patch` | Fixes httplib thread pool with `wait_for()` instead of `wait()` for XNU futex compatibility; passes the full `sockaddr_un` for a pathname Unix socket, since Cosmopolitan's `bind()`/`connect()` reject addresses shorter than `struct sockaddr` and a `--host` socket path under 14 bytes failed with `EINVAL`; also see HTTPS / TLS Support below |@@ -186,7 +188,7 @@ These patches restore llamafile's optimized CPU kernels (TinyBLAS matmul, AVX-51| Patch | Description ||-------|-------------|-| `ggml_src_ggml-cpu_ggml-cpu.c.patch` | Routes MoE matmul (`ggml_compute_forward_mul_mat_id`) through `llamafile_mixmul` / `llamafile_mixmul_iqk`, mirroring the dense-matmul `llamafile_sgemm` hook; reserves work-buffer space for the MoE kernel in `ggml_graph_plan` via `llamafile_mixmul_needs` |+| `ggml_src_ggml-cpu_ggml-cpu.c.patch` | Routes MoE matmul (`ggml_compute_forward_mul_mat_id`) through `llamafile_mixmul` / `llamafile_mixmul_iqk`, mirroring the dense-matmul `llamafile_sgemm` hook; reserves work-buffer space for the MoE kernel in `ggml_graph_plan` via `llamafile_mixmul_needs` (in the `GGML_OP_MUL_MAT_ID` case, after upstream's tiled workspace; on the v0.8.0 bump the hunk's context drifted and `patch` silently placed it in `GGML_OP_OUT_PROD`, so check its position on each bump). v0.8.0 replaced the IQ panel path (`iqp.cpp`) with a tiled k-quant matmul (`ggml-cpu/tiled/`) that runs before these hooks, but it is compiled only under `__AVX__`/`__AVX2__`/`__AVX512VNNI__` and the cosmocc build compiles `ggml-cpu` at baseline ISA, so in llamafile it always declines and the TinyBLAS hooks keep handling those matmuls || `ggml_src_ggml-cpu_ops.cpp.patch` | Routes flash-attention inner loops through llamafile's AVX-512 helpers (`llamafile_fa_vec_dot_f16`, `llamafile_fa_fp16_to_fp32_row`, `llamafile_fa_simd_gemm`) in both the one-chunk and tiled FA paths; also accumulates VKQ in f32 on CPUs lacking native f16 FMA (avoiding costly f16↔f32 round-trips per KV step) || `src_llama-context.cpp.patch` | Defaults `-fa auto` to **off** on CPU-only setups (no GPU devices), since the CPU flash-attention path is slower than the non-FA path on x86; users can still force `-fa on` for memory savings on long contexts |@@ -197,15 +199,15 @@ These patches integrate llamafile's file handling APIs for loading models from b| Patch | Description ||-------|-------------|| `src_llama-mmap.h.patch` | Adds `has_premapped_content()`, `premapped_content()`, and `get_llamafile()` methods to `llama_file` class |-| `src_llama-mmap.cpp.patch` | Under `COSMOCC`, redirects file open/read/seek/tell/close through llamafile API (`llamafile_open_gguf`, `llamafile_read`, etc.); adds premapped content support to `llama_mmap` using llamafile reference counting (`llamafile_ref`/`llamafile_unref`); skips `munmap` for premapped content |+| `src_llama-mmap.cpp.patch` | Under `COSMOCC`, redirects file open/read/seek/tell/close through llamafile API (`llamafile_open_gguf`, `llamafile_read`, etc.); adds premapped content support to `llama_mmap` using llamafile reference counting (`llamafile_ref`/`llamafile_unref`); skips `munmap` for premapped content; on Windows makes `unmap_fragment()` a no-op, as upstream's `_WIN32` branch does (cosmocc never defines `_WIN32`, so llamafile runs the POSIX branch everywhere, and Windows can only unmap a whole view: cosmo's `munmap` fails with `ENOTSUP`/`EINVAL` on a sub-range, which printed "munmap failed" warnings and left the view mapped at exit). Memory use then matches the official llama.cpp Windows build: the whole file stays mapped after GPU offload in both || `ggml_src_gguf.cpp.patch` | Adds `tell()`/`seek()` to `gguf_reader`; under `COSMOCC`, adds `gguf_llamafile_reader` that reads via llamafile API; templatizes `gguf_init_from_reader_impl` so both readers work; redirects `gguf_init_from_file` through `llamafile_open_gguf` (supports `/zip/` paths, `.llamafile` containers) |### Server Integration| Patch | Description ||-------|-------------|-| `tools_server_server.cpp.patch` | Renames upstream's `llama_server()` to `server_main()` and adds `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode; adds Metal/GPU backend trigger before `common_init()`; installs the sandbox (`llamafile_sandbox_server()`, issue #930) — the mechanism lives in `llamafile/sandbox.c`; the patch fills a `llamafile_sandbox_spec` with the on-disk paths the loader opens after the lock (model, mmproj, media dir, LoRA, draft model, control vectors, public path as read; slot-save and prompt cache as read-write), detects outbound-needing features (`--rpc` via argv/env, MCP proxy, server tools) to relax `anet`→`inet`, passes `FLAG_confine_reads` for opt-in unveil(), binds the listening socket first (`ctx_http.bind_socket()`, so a `--host /path.sock` Unix socket exists before `anet` forbids creating one, issue #971), quiesces the log worker (`common_log_pause`/`resume`, so no thread escapes the per-thread filter) around the call, before the HTTP listener spawns and before model load, and skips it in combined/GPU modes; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, `--unsecure` consumption (`llamafile_consume_flag`), and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |-| `tools_server_server-http.h.patch` / `tools_server_server-http.cpp.patch` | Splits the bind out of `server_http_context::start()` into `bind_socket()`, which `start()` calls if it hasn't run yet, so the server can bind before installing the sandbox (issue #971) |+| `tools_server_server.cpp.patch` | Makes `server_main()` the real entry point (upstream's `llama_server(argc, argv)` becomes a thin wrapper around it) and adds `on_ready`/`on_shutdown_available` callbacks for combined TUI+server mode. v0.8.0 (#31113) split the server into `server_run_router()` / `server_run_instance()` / `server_run()`; the callbacks (and `argv`, for `--rpc` detection) now travel in upstream's `server_run_opts`, and `on_ready`, `on_shutdown_available`, and the sandbox all live in `server_run_instance()` (the router stays unsandboxed, as before). The sandbox is also skipped under `--connect` (new in v0.8.0), which spawns `llama-connect` after the lock and would otherwise fail to start; adds Metal/GPU backend trigger before `common_init()`; installs the sandbox (`llamafile_sandbox_server()`, issue #930) — the mechanism lives in `llamafile/sandbox.c`; the patch fills a `llamafile_sandbox_spec` with the on-disk paths the loader opens after the lock (model, mmproj, media dir, LoRA, draft model, control vectors, public path as read; slot-save and prompt cache as read-write), detects outbound-needing features (`--rpc` via argv/env, MCP proxy, server tools) to relax `anet`→`inet`, passes `FLAG_confine_reads` for opt-in unveil(), binds the listening sockets first (`ctx_http.bind_socket()`, so a `--host /path.sock` Unix socket exists before `anet` forbids creating one, issue #971), quiesces the log worker (`common_log_pause`/`resume`, so no thread escapes the per-thread filter) around the call, before the HTTP listener spawns and before model load, and skips it in combined/GPU modes; adds Cosmopolitan-specific standalone `main()` with `cosmo_args`, verbose flag handling, `--unsecure` consumption (`llamafile_consume_flag`), and GPU pre-initialization; handles `LLAMAFILE_TUI` exit to avoid Metal cleanup crashes |+| `tools_server_server-http.h.patch` / `tools_server_server-http.cpp.patch` | Splits the bind out of `server_http_context::start()` into `bind_socket()`, which `start()` calls if it hasn't run yet, so the server can bind before installing the sandbox (issue #971). Since v0.8.0 `start()` binds every listener (one per `--host`) before spawning the HTTP thread pool, and `bind_socket()` does exactly that first part |**GPU decode must stay on the main thread** (`tools_server_server-queue.cpp.patch`).b10441 moved `llama_decode()` off the main thread: `update_slots()` now hands@@ -333,16 +335,15 @@ once upstream has the same change.| Patch | Description ||-------|-------------|-| `ggml_src_ggml-alloc.c.patch` | [PR #25584](https://github.com/ggml-org/llama.cpp/pull/25584), verbatim minus its test. `ggml_backend_alloc_ctx_tensors_from_buft()` splits a context across buffers when a tensor exceeds the backend's `max_size` (1 GiB on Vulkan); when the tail of the context holds only views, the final `alloc_tensor_range()` is skipped and those views never get `ggml_backend_view_init()`. The persistent KV stream views (`layer.k_stream`/`v_stream`) then keep `data == NULL`, and the server's state save/restore hits `GGML_ASSERT(tensor->data != NULL && "tensor not allocated")` in `ggml_backend_tensor_get()`. The fix allocates only parent tensors per split range and initializes every view in one final pass. **Symptom without it:** `llama-server` on Vulkan dies on the *second* request at the default `--parallel 4` (upstream [#29221](https://github.com/ggml-org/llama.cpp/issues/29221); `--parallel 1` is the workaround). Reproduced on an L40S with vanilla b11100, so it is not llamafile-specific; verified fixed on both vanilla and llamafile. Upstream also files it against #19839, #23737 and #21762. |+| `ggml_src_ggml-backend.cpp.patch` (part) | [PR #25584](https://github.com/ggml-org/llama.cpp/pull/25584), still open at v0.8.0. When a buffer type has a `max_size` (1 GiB on Vulkan), the allocation of a context's tensors is split across several buffers; when a tensor larger than `max_size` is followed only by views, the zero-sized tail is never planned as a buffer range and those views never get `ggml_backend_view_init()`. The persistent KV stream views (`layer.k_stream`/`v_stream`) then keep `data == NULL`, and the server's state save/restore hits `GGML_ASSERT(tensor->data != NULL && "tensor not allocated")` in `ggml_backend_tensor_get()`. The fix allocates only parent tensors per range and initializes every view in one final pass. Up to b11100 the splitting lived in `ggml-alloc.c` (where the PR applies, as `ggml_src_ggml-alloc.c.patch`); v0.8.0 (#23671) moved it, bug included, into `ggml_backend_buft_alloc_buffer_n_default()` in `ggml-backend.cpp`, so the same fix is now carried there. **Symptom without it:** `llama-server` on Vulkan dies on the *second* request at the default `--parallel 4` (upstream [#29221](https://github.com/ggml-org/llama.cpp/issues/29221); `--parallel 1` is the workaround). Reproduced on an L40S with vanilla b11100, so it is not llamafile-specific; verified fixed on both vanilla and llamafile at b11100. Upstream also files it against #19839, #23737 and #21762. |### Bug Fixes| Patch | Description ||-------|-------------|-| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach) |-| `ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch` | Fixes unsigned integer underflow in `ggml_backend_vk_get_device_memory` where Vulkan's `heapUsage` can exceed `heapBudget` (clamps to zero instead of wrapping) |+| `ggml_src_ggml-backend-reg.cpp.patch` | Suppresses debug log noise for non-existent backend search paths (irrelevant for llamafile's DSO loading approach). Since v0.8.0 (#28367) a missing path surfaces as a `directory_iterator` `ENOENT`, so only that error is silenced; other enumeration failures are still logged || `src_models_t5.cpp.patch` | Forward-declares the `graph<false>`/`graph<true>` explicit specializations before `build_arch_graph` so clang's `-std=gnu++23` doesn't reject them as specializations after implicit instantiation |-| `ggml_src_ggml.c.patch` | Makes the `ggml_time_ms()`/`ggml_time_us()` call `ggml_time_init()` lazily. The Windows dylibs link their own copy of `ggml.c`, so the DLL's `timer_freq` is never set by the executable's `ggml_init()` and any `ggml_time_*()` call made from inside the DLL divides by zero (`0xC0000094`, surfaced by Cosmopolitan as SIGFPE). First hit when #1051 enabled `-DGGML_CUDA_USE_GRAPHS`, calling `ggml_time_us()` in `ggml-cuda`. Linux/macOS are unaffected (`clock_gettime`, no static divisor). |+| `ggml_src_ggml.c.patch` | Makes the `ggml_time_ms()`/`ggml_time_us()` call `ggml_time_init()` lazily. The Windows dylibs link their own copy of `ggml.c`, so the DLL's `timer_freq` is never set by the executable's `ggml_init()` and any `ggml_time_*()` call made from inside the DLL divides by zero (`0xC0000094`, surfaced by Cosmopolitan as SIGFPE). First hit when #1051 enabled `-DGGML_CUDA_USE_GRAPHS`, calling `ggml_time_us()` in `ggml-cuda`. Linux/macOS are unaffected (`clock_gettime`, no static divisor). v0.8.0 (#30969) made the division floating-point, so it no longer traps, but `inf` cast to `int64_t` is undefined (INT64_MIN on x86) and every DLL-side timing would be garbage, so the lazy init is still needed. |The same clang `-std=gnu++23` problem used to need patches for `eagle3.cpp` and`dflash.cpp`; b11100 defines `build_arch_graph` at the end of both files, afterdiff --git a/llama.cpp.patches/fetch-ui-assets.sh b/llama.cpp.patches/fetch-ui-assets.shindex c81257e2..9c9dfea6 100755--- a/llama.cpp.patches/fetch-ui-assets.sh+++ b/llama.cpp.patches/fetch-ui-assets.sh@@ -125,7 +125,9 @@ echo "Fetching prebuilt web UI assets from Hugging Face..."# build number. "latest" is kept only as a last-resort fallback (enumeration# failed, e.g. offline / API change, or our commit predates every published# tag) so the build can still get *some* UI rather than none.-UI_CUR_BUILD="$(cd "$LLAMA_DIR" && git describe --tags --always 2>/dev/null \+# --match: upstream also tags releases as vX.Y.Z, often on the same commit as a+# bNNNN tag, and a plain describe then prefers it; we need the build number.+UI_CUR_BUILD="$(cd "$LLAMA_DIR" && git describe --tags --match 'b[0-9]*' --always 2>/dev/null \| grep -oE '^b[0-9]+' | grep -oE '[0-9]+' || true)"UI_CANDIDATES=()diff --git a/llama.cpp.patches/llamafile-files/BUILD.mk b/llama.cpp.patches/llamafile-files/BUILD.mkindex 9e670287..68cf7fe8 100644--- a/llama.cpp.patches/llamafile-files/BUILD.mk+++ b/llama.cpp.patches/llamafile-files/BUILD.mk@@ -34,7 +34,8 @@ GGML_SRCS_CPP := \llama.cpp/ggml/src/ggml-cpu/binary-ops.cpp \llama.cpp/ggml/src/ggml-cpu/ggml-cpu.cpp \llama.cpp/ggml/src/ggml-cpu/hbm.cpp \- llama.cpp/ggml/src/ggml-cpu/iqp.cpp \+ llama.cpp/ggml/src/ggml-cpu/tiled/tiled.cpp \+ llama.cpp/ggml/src/ggml-cpu/tiled/tiled-kernel.cpp \llama.cpp/ggml/src/ggml-cpu/ops.cpp \llama.cpp/ggml/src/ggml-cpu/repack.cpp \llama.cpp/ggml/src/ggml-cpu/traits.cpp \@@ -52,6 +53,7 @@ GGML_OBJS := \# ==============================================================================LLAMA_SRCS_CPP := \+ llama.cpp/src/llama-moe-cache.cpp \llama.cpp/src/llama.cpp \llama.cpp/src/models/afmoe.cpp \llama.cpp/src/models/apertus.cpp \@@ -62,11 +64,13 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/bailingmoe.cpp \llama.cpp/src/models/bailingmoe2.cpp \llama.cpp/src/models/bailingmoe3.cpp \+ llama.cpp/src/models/beam.cpp \llama.cpp/src/models/bert.cpp \llama.cpp/src/models/bitnet.cpp \llama.cpp/src/models/bloom.cpp \llama.cpp/src/models/chameleon.cpp \llama.cpp/src/models/chatglm.cpp \+ llama.cpp/src/models/clef.cpp \llama.cpp/src/models/clip.cpp \llama.cpp/src/models/codeshell.cpp \llama.cpp/src/models/cogvlm.cpp \@@ -80,6 +84,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/deepseek2ocr.cpp \llama.cpp/src/models/deepseek32.cpp \llama.cpp/src/models/deepseek4.cpp \+ llama.cpp/src/models/deepseek41.cpp \llama.cpp/src/models/delta-net-base.cpp \llama.cpp/src/models/dflash.cpp \llama.cpp/src/models/dots1.cpp \@@ -94,7 +99,9 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/exaone-moe.cpp \llama.cpp/src/models/falcon-h1.cpp \llama.cpp/src/models/falcon.cpp \+ llama.cpp/src/models/flint.cpp \llama.cpp/src/models/gemma-embedding.cpp \+ llama.cpp/src/models/gemma-embedding2.cpp \llama.cpp/src/models/gemma.cpp \llama.cpp/src/models/gemma2.cpp \llama.cpp/src/models/gemma3.cpp \@@ -104,6 +111,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/glm-dsa.cpp \llama.cpp/src/models/glm4-moe.cpp \llama.cpp/src/models/glm4.cpp \+ llama.cpp/src/models/glm5-next.cpp \llama.cpp/src/models/gpt2.cpp \llama.cpp/src/models/gptneox.cpp \llama.cpp/src/models/granite-hybrid.cpp \@@ -111,6 +119,11 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/granite-switch.cpp \llama.cpp/src/models/granite.cpp \llama.cpp/src/models/granite-swa.cpp \+ llama.cpp/src/models/k2-horizon.cpp \+ llama.cpp/src/models/kolibri1.cpp \+ llama.cpp/src/models/lfm-base.cpp \+ llama.cpp/src/models/lfm3.cpp \+ llama.cpp/src/models/maion-coder.cpp \llama.cpp/src/models/mamba-base.cpp \llama.cpp/src/models/grok.cpp \llama.cpp/src/models/grovemoe.cpp \@@ -152,6 +165,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/modern-bert.cpp \llama.cpp/src/models/mpt.cpp \llama.cpp/src/models/muse-glimmer.cpp \+ llama.cpp/src/models/muse-spark.cpp \llama.cpp/src/models/nanbeige.cpp \llama.cpp/src/models/nemotron-h-moe.cpp \llama.cpp/src/models/nemotron-h.cpp \@@ -196,6 +210,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/rwkv7-base.cpp \llama.cpp/src/models/rwkv7.cpp \llama.cpp/src/models/seed-oss.cpp \+ llama.cpp/src/models/sensenova-u1.cpp \llama.cpp/src/models/smallthinker.cpp \llama.cpp/src/models/smollm3.cpp \llama.cpp/src/models/spark2-5.cpp \@@ -206,6 +221,7 @@ LLAMA_SRCS_CPP := \llama.cpp/src/models/t5.cpp \llama.cpp/src/models/t5encoder.cpp \llama.cpp/src/models/talkie.cpp \+ llama.cpp/src/models/vllm-decision2.cpp \llama.cpp/src/models/wavtokenizer-dec.cpp \llama.cpp/src/models/xverse.cpp \llama.cpp/src/llama-adapter.cpp \@@ -257,6 +273,7 @@ COMMON_SRCS_CPP := \llama.cpp/common/console.cpp \llama.cpp/common/debug.cpp \llama.cpp/common/download.cpp \+ llama.cpp/common/file-lock.cpp \llama.cpp/common/fit.cpp \llama.cpp/common/hf-cache.cpp \llama.cpp/common/imatrix-loader.cpp \@@ -275,6 +292,10 @@ COMMON_SRCS_CPP := \llama.cpp/common/ngram-cache.cpp \llama.cpp/common/ngram-map.cpp \llama.cpp/common/ngram-mod.cpp \+ llama.cpp/common/parsers/harmony-xml.cpp \+ llama.cpp/common/parsers/k2-horizon.cpp \+ llama.cpp/common/parsers/llm-jp-harmony.cpp \+ llama.cpp/common/parsers/muse-spark.cpp \llama.cpp/common/parsers/parsers.cpp \llama.cpp/common/parsers/cohere2moe.cpp \llama.cpp/common/parsers/deepseek.cpp \@@ -291,6 +312,7 @@ COMMON_SRCS_CPP := \llama.cpp/common/parsers/ministral3.cpp \llama.cpp/common/parsers/muse-glimmer.cpp \llama.cpp/common/parsers/qwen3-coder.cpp \+ llama.cpp/common/parsers/translate-gemma.cpp \llama.cpp/common/peg-parser.cpp \llama.cpp/common/preset.cpp \llama.cpp/common/reasoning-budget.cpp \@@ -371,6 +393,10 @@ MTMD_SRCS_CPP := \llama.cpp/tools/mtmd/models/internvl.cpp \llama.cpp/tools/mtmd/models/kimik25.cpp \llama.cpp/tools/mtmd/models/kimivl.cpp \+ llama.cpp/tools/mtmd/models/lfm3a-gen.cpp \+ llama.cpp/tools/mtmd/models/lfm3a.cpp \+ llama.cpp/tools/mtmd/models/lfm3vl.cpp \+ llama.cpp/tools/mtmd/models/ling3vl.cpp \llama.cpp/tools/mtmd/models/llama4.cpp \llama.cpp/tools/mtmd/models/llava.cpp \llama.cpp/tools/mtmd/models/mimo-audio.cpp \@@ -379,6 +405,8 @@ MTMD_SRCS_CPP := \llama.cpp/tools/mtmd/models/minimax-m3.cpp \llama.cpp/tools/mtmd/models/mobilenetv5.cpp \llama.cpp/tools/mtmd/models/muse-glimmer.cpp \+ llama.cpp/tools/mtmd/models/muse-spark-audio.cpp \+ llama.cpp/tools/mtmd/models/nemotron-omni.cpp \llama.cpp/tools/mtmd/models/nemotron-v2-vl.cpp \llama.cpp/tools/mtmd/models/paddleocr.cpp \llama.cpp/tools/mtmd/models/parakeet.cpp \@@ -391,8 +419,11 @@ MTMD_SRCS_CPP := \llama.cpp/tools/mtmd/models/qwen3tts-gen.cpp \llama.cpp/tools/mtmd/models/qwen3tts-spkenc.cpp \llama.cpp/tools/mtmd/models/qwen3vl.cpp \+ llama.cpp/tools/mtmd/models/sensenova-u1.cpp \llama.cpp/tools/mtmd/models/siglip.cpp \+ llama.cpp/tools/mtmd/models/soprano.cpp \llama.cpp/tools/mtmd/models/step3vl.cpp \+ llama.cpp/tools/mtmd/models/voxtral-rt.cpp \llama.cpp/tools/mtmd/models/whisper-enc.cpp \llama.cpp/tools/mtmd/models/yasa2.cpp \llama.cpp/tools/mtmd/models/youtuvl.cpp@@ -497,6 +528,10 @@ TOOL_BENCH_SRCS := \llama.cpp/tools/llama-bench/main.cppTOOL_SERVER_SRCS := \+ llama.cpp/tools/server/server-connect.cpp \+ llama.cpp/tools/server/server-decision.cpp \+ llama.cpp/tools/server/server-models-sched.cpp \+ llama.cpp/tools/server/server-realtime.cpp \llama.cpp/tools/server/server.cpp \llama.cpp/tools/server/server-chat.cpp \llama.cpp/tools/server/server-common.cpp \diff --git a/llama.cpp.patches/patches/common_common.cpp.patch b/llama.cpp.patches/patches/common_common.cpp.patchindex 1b0565e0..b775f70a 100644--- a/llama.cpp.patches/patches/common_common.cpp.patch+++ b/llama.cpp.patches/patches/common_common.cpp.patch@@ -1,7 +1,7 @@diff --git a/common/common.cpp b/common/common.cpp--- a/llama.cpp/common/common.cpp+++ b/llama.cpp/common/common.cpp-@@ -34,6 +34,10 @@+@@ -39,6 +39,10 @@#include <sys/sysctl.h>#endif@@ -12,7 +12,7 @@ diff --git a/common/common.cpp b/common/common.cpp#if defined(_WIN32)#define WIN32_LEAN_AND_MEAN#ifndef NOMINMAX-@@ -76,7 +80,38 @@ common_time_meas::~common_time_meas() {+@@ -80,7 +84,38 @@ common_time_meas::~common_time_meas() {//int32_t common_cpu_get_num_physical_cores() {@@ -52,81 +52,37 @@ diff --git a/common/common.cpp b/common/common.cppint32_t logical_cpus = _system_configuration.ncpus;int32_t smt_threads = _system_configuration.smt_threads;if (smt_threads > 0) {-@@ -1059,6 +1094,20 @@ std::string fs_get_cache_directory() {- if (cache_directory.empty()) {- throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");- }+@@ -1007,6 +1042,16 @@ std::filesystem::path fs_get_cache_directory() {+ if (cache_directory.empty()) {+ throw std::runtime_error("Failed to find %LOCALAPPDATA% directory");+ }+#elif defined(COSMOCC)-+ // We don't know what OS we are running on at compile time, just CPU-+ // architecture. Try various environment variables, fall back to ~/.cache.-+ cache_directory = common_get_env("LOCALAPPDATA");-+ if (cache_directory.empty()) {-+ cache_directory = common_get_env("XDG_CACHE_HOME");-+ }-+ if (cache_directory.empty()) {-+ const std::string home = common_get_env("HOME");-+ if (home.empty()) {-+ throw std::runtime_error("Failed to find $HOME directory");-+ }-+ cache_directory = home + "/.cache/";-+ }- #elif defined(__EMSCRIPTEN__)- GGML_ABORT("not implemented on this platform");++ // We don't know what OS we are running on at compile time, just CPU++ // architecture. Try various environment variables, fall back to ~/.cache.++ cache_directory = common_get_path_from_env("LOCALAPPDATA");++ if (cache_directory.empty()) {++ cache_directory = common_get_path_from_env("XDG_CACHE_HOME");++ }++ if (cache_directory.empty()) {++ cache_directory = get_home_directory() / ".cache";++ }+ #elif defined(__APPLE__)+ cache_directory = get_home_directory() / "Library/Caches";#else-@@ -1104,6 +1153,20 @@ std::string fs_get_config_directory() {+@@ -1025,6 +1070,16 @@ std::filesystem::path fs_get_config_directory() {if (config_directory.empty()) {throw std::runtime_error("Failed to find %APPDATA% directory");}+#elif defined(COSMOCC)+ // We don't know what OS we are running on at compile time, just CPU+ // architecture. Try various environment variables, fall back to ~/.config.-+ config_directory = common_get_env("APPDATA");++ config_directory = common_get_path_from_env("APPDATA");+ if (config_directory.empty()) {-+ config_directory = common_get_env("XDG_CONFIG_HOME");++ config_directory = common_get_path_from_env("XDG_CONFIG_HOME");+ }+ if (config_directory.empty()) {-+ const std::string home = common_get_env("HOME");-+ if (home.empty()) {-+ throw std::runtime_error("Failed to find $HOME directory");-+ }-+ config_directory = home + "/.config/";++ config_directory = get_home_directory() / ".config";+ }- #elif defined(__EMSCRIPTEN__)- // caller decides what to do when there is no config directory- throw std::runtime_error("not implemented on this platform");-@@ -1296,6 +1359,26 @@ common_init_result::common_init_result(common_params & params, bool model_only)- COM_TRC("%s", "fitting params to device memory ...\n");- COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n");--+ // if a multimodal projection model is specified and will be loaded on GPU,-+ // add its estimated size to the fit margins so the fit leaves enough room for it-+ std::vector<size_t> fit_params_target = params.fit_params_target;-+ if (!params.mmproj.path.empty() && params.mmproj_use_gpu) {-+ struct gguf_init_params gparams = { /*.no_alloc =*/ true, /*.ctx =*/ nullptr };-+ struct gguf_context * gctx = gguf_init_from_file(params.mmproj.path.c_str(), gparams);-+ if (gctx != nullptr) {-+ size_t mmproj_size = 0;-+ for (int64_t i = 0; i < gguf_get_n_tensors(gctx); i++) {-+ mmproj_size += gguf_get_tensor_size(gctx, i);-+ }-+ gguf_free(gctx);-+ LOG_INF("%s: reserving %.2f MiB extra for mmproj model '%s'\n", __func__,-+ mmproj_size / (1024.0 * 1024.0), params.mmproj.path.c_str());-+ for (size_t & target : fit_params_target) {-+ target += mmproj_size;-+ }-+ }-+ }-+- // the draft context is created from the same base params and follows the main context, fit both together- const bool has_draft = params.speculative.has_dft();- const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(),-@@ -1320,7 +1403,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)- common_fit_params(params.model.path.c_str(), &mparams, &cparams,- params.tensor_split,- params.tensor_buft_overrides.data(),-- params.fit_params_target.data(),-+ fit_params_target.data(),- params.fit_params_min_ctx,- has_draft || spec_mtp ? &extra : nullptr,- params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR);+ #else+ config_directory = common_get_path_from_env("XDG_CONFIG_HOME");+ if (config_directory.empty()) {diff --git a/llama.cpp.patches/patches/common_common.h.patch b/llama.cpp.patches/patches/common_common.h.patchnew file mode 100644index 00000000..739457a6--- /dev/null+++ b/llama.cpp.patches/patches/common_common.h.patch@@ -0,0 +1,12 @@+diff --git a/common/common.h b/common/common.h+--- a/llama.cpp/common/common.h++++ b/llama.cpp/common/common.h+@@ -629,7 +629,7 @@ struct common_params {+ std::string cls_sep = "\t"; // separator of classification sequences++ // server params+- int32_t port = 9931; // server listens on this network port++ int32_t port = 8080; // server listens on this network port (llamafile: keeps its documented 8080, upstream moved to 9931 in #30159)+ bool reuse_port = false; // allow multiple sockets to bind to the same port+ int32_t timeout_read = 3600; // http read timeout in seconds+ int32_t timeout_write = timeout_read; // http write timeout in secondsdiff --git a/llama.cpp.patches/patches/common_download.cpp.patch b/llama.cpp.patches/patches/common_download.cpp.patchindex a731fcb1..ec9a78e8 100644--- a/llama.cpp.patches/patches/common_download.cpp.patch+++ b/llama.cpp.patches/patches/common_download.cpp.patch@@ -1,7 +1,7 @@diff --git a/common/download.cpp b/common/download.cpp--- a/llama.cpp/common/download.cpp+++ b/llama.cpp/common/download.cpp-@@ -24,6 +24,8 @@+@@ -25,6 +25,8 @@#ifndef __EMSCRIPTEN__#ifdef __linux__#include <linux/limits.h>diff --git a/llama.cpp.patches/patches/common_http.h.patch b/llama.cpp.patches/patches/common_http.h.patchindex a598abda..3f628d16 100644--- a/llama.cpp.patches/patches/common_http.h.patch+++ b/llama.cpp.patches/patches/common_http.h.patch@@ -1,7 +1,7 @@diff --git a/common/http.h b/common/http.h--- a/llama.cpp/common/http.h+++ b/llama.cpp/common/http.h-@@ -40,10 +40,15 @@ static common_http_url common_http_parse_url(const std::string & url) {+@@ -42,10 +42,15 @@ static common_http_url common_http_parse_url(const std::string & url) {}auto rest = url.substr(scheme_end + 3);@@ -19,7 +19,7 @@ diff --git a/common/http.h b/common/http.hauto colon_pos = auth.find(':');if (colon_pos != std::string::npos) {parts.user = auth.substr(0, colon_pos);-@@ -51,18 +56,14 @@ static common_http_url common_http_parse_url(const std::string & url) {+@@ -53,18 +58,14 @@ static common_http_url common_http_parse_url(const std::string & url) {} else {parts.user = auth;}@@ -43,7 +43,7 @@ diff --git a/common/http.h b/common/http.h// split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986)std::string port_str;-@@ -74,6 +75,8 @@ static common_http_url common_http_parse_url(const std::string & url) {+@@ -76,6 +77,8 @@ static common_http_url common_http_parse_url(const std::string & url) {auto after = parts.host.substr(close + 1);if (!after.empty() && after.front() == ':') {port_str = after.substr(1);@@ -52,7 +52,7 @@ diff --git a/common/http.h b/common/http.h}parts.host = parts.host.substr(1, close - 1);} else {-@@ -85,7 +88,14 @@ static common_http_url common_http_parse_url(const std::string & url) {+@@ -87,7 +90,14 @@ static common_http_url common_http_parse_url(const std::string & url) {}if (!port_str.empty()) {@@ -67,7 +67,7 @@ diff --git a/common/http.h b/common/http.h} else if (parts.scheme == "http") {parts.port = 80;} else if (parts.scheme == "https") {-@@ -104,7 +114,7 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:+@@ -218,7 +228,7 @@ static std::pair<httplib::Client, common_http_url> common_http_client(const std:throw std::runtime_error("error: invalid URL format");}diff --git a/llama.cpp.patches/patches/common_log.cpp.patch b/llama.cpp.patches/patches/common_log.cpp.patchindex e80ea789..99ebfa41 100644--- a/llama.cpp.patches/patches/common_log.cpp.patch+++ b/llama.cpp.patches/patches/common_log.cpp.patch@@ -1,15 +1,18 @@diff --git a/common/log.cpp b/common/log.cpp--- a/llama.cpp/common/log.cpp+++ b/llama.cpp/common/log.cpp-@@ -25,6 +25,7 @@- # define fileno _fileno- #else- # include <unistd.h>-+# include <csignal>- #endif // defined(_WIN32)+@@ -14,6 +14,10 @@+ #include <vector>+ #include <algorithm>++#if !defined(_WIN32)++# include <csignal>++#endif++int common_log_verbosity_thold = LOG_DEFAULT_LLAMA;-@@ -243,7 +244,13 @@ public:++ int common_log_get_verbosity_thold(void) {+@@ -240,7 +244,13 @@ public:std::unique_lock<std::mutex> lock(mtx);// block if the queue is full@@ -24,7 +27,7 @@ diff --git a/common/log.cpp b/common/log.cppif (!running) {// discard messages while the worker thread is paused-@@ -310,7 +317,13 @@ public:+@@ -307,7 +317,13 @@ public:std::unique_lock<std::mutex> lock(mtx);// block if the queue is full@@ -39,7 +42,7 @@ diff --git a/common/log.cpp b/common/log.cppif (!running) {// discard messages while the worker thread is paused-@@ -345,9 +358,32 @@ public:+@@ -342,9 +358,32 @@ public:running = true;thrd = std::thread([this]() {diff --git a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patchindex 013b0f23..5271f6ca 100644--- a/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch+++ b/llama.cpp.patches/patches/ggml_include_ggml-backend.h.patch@@ -18,7 +18,7 @@ diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h#ifdef __cplusplusextern "C" {#endif-@@ -210,19 +220,19 @@ extern "C" {+@@ -212,19 +222,19 @@ extern "C" {typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors);// Split buffer type for tensor parallelism (old)diff --git a/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patch b/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patchdeleted file mode 100644index 05ca1a25..00000000--- a/llama.cpp.patches/patches/ggml_src_ggml-alloc.c.patch+++ /dev/null@@ -1,67 +0,0 @@-diff --git a/ggml/src/ggml-alloc.c b/ggml/src/ggml-alloc.c---- a/llama.cpp/ggml/src/ggml-alloc.c-+++ b/llama.cpp/ggml/src/ggml-alloc.c-@@ -1124,6 +1124,20 @@ static void free_buffers(ggml_backend_buffer_t ** buffers, const size_t * n_buff- free(*buffers);- }--+static bool alloc_ctx_tensors_finalize_views(struct ggml_context * ctx) {-+ for (struct ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) {-+ if (t->view_src != NULL && t->buffer == NULL) {-+ enum ggml_status status = ggml_backend_view_init(t);-+ if (status != GGML_STATUS_SUCCESS) {-+ GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);-+ return false;-+ }-+ }-+ }-+-+ return true;-+}-+- static bool alloc_tensor_range(struct ggml_context * ctx,- struct ggml_tensor * first, struct ggml_tensor * last,- ggml_backend_buffer_type_t buft, size_t size,-@@ -1142,24 +1156,14 @@ static bool alloc_tensor_range(struct ggml_context * ctx,- struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);-- for (struct ggml_tensor * t = first; t != last; t = ggml_get_next_tensor(ctx, t)) {-- enum ggml_status status = GGML_STATUS_SUCCESS;-- if (t->data == NULL) {-- if (t->view_src == NULL) {-- status = ggml_tallocr_alloc(&tallocr, t);-- } else if (t->buffer == NULL) {-- status = ggml_backend_view_init(t);-- }-- } else {-- if (t->view_src != NULL && t->buffer == NULL) {-- // view of a pre-allocated tensor-- status = ggml_backend_view_init(t);-+ if (t->view_src == NULL && t->data == NULL) {-+ enum ggml_status status = ggml_tallocr_alloc(&tallocr, t);-+ if (status != GGML_STATUS_SUCCESS) {-+ GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t->name);-+ free_buffers(buffers, n_buffers);-+ return false;- }- }-- if (status != GGML_STATUS_SUCCESS) {-- GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t->name);-- free_buffers(buffers, n_buffers);-- return false;-- }- }-- return true;-@@ -1217,6 +1221,11 @@ static ggml_backend_buffer_t ggml_backend_alloc_ctx_tensors_from_buft_impl(- return NULL;- }--+ if (!alloc_ctx_tensors_finalize_views(ctx)) {-+ free_buffers(&buffers, &n_buffers);-+ return NULL;-+ }-+- ggml_backend_buffer_t buffer;- if (n_buffers == 1) {- buffer = buffers[0];diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patchindex 138340bf..4210fa5a 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-impl.h.patch@@ -1,31 +1,37 @@diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h--- a/llama.cpp/ggml/src/ggml-backend-impl.h+++ b/llama.cpp/ggml/src/ggml-backend-impl.h-@@ -15,17 +15,17 @@ extern "C" {+@@ -15,21 +15,21 @@ extern "C" {//struct ggml_backend_buffer_type_i {-- const char * (*get_name) (ggml_backend_buffer_type_t buft);-+ const char * (GGML_CALL *get_name) (ggml_backend_buffer_type_t buft);+- const char * (*get_name) (ggml_backend_buffer_type_t buft);++ const char * (GGML_CALL *get_name) (ggml_backend_buffer_type_t buft);// allocate a buffer of this type-- ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);-+ ggml_backend_buffer_t (GGML_CALL *alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);+- ggml_backend_buffer_t (*alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);++ ggml_backend_buffer_t (GGML_CALL *alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);+ // (optional) allocate tensors from a list into a buffer of this type (defaults to alloc_buffer + linear allocator)+- ggml_backend_buffer_t (*alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);++ ggml_backend_buffer_t (GGML_CALL *alloc_buffer_n) (ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);// tensor alignment-- size_t (*get_alignment) (ggml_backend_buffer_type_t buft);-+ size_t (GGML_CALL *get_alignment) (ggml_backend_buffer_type_t buft);+- size_t (*get_alignment) (ggml_backend_buffer_type_t buft);++ size_t (GGML_CALL *get_alignment) (ggml_backend_buffer_type_t buft);// (optional) max buffer size that can be allocated (defaults to SIZE_MAX)-- size_t (*get_max_size) (ggml_backend_buffer_type_t buft);-+ size_t (GGML_CALL *get_max_size) (ggml_backend_buffer_type_t buft);+- size_t (*get_max_size) (ggml_backend_buffer_type_t buft);++ size_t (GGML_CALL *get_max_size) (ggml_backend_buffer_type_t buft);// (optional) data size needed to allocate the tensor, including padding (defaults to ggml_nbytes)-- size_t (*get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);-+ size_t (GGML_CALL *get_alloc_size)(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);+- size_t (*get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);++ size_t (GGML_CALL *get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);+ // (optional) total data size needed to allocate the given tensors, including padding and splitting (defaults to per-tensor get_alloc_size)+- size_t (*get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);++ size_t (GGML_CALL *get_alloc_size_n)(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors);// (optional) check if tensor data is in host memory and uses standard ggml tensor layout (defaults to false)-- bool (*is_host) (ggml_backend_buffer_type_t buft);-+ bool (GGML_CALL *is_host) (ggml_backend_buffer_type_t buft);+- bool (*is_host) (ggml_backend_buffer_type_t buft);++ bool (GGML_CALL *is_host) (ggml_backend_buffer_type_t buft);};struct ggml_backend_buffer_type {-@@ -44,26 +44,30 @@ extern "C" {+@@ -48,26 +48,30 @@ extern "C" {//struct ggml_backend_buffer_i {@@ -68,7 +74,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend_buffer {-@@ -114,45 +118,45 @@ extern "C" {+@@ -115,45 +119,45 @@ extern "C" {struct ggml_backend_graph_optimize_params {// keep `tensor` allocated at least until `until` (a node of the same graph) has been computed// can be called multiple times for the same tensor: the longest lifetime applies@@ -131,7 +137,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend {-@@ -175,46 +179,46 @@ extern "C" {+@@ -176,46 +180,46 @@ extern "C" {// the current functions to obtain the properties can remain, since they are more convenient for often used propertiesstruct ggml_backend_device_i {// device name: short identifier for this device, such as "CPU" or "CUDA0"@@ -193,7 +199,7 @@ diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h};struct ggml_backend_device {-@@ -228,15 +232,15 @@ extern "C" {+@@ -229,15 +233,15 @@ extern "C" {//struct ggml_backend_reg_i {diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patchindex df78814f..5e53e54e 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-meta.cpp.patch@@ -75,19 +75,22 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_ASSERT(ggml_backend_buft_is_meta(buft));const ggml_backend_meta_buffer_type_context * meta_buft_ctx = (const ggml_backend_meta_buffer_type_context *) buft->context;return meta_buft_ctx->name.c_str();-@@ -288,9 +288,9 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backen+@@ -288,11 +288,11 @@ static ggml_backend_buffer_type_t ggml_backend_meta_buft_simple_buft(ggml_backenreturn meta_buft_ctx->simple_bufts[index];}-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);+static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size);+-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors);++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors);+-static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {+static size_t GGML_CALL ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);size_t max_alignment = 1;for (size_t i = 0; i < n_simple_bufts; i++) {-@@ -301,7 +301,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_ty+@@ -303,7 +303,7 @@ static size_t ggml_backend_meta_buffer_type_get_alignment(ggml_backend_buffer_tyreturn max_alignment;}@@ -96,7 +99,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);size_t max_size = SIZE_MAX;for (size_t i = 0; i < n_simple_bufts; i++) {-@@ -310,7 +310,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typ+@@ -312,7 +312,7 @@ static size_t ggml_backend_meta_buffer_type_get_max_size(ggml_backend_buffer_typreturn max_size;}@@ -105,7 +108,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);size_t max_alloc_size = 0;for (size_t i = 0; i < n_simple_bufts; i++) {-@@ -320,7 +320,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_t+@@ -322,7 +322,7 @@ static size_t ggml_backend_meta_buffer_type_get_alloc_size(ggml_backend_buffer_treturn max_alloc_size;}@@ -114,7 +117,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);for (size_t i = 0; i < n_simple_bufts; i++) {if (!ggml_backend_buft_is_host(ggml_backend_meta_buft_simple_buft(buft, i))) {-@@ -343,7 +343,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {+@@ -347,7 +347,7 @@ bool ggml_backend_buft_is_meta(ggml_backend_buffer_type_t buft) {return buft != nullptr && buft->iface.get_name == ggml_backend_meta_buffer_type_iface.get_name;}@@ -123,7 +126,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppstatic std::map<ggml_backend_dev_t, struct ggml_backend_buffer_type> meta_bufts;GGML_ASSERT(ggml_backend_dev_is_meta(dev));{-@@ -370,7 +370,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_+@@ -374,7 +374,7 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_buffer_type(ggml_return &result.first->second;}@@ -132,7 +135,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_ASSERT(ggml_backend_dev_is_meta(dev));const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;-@@ -453,7 +453,7 @@ struct ggml_backend_meta_buffer_context {+@@ -457,7 +457,7 @@ struct ggml_backend_meta_buffer_context {}};@@ -141,7 +144,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_ASSERT(ggml_backend_buffer_is_meta(buffer));ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;delete buf_ctx;-@@ -1172,7 +1172,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(co+@@ -1197,7 +1197,7 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(coreturn ggml_backend_meta_get_split_state(buf_ctx->get_simple_tensor_container(tensor), tensor, assume_sync);}@@ -150,7 +153,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_UNUSED(buffer);return (void *) 0x1000000000000000; // FIXME}-@@ -1304,14 +1304,14 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_m+@@ -1335,14 +1335,14 @@ static enum ggml_status ggml_backend_meta_buffer_init_tensor_impl(ggml_backend_mreturn GGML_STATUS_SUCCESS;}@@ -167,7 +170,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) {const size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);const ggml_backend_meta_split_state split_state =-@@ -1413,7 +1413,7 @@ static void ggml_backend_meta_buffer_memset_tensor(+@@ -1444,7 +1444,7 @@ static void ggml_backend_meta_buffer_memset_tensor(}}@@ -176,7 +179,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);-@@ -1541,7 +1541,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg+@@ -1572,7 +1572,7 @@ static void ggml_backend_meta_buffer_set_tensor(ggml_backend_buffer_t buffer, gg}}@@ -185,7 +188,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_bufs = ggml_backend_meta_buffer_n_bufs(buffer);const ggml_backend_meta_split_state split_state = ggml_backend_meta_get_split_state(tensor, /*assume_sync =*/ false);GGML_ASSERT(ggml_is_contiguous(tensor) || split_state.axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED);-@@ -1641,14 +1641,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co+@@ -1672,14 +1672,14 @@ static void ggml_backend_meta_buffer_get_tensor(ggml_backend_buffer_t buffer, co}}@@ -202,7 +205,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_ASSERT(ggml_backend_buffer_is_meta(buffer));ggml_backend_meta_buffer_context * buf_ctx = (ggml_backend_meta_buffer_context *) buffer->context;for (size_t i = 0; i < buf_ctx->bufs.size(); i++) {-@@ -1684,7 +1684,7 @@ void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_+@@ -1715,7 +1715,7 @@ void ggml_backend_meta_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_}}@@ -211,7 +214,16 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);const ggml_init_params params = {-@@ -1851,20 +1851,20 @@ struct ggml_backend_meta_context {+@@ -1740,7 +1740,7 @@ static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer(ggml_bac+ return ggml_backend_buffer_init(buft, ggml_backend_meta_buffer_iface, buf_ctx, max_size);+ }++-static ggml_backend_buffer_t ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors) {++static ggml_backend_buffer_t GGML_CALL ggml_backend_meta_buffer_type_alloc_buffer_n(ggml_backend_buffer_type_t buft, ggml_tensor ** tensors, int n_tensors) {+ const size_t n_simple_bufts = ggml_backend_meta_buft_n_bufts(buft);++ constexpr size_t compute_headroom = 16; // Maximum number of views per statically allocated tensor that can be created between evals.+@@ -1883,20 +1883,20 @@ struct ggml_backend_meta_context {}};@@ -235,7 +247,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_backends = ggml_backend_meta_n_backends(backend);GGML_ASSERT(offset == 0);GGML_ASSERT(ggml_is_contiguous(tensor));-@@ -1909,7 +1909,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens+@@ -1941,7 +1941,7 @@ static void ggml_backend_meta_set_tensor_async(ggml_backend_t backend, ggml_tens}}@@ -244,7 +256,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppconst size_t n_backends = ggml_backend_meta_n_backends(backend);GGML_ASSERT(offset == 0);GGML_ASSERT(ggml_is_contiguous(tensor));-@@ -1954,14 +1954,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm+@@ -1986,14 +1986,14 @@ static void ggml_backend_meta_get_tensor_async(ggml_backend_t backend, const ggm}}@@ -261,7 +273,7 @@ diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppGGML_ASSERT(cgraph->grads == nullptr);const size_t n_backends = ggml_backend_meta_n_backends(backend);ggml_backend_meta_context * backend_ctx = (ggml_backend_meta_context *) backend->context;-@@ -2487,7 +2487,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {+@@ -2523,7 +2523,7 @@ bool ggml_backend_is_meta(ggml_backend_t backend) {return backend != nullptr && backend->iface.get_name == ggml_backend_meta_i.get_name;}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patchindex 63af2d06..468fbf80 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend-reg.cpp.patch@@ -1,12 +1,13 @@diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp--- a/llama.cpp/ggml/src/ggml-backend-reg.cpp+++ b/llama.cpp/ggml/src/ggml-backend-reg.cpp-@@ -510,7 +510,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,- if (ec) {- GGML_LOG_DEBUG("%s: posix_stat(%s) failure, error-message: %s\n", __func__, path_str(search_path).c_str(), ec.message().c_str());- } else {-- GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());-+ // GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());+@@ -540,7 +540,8 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,}- continue;}++- if (dir_ec) {++ // llamafile: a search path that does not exist is expected (backends are loaded as DSOs elsewhere); don't log it++ if (dir_ec && dir_ec != std::errc::no_such_file_or_directory) {+ GGML_LOG_DEBUG("%s: failed to enumerate %s: %s\n", __func__, path_str(search_path).c_str(), dir_ec.message().c_str());+ }+ }diff --git a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patchindex 4337f9fa..3f68a78c 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-backend.cpp.patch@@ -1,7 +1,71 @@diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp--- a/llama.cpp/ggml/src/ggml-backend.cpp+++ b/llama.cpp/ggml/src/ggml-backend.cpp-@@ -121,7 +121,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {+@@ -108,34 +108,22 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe++ struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);++- // allocate tensors in the current buffer++ // allocate the parent tensors in the current buffer; views are initialized++ // in a single pass below, once every buffer exists (llamafile: carries++ // upstream PR #25584 — a range made only of views is never planned when it++ // follows a tensor larger than max_size, so its views kept data == NULL)+ struct ggml_tensor * t_failed = NULL;+ for (int j = item.first; j < item.last; j++) {+ struct ggml_tensor * t = tensors[j];+- if (t->data == NULL) {+- if (t->view_src == NULL) {+- if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {+- t_failed = t;+- break;+- }+- } else if (t->buffer == NULL) {+- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {+- t_failed = t;+- break;+- }+- }+- } else {+- if (t->view_src != NULL && t->buffer == NULL) {+- // view of a pre-allocated tensor+- if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {+- t_failed = t;+- break;+- }++ if (t->view_src == NULL && t->data == NULL) {++ if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {++ t_failed = t;++ break;+ }+ }+ }+ if (t_failed != NULL) {+- GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);++ GGML_LOG_ERROR("%s: failed to allocate tensor %s\n", __func__, t_failed->name);+ for (ggml_backend_buffer_t b : buffers) {+ ggml_backend_buffer_free(b);+ }+@@ -150,6 +138,19 @@ static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backe+ return NULL;+ }+++ for (int j = 0; j < n_tensors; j++) {++ struct ggml_tensor * t = tensors[j];++ if (t->view_src != NULL && t->buffer == NULL) {++ if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {++ GGML_LOG_ERROR("%s: failed to initialize view tensor %s\n", __func__, t->name);++ for (ggml_backend_buffer_t b : buffers) {++ ggml_backend_buffer_free(b);++ }++ return NULL;++ }++ }++ }+++ if (buffers.size() == 1) {+ return buffers[0];+ }+@@ -261,7 +262,14 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {if (buffer->iface.free_buffer != NULL) {buffer->iface.free_buffer(buffer);}@@ -17,7 +81,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp}size_t ggml_backend_buffer_get_size(ggml_backend_buffer_t buffer) {-@@ -682,7 +689,7 @@ struct ggml_backend_multi_buffer_context {+@@ -822,7 +830,7 @@ struct ggml_backend_multi_buffer_context {size_t n_buffers;};@@ -26,7 +90,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;for (size_t i = 0; i < ctx->n_buffers; i++) {-@@ -693,7 +700,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)+@@ -833,7 +841,7 @@ static void ggml_backend_multi_buffer_free_buffer(ggml_backend_buffer_t buffer)free(ctx);}@@ -35,7 +99,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);ggml_backend_multi_buffer_context * ctx = (ggml_backend_multi_buffer_context *) buffer->context;for (size_t i = 0; i < ctx->n_buffers; i++) {-@@ -713,6 +720,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {+@@ -853,6 +861,7 @@ static const struct ggml_backend_buffer_i ggml_backend_multi_buffer_i = {/* .cpy_tensor = */ NULL,/* .clear = */ ggml_backend_multi_buffer_clear,/* .reset = */ NULL,@@ -43,7 +107,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp};ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers) {-@@ -1063,6 +1071,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru+@@ -1211,6 +1220,18 @@ static void ggml_backend_sched_set_if_supported(ggml_backend_sched_t sched, stru}// assigns backends to ops and splits the graph into subgraphs that can be computed on the same backend@@ -62,7 +126,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppvoid ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph) {// reset splitssched->n_splits = 0;-@@ -1445,13 +1465,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra+@@ -1609,13 +1630,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgrastd::unordered_map<ggml_tensor *, std::vector<ggml_tensor *>> alloc_deps;struct ggml_backend_graph_optimize_params opt_params = {@@ -77,7 +141,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp/* .user_data = */ &alloc_deps,};-@@ -2352,7 +2366,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t+@@ -2466,7 +2481,7 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t// CPU backend - buffer@@ -86,7 +150,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);uintptr_t data = (uintptr_t)buffer->context;-@@ -2364,33 +2378,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {+@@ -2478,33 +2493,33 @@ static void * ggml_backend_cpu_buffer_get_base(ggml_backend_buffer_t buffer) {return (void *)data;}@@ -125,7 +189,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(src);if (ggml_backend_buffer_is_host(src->buffer)) {memcpy(dst->data, src->data, ggml_nbytes(src));-@@ -2401,7 +2415,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con+@@ -2515,7 +2530,7 @@ static bool ggml_backend_cpu_buffer_cpy_tensor(ggml_backend_buffer_t buffer, conGGML_UNUSED(buffer);}@@ -134,7 +198,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppGGML_ASSERT(buffer);memset(buffer->context, value, buffer->size);}-@@ -2418,6 +2432,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {+@@ -2532,6 +2547,7 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_i = {/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,/* .reset = */ NULL,@@ -142,7 +206,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp};static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {-@@ -2432,19 +2447,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {+@@ -2546,19 +2562,20 @@ static const struct ggml_backend_buffer_i ggml_backend_cpu_buffer_from_ptr_i = {/* .cpy_tensor = */ ggml_backend_cpu_buffer_cpy_tensor,/* .clear = */ ggml_backend_cpu_buffer_clear,/* .reset = */ NULL,@@ -165,7 +229,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppvoid * data = ggml_aligned_malloc(size);if (data == NULL) {-@@ -2455,13 +2471,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_back+@@ -2569,13 +2586,13 @@ static ggml_backend_buffer_t ggml_backend_cpu_buffer_type_alloc_buffer(ggml_backreturn ggml_backend_buffer_init(buft, ggml_backend_cpu_buffer_i, data, size);}@@ -181,7 +245,7 @@ diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cppreturn true;GGML_UNUSED(buft);-@@ -2484,7 +2500,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {+@@ -2600,7 +2617,7 @@ ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {return &ggml_backend_cpu_buffer_type;}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patchindex d388f878..1789da20 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.c.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c--- a/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c+++ b/llama.cpp/ggml/src/ggml-cpu/ggml-cpu.c-@@ -1555,6 +1555,26 @@ static void ggml_compute_forward_mul_mat_id(+@@ -1559,6 +1559,26 @@ static void ggml_compute_forward_mul_mat_id(const struct ggml_tensor * src1 = dst->src[1];const struct ggml_tensor * ids = dst->src[2];@@ -28,7 +28,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cGGML_TENSOR_BINARY_OP_LOCALSconst int ith = params->ith;-@@ -1691,6 +1711,24 @@ static void ggml_compute_forward_mul_mat_id(+@@ -1692,6 +1712,24 @@ static void ggml_compute_forward_mul_mat_id(const int64_t nr0 = ne01;const int64_t nr1 = cne1;@@ -53,10 +53,10 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.cint chunk_size = 16;if (nr0 == 1 || nr1 == 1) {chunk_size = 64;-@@ -2919,6 +2957,19 @@ struct ggml_cplan ggml_graph_plan(- if (ggml_cpu_iqp_supports_mul_mat_id(node)) {- cur += n_tasks * ggml_cpu_iqp_scratch_size(node) + 64;- }+@@ -2921,6 +2959,19 @@ struct ggml_cplan ggml_graph_plan(+ // Workspace for tiled (see tiled.h)+ cur = GGML_PAD(cur, 64);+ cur += ggml_tiled_wdata_size(n_tasks, node);+#if GGML_USE_LLAMAFILE+ // [jart] reserve work-buffer space for llamafile_mixmul.+ // Without this, MixMul::allocate_shared_memory() indiff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patchindex fc7397ea..edc8d77a 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ggml-cpu.cpp.patch@@ -151,7 +151,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppconst struct ggml_tensor * src0 = op->src[0];const struct ggml_tensor * src1 = op->src[1];-@@ -483,7 +483,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st+@@ -486,7 +486,7 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st}}@@ -160,7 +160,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppreturn ggml_backend_buft_is_host(buft) || ggml_backend_cpu_is_extra_buffer_type(buft);GGML_UNUSED(dev);}-@@ -508,19 +508,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {+@@ -511,19 +511,19 @@ static const struct ggml_backend_device_i ggml_backend_cpu_device_i = {// CPU backend - backend (reg)@@ -183,7 +183,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppGGML_ASSERT(index == 0);static ggml_backend_cpu_device_context ctx;-@@ -535,7 +535,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg+@@ -538,7 +538,7 @@ static ggml_backend_dev_t ggml_backend_cpu_reg_get_device(ggml_backend_reg_t reg// This is intended to replace the the ggml_cpu_has_* functions when loading the CPU backend dynamically,// and additionally to allow other backends to expose their own list of features that applications can query using the same API@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppstatic std::vector<ggml_backend_feature> features = []() {ggml_cpu_init();-@@ -652,7 +652,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r+@@ -655,7 +655,7 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t rGGML_UNUSED(reg);}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patchindex b9208666..39584e02 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_ops.cpp.patch@@ -20,7 +20,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp// ggml_compute_forward_dupstatic void ggml_compute_forward_dup_same_cont(-@@ -8693,6 +8705,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(+@@ -8897,6 +8909,15 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type");GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type");@@ -36,7 +36,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppint ith = params->ith;for (int ir = ir0; ir < ir1; ++ir) {-@@ -8712,10 +8733,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(+@@ -8916,10 +8937,22 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(ggml_fp16_t * VKQ16 = (ggml_fp16_t *) (VKQ32 + 1*DV); // (temporary) FP16 VKQ accumulatorggml_fp16_t * Q_q = (ggml_fp16_t *) (VKQ32 + 2*DV); // (temporary) buffer for Q converted to quantized/FP16@@ -62,7 +62,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp}const ggml_fp16_t * mp = mask ? (ggml_fp16_t *)((char *) mask->data + iq1*mask->nb[1] + (iq2%mask->ne[2])*mask->nb[2] + (iq3%mask->ne[3])*mask->nb[3]) : NULL;-@@ -8744,7 +8777,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(+@@ -8948,7 +8981,13 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(float s; // KQ valueconst char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3);@@ -76,7 +76,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpps = s*scale; // scale KQ value-@@ -8761,21 +8800,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(+@@ -8965,21 +9004,36 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(const char * v_data = ((const char *) v->data + (ic*nbv1 + iv2*nbv2 + iv3*nbv3));@@ -116,7 +116,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp} else {if (s > M) {// s is new maximum, ms < 1.0f, vs == expf(s - s) == 1.0f-@@ -8783,26 +8837,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(+@@ -8987,26 +9041,20 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(ms = expf(Mold - M);// V = V*expf(Mold - M)@@ -146,7 +146,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cppfor (int64_t d = 0; d < DV; ++d) {VKQ32[d] = GGML_CPU_FP16_TO_FP32(VKQ16[d]);}-@@ -9023,8 +9071,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(+@@ -9227,8 +9275,24 @@ static void ggml_compute_forward_flash_attn_ext_tiled(const char * k_data = (const char *)k->data + (ic + tk)*nbk1 + ik2*nbk2 + ik3*nbk3;if (kv_type == GGML_TYPE_F16) {const ggml_fp16_t * k_f16 = (const ggml_fp16_t *)k_data;@@ -173,7 +173,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp}} else {const float * k_f32_src = (const float *)k_data;-@@ -9034,7 +9098,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(+@@ -9238,7 +9302,18 @@ static void ggml_compute_forward_flash_attn_ext_tiled(}}memset(KQ, 0, Q_TILE_SZ * KV_TILE_SZ * sizeof(float));@@ -191,8 +191,8 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp+#endifggml_vec_scale_f32(Q_TILE_SZ * KV_TILE_SZ, KQ, scale);- // Set padded KQ entries to -inf so softmax gives them zero weight-@@ -9087,7 +9162,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(+ if (logit_softcap != 0.0f) {+@@ -9291,7 +9366,15 @@ static void ggml_compute_forward_flash_attn_ext_tiled(for (int tk = 0; tk < kv_tile; tk++) {const char * v_data = (const char *)v->data + (ic + tk)*nbv1 + iv2*nbv2 + iv3*nbv3;if (kv_type == GGML_TYPE_F16) {@@ -208,7 +208,7 @@ diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp} else {memcpy(V32 + tk * DV, v_data, DV * sizeof(float));}-@@ -9097,7 +9180,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(+@@ -9301,7 +9384,16 @@ static void ggml_compute_forward_flash_attn_ext_tiled(memset(KQ + tq * KV_TILE_SZ, 0, KV_TILE_SZ * sizeof(float));}}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patchindex c067ae03..1df7c54e 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cpu_repack.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp--- a/llama.cpp/ggml/src/ggml-cpu/repack.cpp+++ b/llama.cpp/ggml/src/ggml-cpu/repack.cpp-@@ -5140,14 +5140,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons+@@ -5145,14 +5145,14 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(consreturn nullptr;}@@ -18,7 +18,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cppconst void * data, size_t offset, size_t size) {GGML_ASSERT(offset == 0);GGML_ASSERT(size == ggml_nbytes(tensor));-@@ -5159,13 +5159,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buff+@@ -5164,13 +5164,13 @@ static void ggml_backend_cpu_repack_buffer_set_tensor(ggml_backend_buffer_t buffGGML_UNUSED(buffer);}@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cppggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(ggml_backend_cpu_buffer_type(), size);if (buffer == nullptr) {-@@ -5180,7 +5180,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(gg+@@ -5185,7 +5185,7 @@ static ggml_backend_buffer_t ggml_backend_cpu_repack_buffer_type_alloc_buffer(ggreturn buffer;}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patchindex 8a2e8a33..b5168512 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_common.cuh.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh--- a/llama.cpp/ggml/src/ggml-cuda/common.cuh+++ b/llama.cpp/ggml/src/ggml-cuda/common.cuh-@@ -336,9 +336,15 @@ static bool fast_bf16_hardware_available(const int cc) {+@@ -337,9 +337,15 @@ static bool fast_bf16_hardware_available(const int cc) {}static bool bf16_mma_hardware_available(const int cc) {diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patchindex 0a13aeef..ad35547e 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_convert.cu.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu--- a/llama.cpp/ggml/src/ggml-cuda/convert.cu+++ b/llama.cpp/ggml/src/ggml-cuda/convert.cu-@@ -513,6 +513,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {+@@ -519,6 +519,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:return dequantize_row_q6_K_cuda;@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:-@@ -531,6 +532,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {+@@ -537,6 +538,7 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {return dequantize_row_iq4_xs_cuda;case GGML_TYPE_IQ3_S:return dequantize_row_iq3_s_cuda;@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_MXFP4:return dequantize_row_mxfp4_cuda;case GGML_TYPE_NVFP4:-@@ -573,6 +575,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+@@ -579,6 +581,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_q5_K_cuda;case GGML_TYPE_Q6_K:return dequantize_row_q6_K_cuda;@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cucase GGML_TYPE_IQ2_XXS:return dequantize_row_iq2_xxs_cuda;case GGML_TYPE_IQ2_XS:-@@ -591,6 +594,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {+@@ -597,6 +600,7 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {return dequantize_row_iq4_xs_cuda;case GGML_TYPE_IQ3_S:return dequantize_row_iq3_s_cuda;diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patchindex 01dd1f39..47e4086a 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_cpy.cu.patch@@ -17,7 +17,7 @@ diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu// check if a same-type copy reduces to a 2D strided copy (height rows of width// contiguous bytes), so it can use cudaMemcpy2DAsync instead of the scalar kernel-@@ -524,9 +526,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg+@@ -525,9 +527,11 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg} else if (src0->type == GGML_TYPE_Q5_0 && src1->type == GGML_TYPE_F32) {ggml_cpy_q5_0_f32_cuda(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patchindex 72ec44fb..6f5dfd83 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_ggml-cuda.cu.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu--- a/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/llama.cpp/ggml/src/ggml-cuda/ggml-cuda.cu-@@ -738,7 +738,7 @@ struct ggml_backend_cuda_buffer_context {+@@ -781,7 +781,7 @@ struct ggml_backend_cuda_buffer_context {}};@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;delete ctx;}-@@ -747,12 +747,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {+@@ -790,12 +790,12 @@ static bool ggml_backend_buffer_is_cuda(ggml_backend_buffer_t buffer) {return buffer->iface.free_buffer == ggml_backend_cuda_buffer_free_buffer;}@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;if (tensor->view_src != NULL) {-@@ -773,7 +773,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_buffer+@@ -817,7 +817,7 @@ static enum ggml_status ggml_backend_cuda_buffer_init_tensor(ggml_backend_bufferreturn GGML_STATUS_SUCCESS;}@@ -34,7 +34,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -781,7 +781,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,+@@ -825,7 +825,7 @@ static void ggml_backend_cuda_buffer_memset_tensor(ggml_backend_buffer_t buffer,CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -43,7 +43,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -789,7 +789,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, gg+@@ -833,7 +833,7 @@ static void ggml_backend_cuda_buffer_set_tensor(ggml_backend_buffer_t buffer, ggCUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -52,7 +52,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;ggml_cuda_set_device(ctx->device);-@@ -797,7 +797,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, co+@@ -841,7 +841,7 @@ static void ggml_backend_cuda_buffer_get_tensor(ggml_backend_buffer_t buffer, coCUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -61,7 +61,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cusize_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *) buffer->context;-@@ -807,7 +807,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,+@@ -851,7 +851,7 @@ static void ggml_backend_cuda_buffer_set_tensor_2d(ggml_backend_buffer_t buffer,CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -70,7 +70,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cusize_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;-@@ -817,7 +817,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,+@@ -861,7 +861,7 @@ static void ggml_backend_cuda_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -79,7 +79,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (ggml_backend_buffer_is_cuda(src->buffer)) {ggml_backend_cuda_buffer_context * src_ctx = (ggml_backend_cuda_buffer_context *)src->buffer->context;ggml_backend_cuda_buffer_context * dst_ctx = (ggml_backend_cuda_buffer_context *)dst->buffer->context;-@@ -842,7 +842,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, co+@@ -886,7 +886,7 @@ static bool ggml_backend_cuda_buffer_cpy_tensor(ggml_backend_buffer_t buffer, coGGML_UNUSED(buffer);}@@ -88,7 +88,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context;ggml_cuda_set_device(ctx->device);-@@ -850,6 +850,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_t+@@ -894,6 +894,14 @@ static void ggml_backend_cuda_buffer_clear(ggml_backend_buffer_t buffer, uint8_tCUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));}@@ -103,7 +103,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.custatic const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {/* .free_buffer = */ ggml_backend_cuda_buffer_free_buffer,/* .get_base = */ ggml_backend_cuda_buffer_get_base,-@@ -862,6 +870,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {+@@ -906,6 +914,7 @@ static const ggml_backend_buffer_i ggml_backend_cuda_buffer_interface = {/* .cpy_tensor = */ ggml_backend_cuda_buffer_cpy_tensor,/* .clear = */ ggml_backend_cuda_buffer_clear,/* .reset = */ NULL,@@ -111,7 +111,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu};// cuda buffer type-@@ -870,7 +879,7 @@ struct ggml_backend_cuda_buffer_type_context {+@@ -914,7 +923,7 @@ struct ggml_backend_cuda_buffer_type_context {std::string name;};@@ -120,7 +120,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_type_context * ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;return ctx->name.c_str();-@@ -880,7 +889,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {+@@ -924,7 +933,7 @@ static bool ggml_backend_buft_is_cuda(ggml_backend_buffer_type_t buft) {return buft->iface.get_name == ggml_backend_cuda_buffer_type_get_name;}@@ -129,7 +129,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *)buft->context;ggml_cuda_set_device(buft_ctx->device);-@@ -899,13 +908,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac+@@ -943,13 +952,13 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bacreturn ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size);}@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_buffer_type_context * buft_ctx = (ggml_backend_cuda_buffer_type_context *) buft->context;size_t size = tensor->op == GGML_OP_FLASH_ATTN_EXT-@@ -1259,7 +1268,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg+@@ -1305,7 +1314,7 @@ static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct gg// host buffer type@@ -154,7 +154,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cureturn GGML_CUDA_NAME "_Host";GGML_UNUSED(buft);-@@ -1269,7 +1278,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {+@@ -1315,7 +1324,7 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) {return buft->iface.get_name == ggml_backend_cuda_host_buffer_type_name;}@@ -163,7 +163,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuCUDA_CHECK(cudaFreeHost(buffer->context));}-@@ -1291,17 +1300,25 @@ static void * ggml_cuda_host_malloc(size_t size) {+@@ -1337,22 +1346,30 @@ static void * ggml_cuda_host_malloc(size_t size) {return ptr;}@@ -191,7 +191,13 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cureturn buffer;}-@@ -1655,6 +1672,14 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml++-static size_t ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {++static size_t GGML_CALL ggml_backend_cuda_host_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {+ GGML_UNUSED(buft);+ // very large pinned allocations can fail (e.g. on Windows) and fall back to pageable memory, smaller ones succeed+ return 4ull*1024*1024*1024;+@@ -1739,6 +1756,14 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml}}@@ -206,7 +212,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuswitch (compute_type) {case GGML_TYPE_F32:ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F32>(ctx, src0, src1, dst);-@@ -2429,20 +2454,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg+@@ -2630,20 +2655,20 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg// backend@@ -230,7 +236,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2451,7 +2476,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tens+@@ -2652,7 +2677,7 @@ static void ggml_backend_cuda_set_tensor_async(ggml_backend_t backend, ggml_tensCUDA_CHECK(cudaMemcpyAsync((char *) tensor->data + offset, data, size, cudaMemcpyHostToDevice, cuda_ctx->stream()));}@@ -239,7 +245,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2460,7 +2485,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggm+@@ -2661,7 +2686,7 @@ static void ggml_backend_cuda_get_tensor_async(ggml_backend_t backend, const ggmCUDA_CHECK(cudaMemcpyAsync(data, (const char *) tensor->data + offset, size, cudaMemcpyDeviceToHost, cuda_ctx->stream()));}@@ -248,7 +254,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cusize_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2471,7 +2496,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct+@@ -2672,7 +2697,7 @@ static void ggml_backend_cuda_set_tensor_2d_async(ggml_backend_t backend, struct(char *) tensor->data + offset, stride_tensor, data, stride_data, size, n_copies, cudaMemcpyHostToDevice, cuda_ctx->stream()));}@@ -257,7 +263,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cusize_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_backend_buffer_t buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer;-@@ -2482,7 +2507,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, const+@@ -2683,7 +2708,7 @@ static void ggml_backend_cuda_get_tensor_2d_async(ggml_backend_t backend, constdata, stride_data, (const char *) tensor->data + offset, stride_tensor, size, n_copies, cudaMemcpyDeviceToHost, cuda_ctx->stream()));}@@ -266,7 +272,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_buffer_t buf_src = src->view_src ? src->view_src->buffer : src->buffer;ggml_backend_buffer_t buf_dst = dst->view_src ? dst->view_src->buffer : dst->buffer;-@@ -2541,7 +2566,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_+@@ -2742,7 +2767,7 @@ static bool ggml_backend_cuda_cpy_tensor_async(ggml_backend_t backend_src, ggml_return true;}@@ -275,7 +281,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;CUDA_CHECK(cudaStreamSynchronize(cuda_ctx->stream()));-@@ -4417,7 +4442,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co+@@ -4899,7 +4924,7 @@ static bool ggml_cuda_graph_set_enabled(ggml_backend_cuda_context * cuda_ctx, co}#endif // USE_CUDA_GRAPH@@ -284,7 +290,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;ggml_cuda_set_device(cuda_ctx->device);-@@ -4476,13 +4501,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,+@@ -4962,13 +4987,13 @@ static enum ggml_status ggml_backend_cuda_graph_compute(ggml_backend_t backend,return GGML_STATUS_SUCCESS;}@@ -300,7 +306,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context;if (ggml_backend_is_cuda(backend)) {-@@ -4501,7 +4526,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev+@@ -4987,7 +5012,7 @@ static void ggml_backend_cuda_event_wait(ggml_backend_t backend, ggml_backend_ev}}@@ -308,26 +314,26 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu+static void GGML_CALL ggml_backend_cuda_graph_optimize(ggml_backend_t backend, ggml_cgraph * cgraph, ggml_backend_graph_optimize_params * params) {ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context;- static const bool disable_fusion = getenv("GGML_CUDA_DISABLE_FUSION") != nullptr && std::atoi(getenv("GGML_CUDA_DISABLE_FUSION"));-@@ -4835,7 +4860,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot+ const bool disable_fusion = ggml_cuda_fusion_disabled();+@@ -5422,7 +5447,7 @@ void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * tot*total /= share_count;}-bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {+bool GGML_CALL ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ if (!ggml_cuda_register_host_enabled()) {return false;}-@@ -4858,7 +4883,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {+@@ -5445,7 +5470,7 @@ bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size) {#endif // CUDART_VERSION >= 11010 || defined(GGML_USE_MUSA)}-void ggml_backend_cuda_unregister_host_buffer(void * buffer) {+void GGML_CALL ggml_backend_cuda_unregister_host_buffer(void * buffer) {- if (getenv("GGML_CUDA_REGISTER_HOST") == nullptr) {+ if (!ggml_cuda_register_host_enabled()) {return;}-@@ -4881,12 +4906,12 @@ struct ggml_backend_cuda_device_context {+@@ -5468,12 +5493,12 @@ struct ggml_backend_cuda_device_context {int op_offload_min_batch_size;};@@ -342,7 +348,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;return ctx->description.c_str();}-@@ -4967,7 +4992,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k+@@ -5554,7 +5579,7 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k}#endif // defined(__linux__)@@ -351,7 +357,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;ggml_cuda_set_device(ctx->device);cudaError_t err = cudaMemGetInfo(free, total);-@@ -5008,7 +5033,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *+@@ -5594,7 +5619,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t **total /= share_count;}@@ -360,7 +366,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context;cudaDeviceProp prop;-@@ -5019,7 +5044,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend+@@ -5605,7 +5630,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend: GGML_BACKEND_DEVICE_TYPE_GPU;}@@ -369,7 +375,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;props->name = ggml_backend_cuda_device_get_name(dev);-@@ -5044,24 +5069,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back+@@ -5630,24 +5655,24 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back};}@@ -398,7 +404,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;// check if all the sources are allocated on this device-@@ -5162,6 +5187,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -5757,6 +5782,13 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:case GGML_TYPE_Q8_K:@@ -412,7 +418,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cucase GGML_TYPE_IQ1_M:case GGML_TYPE_IQ1_S:case GGML_TYPE_IQ2_S:-@@ -5171,6 +5203,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -5766,6 +5798,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gcase GGML_TYPE_IQ3_XXS:case GGML_TYPE_IQ4_NL:case GGML_TYPE_IQ4_XS:@@ -420,7 +426,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cucase GGML_TYPE_BF16:return true;default:-@@ -5280,9 +5313,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -5873,9 +5906,14 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const gif (src0_type == GGML_TYPE_Q5_1 && src1_type == GGML_TYPE_F32) {return true;}@@ -435,7 +441,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuif (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) {return true;}-@@ -5519,7 +5557,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g+@@ -6121,7 +6159,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g}}@@ -444,7 +450,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context;const bool integrated = ggml_cuda_info().devices[dev_ctx->device].integrated;return (ggml_backend_buft_is_cuda(buft) && buft->device == dev) || (integrated && ggml_backend_buft_is_cuda_host(buft));-@@ -5540,13 +5578,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {+@@ -6142,13 +6180,13 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {}}@@ -460,7 +466,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu#ifdef GGML_CUDA_NO_PEER_COPYGGML_UNUSED(dev);return nullptr;-@@ -5565,14 +5603,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_+@@ -6167,14 +6205,14 @@ static ggml_backend_event_t ggml_backend_cuda_device_event_new(ggml_backend_dev_#endif}@@ -477,7 +483,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuGGML_UNUSED(dev);CUDA_CHECK(cudaEventSynchronize((cudaEvent_t)event->context));}-@@ -5601,23 +5639,23 @@ struct ggml_backend_cuda_reg_context {+@@ -6203,23 +6241,23 @@ struct ggml_backend_cuda_reg_context {std::vector<ggml_backend_dev_t> devices;};@@ -505,7 +511,7 @@ diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.custatic std::vector<ggml_backend_feature> features = []() {std::vector<ggml_backend_feature> features;#define _STRINGIFY(...) #__VA_ARGS__-@@ -5674,7 +5712,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t+@@ -6276,7 +6314,7 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_tGGML_UNUSED(reg);}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patchindex 673e761f..67bedc19 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cu.patch@@ -2,22 +2,22 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cu+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cu@@ -45,6 +45,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con- mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);+ mul_mat_q_case<GGML_TYPE_Q6_K, GGML_PREC_Q8>(ctx, args, stream);break;// -----------------------------------------------------------------------+#ifndef GGML_CUDA_NO_IQ_QUANTScase GGML_TYPE_IQ1_S:- mul_mat_q_case<GGML_TYPE_IQ1_S>(ctx, args, stream);+ mul_mat_q_case<GGML_TYPE_IQ1_S, GGML_PREC_Q8>(ctx, args, stream);break;@@ -69,6 +70,7 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, concase GGML_TYPE_IQ4_NL:- mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);+ mul_mat_q_case<GGML_TYPE_IQ4_NL, GGML_PREC_Q8>(ctx, args, stream);break;+#endif // GGML_CUDA_NO_IQ_QUANTS// -----------------------------------------------------------------------case GGML_TYPE_MXFP4:- mul_mat_q_case<GGML_TYPE_MXFP4>(ctx, args, stream);-@@ -285,6 +287,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+ // src1 at Q4 uses the native FP4 instructions, which are Blackwell-only+@@ -405,6 +407,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_Q5_K:case GGML_TYPE_Q6_K:// -------------------------------------------------@@ -25,7 +25,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cucase GGML_TYPE_IQ1_S:case GGML_TYPE_IQ2_XXS:case GGML_TYPE_IQ2_XS:-@@ -293,6 +296,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+@@ -413,6 +416,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_tcase GGML_TYPE_IQ3_S:case GGML_TYPE_IQ4_XS:case GGML_TYPE_IQ4_NL:@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu// -------------------------------------------------case GGML_TYPE_MXFP4:case GGML_TYPE_NVFP4:-@@ -369,9 +373,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t+@@ -489,9 +493,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_treturn ne11 <= 128;case GGML_TYPE_Q6_K:return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patchindex e8b3f216..e75ec5e1 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-cuda_mmq.cuh.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh--- a/llama.cpp/ggml/src/ggml-cuda/mmq.cuh+++ b/llama.cpp/ggml/src/ggml-cuda/mmq.cuh-@@ -1585,6 +1585,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);+@@ -1634,6 +1634,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q4_K);extern DECL_MMQ_CASE(GGML_TYPE_Q5_K);extern DECL_MMQ_CASE(GGML_TYPE_Q6_K);// -----------------------------------------@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuhextern DECL_MMQ_CASE(GGML_TYPE_IQ1_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS);-@@ -1593,6 +1594,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);+@@ -1642,6 +1643,7 @@ extern DECL_MMQ_CASE(GGML_TYPE_IQ3_XXS);extern DECL_MMQ_CASE(GGML_TYPE_IQ3_S);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_NL);extern DECL_MMQ_CASE(GGML_TYPE_IQ4_XS);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patchindex 6f17a51e..20f3858b 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggml-metal-fusion.cpp--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.cpp+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.cpp-@@ -717,7 +717,7 @@ static bool ggml_metal_fusion_match_raw_pattern(+@@ -811,7 +811,7 @@ static bool ggml_metal_fusion_match_raw_pattern(static void ggml_metal_fusion_add_pattern_alloc_deps(void * user_data,@@ -10,7 +10,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.cpp b/ggml/src/ggml-metal/ggmggml_cgraph * gf,const ggml_metal_fusion * fusion,int node_idx) {-@@ -739,7 +739,7 @@ static void ggml_metal_fusion_add_pattern_alloc_deps(+@@ -833,7 +833,7 @@ static void ggml_metal_fusion_add_pattern_alloc_deps(void ggml_metal_fusion_add_alloc_deps(void * user_data,diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patchindex 5dbbf36a..6aa4979d 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal-fusion.h.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-metal-fusion.h--- a/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.h+++ b/llama.cpp/ggml/src/ggml-metal/ggml-metal-fusion.h-@@ -8,6 +8,7 @@+@@ -9,6 +9,7 @@#pragma once#include "ggml-impl.h"@@ -9,7 +9,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal-fusion.h b/ggml/src/ggml-metal/ggml-#include <stdbool.h>-@@ -51,7 +52,7 @@ ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * f+@@ -55,7 +56,7 @@ ggml_metal_fusion_id ggml_metal_fusion_get_id(const struct ggml_metal_fusion * f// apply any alloc-dependencies required by the fused kernels during graph optimizevoid ggml_metal_fusion_add_alloc_deps(void * user_data,diff --git a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patchindex d5b1b90c..4e943d42 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-metal_ggml-metal.cpp.patch@@ -155,7 +155,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal};static bool ggml_backend_buffer_is_metal(ggml_backend_buffer_t buffer) {-@@ -256,33 +266,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_+@@ -259,33 +269,33 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_// default (shared) buffer type@@ -195,7 +195,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -332,33 +342,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi+@@ -337,33 +347,33 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_shared(int devi// default (private) buffer type@@ -235,7 +235,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -407,34 +417,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev+@@ -414,34 +424,34 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_private(int dev// mapped buffer type@@ -276,7 +276,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn false;GGML_UNUSED(buft);-@@ -485,13 +495,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi+@@ -494,13 +504,13 @@ static ggml_backend_buffer_type_t ggml_backend_metal_buffer_type_mapped(int devi// backend@@ -292,7 +292,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_t ctx = (ggml_metal_t)backend->context;// wait for any ongoing async operations to finish-@@ -502,25 +512,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {+@@ -511,25 +521,25 @@ static void ggml_backend_metal_free(ggml_backend_t backend) {free(backend);}@@ -322,7 +322,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalif (!ggml_backend_is_metal(backend_src) || !ggml_backend_is_metal(backend_dst)) {return false;}-@@ -541,27 +551,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggml+@@ -550,27 +560,27 @@ static bool ggml_backend_metal_cpy_tensor_async(ggml_backend_t backend_src, ggmlreturn ggml_metal_cpy_tensor_async(ctx_src, ctx_dst, src, dst);}@@ -354,7 +354,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalGGML_ASSERT(params && params->add_alloc_dep);// keep the MoE weighted-reduction inputs alive until the fused output so the-@@ -659,7 +669,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {+@@ -668,7 +678,7 @@ void ggml_backend_metal_capture_next_compute(ggml_backend_t backend) {// backend device@@ -363,7 +363,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);-@@ -667,25 +677,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {+@@ -676,25 +686,25 @@ static const char * ggml_backend_metal_device_get_name(ggml_backend_dev_t dev) {return props_dev->name;}@@ -393,7 +393,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalprops->name = ggml_backend_metal_device_get_name(dev);props->description = ggml_backend_metal_device_get_description(dev);props->type = ggml_backend_metal_device_get_type(dev);-@@ -701,7 +711,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac+@@ -710,7 +720,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac};}@@ -402,7 +402,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_t ctx = ggml_metal_init(ctx_dev);-@@ -726,7 +736,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_t+@@ -735,7 +745,7 @@ static ggml_backend_t ggml_backend_metal_device_init_backend(ggml_backend_dev_tGGML_UNUSED(params);}@@ -411,7 +411,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx_dev);-@@ -734,7 +744,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggml+@@ -743,7 +753,7 @@ static ggml_backend_buffer_type_t ggml_backend_metal_device_get_buffer_type(ggmlreturn props_dev->use_shared_buffers ? ggml_backend_metal_buffer_type_shared(props_dev->device) : ggml_backend_metal_buffer_type_private(props_dev->device);}@@ -420,7 +420,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_buffer_t res = ggml_metal_buffer_map(ctx_dev, ptr, size, max_tensor_size);-@@ -744,13 +754,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backen+@@ -753,13 +763,13 @@ static ggml_backend_buffer_t ggml_backend_metal_device_buffer_mapped(ggml_backenreturn ggml_backend_buffer_init(ggml_backend_metal_buffer_type_mapped(props_dev->device), ggml_backend_metal_buffer_shared_i, res, size);}@@ -436,7 +436,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturnbuft->device == dev && (buft->iface.get_name == ggml_backend_metal_buffer_type_shared_get_name ||-@@ -771,7 +781,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {+@@ -780,7 +790,7 @@ static int64_t get_op_batch_size(const ggml_tensor * op) {}}@@ -445,7 +445,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;return (op->op == GGML_OP_MUL_MAT ||-@@ -779,7 +789,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const g+@@ -788,7 +798,7 @@ static bool ggml_backend_metal_device_offload_op(ggml_backend_dev_t dev, const gget_op_batch_size(op) >= ggml_metal_device_get_props(ctx_dev)->op_offload_min_batch_size;}@@ -454,7 +454,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t event = ggml_metal_device_event_init(ctx_dev);-@@ -793,7 +803,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_dev+@@ -802,7 +812,7 @@ static ggml_backend_event_t ggml_backend_metal_device_event_new(ggml_backend_devreturn ev;}@@ -463,7 +463,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t ev = (ggml_metal_event_t)event->context;-@@ -803,7 +813,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_ba+@@ -812,7 +822,7 @@ static void ggml_backend_metal_device_event_free(ggml_backend_dev_t dev, ggml_badelete event;}@@ -472,7 +472,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_metal_device_t ctx_dev = (ggml_metal_device_t)dev->context;ggml_metal_event_t evt = (ggml_metal_event_t)event->context;-@@ -855,18 +865,18 @@ struct ggml_backend_metal_reg_deleter {+@@ -864,18 +874,18 @@ struct ggml_backend_metal_reg_deleter {typedef std::unique_ptr<struct ggml_backend_metal_reg, ggml_backend_metal_reg_deleter> ggml_backend_metal_reg_ptr;@@ -494,7 +494,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalggml_backend_metal_reg_t ctx = (ggml_backend_metal_reg_t)reg->context;GGML_ASSERT(index < ctx->devices.size());return ctx->devices[index];-@@ -879,7 +889,7 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {+@@ -888,7 +898,7 @@ static ggml_backend_feature g_ggml_backend_metal_features[] = {{ NULL, NULL },};@@ -503,7 +503,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metalreturn g_ggml_backend_metal_features;GGML_UNUSED(reg);-@@ -936,7 +946,7 @@ static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, b+@@ -945,7 +955,7 @@ static void ggml_backend_metal_fusion_set_enabled(ggml_backend_fusion_t finfo, bggml_metal_fusion_info_set_enabled((struct ggml_metal_fusion_info *) finfo, enabled);}diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patchindex 15445f90..b35ecf46 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-buffers.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp b/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-buffers.cpp-@@ -754,6 +754,7 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {+@@ -837,6 +837,7 @@ ggml_backend_buffer_i ggml_backend_vk_buffer_interface = {/* .cpy_tensor = */ ggml_backend_vk_buffer_cpy_tensor,/* .clear = */ ggml_backend_vk_buffer_clear,/* .reset = */ NULL,diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patchindex 779fdcf5..cab214ba 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan-common.h.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-common.h b/ggml/src/ggml-vulkan/ggml-vulkan-common.h--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-common.h+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan-common.h-@@ -209,26 +209,27 @@ bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgr+@@ -213,26 +213,27 @@ bool ggml_vk_can_fuse_snake(ggml_backend_vk_context * ctx, const struct ggml_cgrbool ggml_vk_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b, bool elementwise);bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);uint32_t ggml_vk_fuse_multi_add(ggml_backend_vk_context * ctx, const struct ggml_cgraph * cgraph, int node_idx);diff --git a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patchindex 36b4784f..62b24289 100644--- a/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml-vulkan_ggml-vulkan.cpp.patch@@ -1,7 +1,7 @@diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp--- a/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/llama.cpp/ggml/src/ggml-vulkan/ggml-vulkan.cpp-@@ -12492,20 +12492,28 @@ bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {+@@ -13096,20 +13096,28 @@ bool ggml_backend_buffer_is_vk(ggml_backend_buffer_t buffer) {return buffer->buft->iface.get_name == ggml_backend_vk_buffer_type_name;}@@ -33,7 +33,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_init_tensor(" << buffer << " (" << buffer->context << "), " << tensor << ")");if (tensor->view_src != nullptr) {GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);-@@ -12513,7 +12521,7 @@ enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t buffer+@@ -13117,7 +13125,7 @@ enum ggml_status ggml_backend_vk_buffer_init_tensor(ggml_backend_buffer_t bufferreturn GGML_STATUS_SUCCESS;}@@ -42,7 +42,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_memset_tensor(" << buffer << ", " << tensor << ", " << value << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;-@@ -12526,7 +12534,7 @@ void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_ten+@@ -13130,7 +13138,7 @@ void ggml_backend_vk_buffer_memset_tensor(ggml_backend_buffer_t buffer, ggml_tenggml_vk_buffer_memset(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, val32, size);}@@ -51,7 +51,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;vk_buffer buf = buf_ctx->dev_buffer;-@@ -12538,7 +12546,7 @@ void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor+@@ -13142,7 +13150,7 @@ void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensorggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}@@ -60,7 +60,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vusize_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_buffer_set_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<n_copies << ", " << stride_tensor << ", " << stride_data << ")");-@@ -12552,7 +12560,7 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_ten+@@ -13156,7 +13164,7 @@ void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tenggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);}@@ -69,7 +69,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ")");ggml_backend_vk_buffer_context * buf_ctx = (ggml_backend_vk_buffer_context *)buffer->context;-@@ -12565,7 +12573,7 @@ void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_+@@ -13169,7 +13177,7 @@ void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, size);}@@ -78,7 +78,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vusize_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_buffer_get_tensor_2d(" << buffer << ", " << tensor << ", " << data << ", " << offset << ", " << size << ", " <<n_copies << ", " << stride_tensor << ", " << stride_data << ")");-@@ -12580,7 +12588,7 @@ void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const gg+@@ -13184,7 +13192,7 @@ void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, const ggggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);}@@ -87,7 +87,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuif (ggml_nbytes(src) == 0) {return true;}-@@ -12601,19 +12609,19 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_+@@ -13205,19 +13213,19 @@ bool ggml_backend_vk_buffer_cpy_tensor(ggml_backend_buffer_t buffer, const ggml_UNUSED(buffer);}@@ -110,7 +110,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_MEMORY("ggml_backend_vk_buffer_type_alloc_buffer(" << size << ")");ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *) buft->context;-@@ -12629,17 +12637,17 @@ ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buff+@@ -13233,17 +13241,17 @@ ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backend_buffreturn ggml_backend_buffer_init(buft, ggml_backend_vk_buffer_interface, bufctx, size);}@@ -131,7 +131,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vureturn ggml_nbytes(tensor);UNUSED(buft);-@@ -12655,18 +12663,18 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {+@@ -13259,18 +13267,18 @@ ggml_backend_buffer_type_t ggml_backend_vk_buffer_type(size_t dev_num) {return &dev->buffer_type;}@@ -145,7 +145,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu-static void ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {+static void GGML_CALL ggml_backend_vk_host_buffer_free_buffer(ggml_backend_buffer_t buffer) {VK_LOG_MEMORY("ggml_backend_vk_host_buffer_free_buffer()");- ggml_vk_host_free(vk_instance.devices[0], buffer->context);+ ggml_vk_host_free(buffer->context);}-static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {@@ -153,7 +153,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_MEMORY("ggml_backend_vk_host_buffer_type_alloc_buffer(" << size << ")");size += 32; // Behave like the CPU buffer type-@@ -12682,19 +12690,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_+@@ -13284,19 +13292,20 @@ static ggml_backend_buffer_t ggml_backend_vk_host_buffer_type_alloc_buffer(ggml_ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);buffer->buft = buft;buffer->iface.free_buffer = ggml_backend_vk_host_buffer_free_buffer;@@ -176,7 +176,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vureturn vk_instance.devices[0]->suballocation_block_size;UNUSED(buft);-@@ -12721,13 +12730,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {+@@ -13325,13 +13334,13 @@ ggml_backend_buffer_type_t ggml_backend_vk_host_buffer_type() {return &ggml_backend_vk_buffer_type_host;}@@ -192,7 +192,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;VK_LOG_DEBUG("ggml_backend_vk_free(" << ctx->name << ")");-@@ -12743,7 +12752,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_b+@@ -13347,7 +13356,7 @@ static ggml_backend_buffer_type_t ggml_backend_vk_get_default_buffer_type(ggml_breturn &ctx->device->buffer_type;}@@ -201,7 +201,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vusize_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_set_tensor_2d_async(" << size << ", " << n_copies << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -12801,12 +12810,12 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten+@@ -13405,12 +13414,12 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten}}@@ -216,7 +216,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vusize_t size, size_t n_copies, size_t stride_tensor, size_t stride_data) {VK_LOG_DEBUG("ggml_backend_vk_get_tensor_2d_async(" << size << ", " << n_copies << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -12857,12 +12866,12 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg+@@ -13461,12 +13470,12 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg}}@@ -231,7 +231,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_cpy_tensor_async(" << src << " -> " << dst << ", size=" << ggml_nbytes(src) << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend_dst->context;-@@ -13004,7 +13013,7 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {+@@ -13608,7 +13617,7 @@ void ggml_vk_synchronize(ggml_backend_vk_context * ctx) {}}@@ -240,7 +240,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_synchronize()");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -13741,7 +13750,7 @@ static int32_t find_first_set(uint32_t x) {+@@ -14358,7 +14367,7 @@ static int32_t find_first_set(uint32_t x) {return ret;}@@ -249,7 +249,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_graph_compute(" << cgraph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -14236,7 +14245,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg+@@ -14862,7 +14871,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cgUNUSED(backend);}@@ -258,7 +258,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vu{VK_LOG_DEBUG("ggml_vk_graph_optimize(" << graph->n_nodes << " nodes)");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;-@@ -14617,7 +14626,7 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph,+@@ -15249,7 +15258,7 @@ void ggml_vk_graph_optimize(ggml_backend_t backend, struct ggml_cgraph * graph,}}@@ -267,7 +267,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_event_record(backend=" << backend << ", event=" << event << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;vk_event *vkev = (vk_event *)event->context;-@@ -14655,7 +14664,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_ev+@@ -15287,7 +15296,7 @@ static void ggml_backend_vk_event_record(ggml_backend_t backend, ggml_backend_evctx->compute_ctx.reset();}@@ -276,19 +276,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_event_wait(backend=" << backend << ", event=" << event << ")");ggml_backend_vk_context * ctx = (ggml_backend_vk_context *)backend->context;vk_event *vkev = (vk_event *)event->context;-@@ -14756,7 +14765,10 @@ void ggml_backend_vk_get_device_memory(int device, size_t * free, size_t * total- *total += heap.size;-- if (membudget_supported && i < budgetprops.heapUsage.size()) {-- *free += budgetprops.heapBudget[i] - budgetprops.heapUsage[i];-+ // heapUsage can exceed heapBudget (permitted by spec); clamp to 0 to avoid underflow-+ if (budgetprops.heapBudget[i] > budgetprops.heapUsage[i]) {-+ *free += budgetprops.heapBudget[i] - budgetprops.heapUsage[i];-+ }- } else {- *free += heap.size;- }-@@ -14813,38 +14825,38 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {+@@ -15448,38 +15457,38 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {return std::string(pci_bus_id);}@@ -334,7 +322,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;props->name = ggml_backend_vk_device_get_name(dev);-@@ -14861,13 +14873,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml+@@ -15496,13 +15505,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml};}@@ -350,7 +338,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;const vk_device& device = ggml_vk_get_device(ctx->device);-@@ -15522,7 +15534,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm+@@ -16164,7 +16173,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggmUNUSED(dev);}@@ -359,7 +347,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuif (buft->iface.get_name != ggml_backend_vk_buffer_type_name) {return false;}-@@ -15533,13 +15545,13 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_ba+@@ -16175,13 +16184,13 @@ static bool ggml_backend_vk_device_supports_buft(ggml_backend_dev_t dev, ggml_bareturn buft_ctx->device->idx == ctx->device;}@@ -375,7 +363,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15562,7 +15574,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t+@@ -16204,7 +16213,7 @@ static ggml_backend_event_t ggml_backend_vk_device_event_new(ggml_backend_dev_t};}@@ -384,7 +372,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15582,7 +15594,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backe+@@ -16224,7 +16233,7 @@ static void ggml_backend_vk_device_event_free(ggml_backend_dev_t dev, ggml_backedelete event;}@@ -393,7 +381,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_device_event_synchronize(backend=" << dev << ", event=" << event << ")");ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;auto device = ggml_vk_get_device(ctx->device);-@@ -15614,7 +15626,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm+@@ -16256,7 +16265,7 @@ static void ggml_backend_vk_device_event_synchronize(ggml_backend_dev_t dev, ggm}}@@ -402,7 +390,7 @@ diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vuVK_LOG_DEBUG("ggml_backend_vk_device_buffer_from_host_ptr(backend=" << dev << ", ptr=" << ptr << ", size=" << size << ")");GGML_UNUSED(max_tensor_size);-@@ -15652,17 +15664,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {+@@ -16294,17 +16303,17 @@ static const struct ggml_backend_device_i ggml_backend_vk_device_i = {/* .event_synchronize = */ ggml_backend_vk_device_event_synchronize,};diff --git a/llama.cpp.patches/patches/ggml_src_ggml.c.patch b/llama.cpp.patches/patches/ggml_src_ggml.c.patchindex d74076b8..43dff796 100644--- a/llama.cpp.patches/patches/ggml_src_ggml.c.patch+++ b/llama.cpp.patches/patches/ggml_src_ggml.c.patch@@ -1,17 +1,17 @@diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c--- a/llama.cpp/ggml/src/ggml.c+++ b/llama.cpp/ggml/src/ggml.c-@@ -548,11 +548,13 @@ void ggml_time_init(void) {+@@ -556,11 +556,13 @@ void ggml_time_init(void) {InitOnceExecuteOnce(&once, ggml_time_init_once, NULL, NULL);}int64_t ggml_time_ms(void) {+ ggml_time_init(); // llamafile: GPU DLLs carry their own ggml.c; init lazily so timer_freq is never 0LARGE_INTEGER t;QueryPerformanceCounter(&t);- return ((t.QuadPart-timer_start) * 1000) / timer_freq;+ return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000);}int64_t ggml_time_us(void) {+ ggml_time_init(); // llamafile: see ggml_time_msLARGE_INTEGER t;QueryPerformanceCounter(&t);- return ((t.QuadPart-timer_start) * 1000000) / timer_freq;+ return (int64_t) (((double) (t.QuadPart - timer_start) / timer_freq) * 1000000);diff --git a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patchindex 0036961e..c7e66409 100644--- a/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch+++ b/llama.cpp.patches/patches/ggml_src_gguf.cpp.patch@@ -12,7 +12,7 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp#include <cinttypes>#include <cstddef>#include <cstdint>-@@ -997,7 +1001,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, stru+@@ -1014,7 +1018,54 @@ struct gguf_context * gguf_init_from_buffer(const void * data, size_t size, strureturn gguf_init_from_reader(gr, params);}@@ -67,7 +67,7 @@ diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cppFILE * file = ggml_fopen(fname, "rb");if (!file) {-@@ -1008,6 +1059,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_p+@@ -1025,6 +1076,7 @@ struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_pstruct gguf_context * result = gguf_init_from_file_ptr(file, params);fclose(file);return result;diff --git a/llama.cpp.patches/patches/src_llama-context.cpp.patch b/llama.cpp.patches/patches/src_llama-context.cpp.patchindex f99ba6a1..b9e48a5c 100644--- a/llama.cpp.patches/patches/src_llama-context.cpp.patch+++ b/llama.cpp.patches/patches/src_llama-context.cpp.patch@@ -1,7 +1,7 @@diff --git a/src/llama-context.cpp b/src/llama-context.cpp--- a/llama.cpp/src/llama-context.cpp+++ b/llama.cpp/src/llama-context.cpp-@@ -553,6 +553,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3+@@ -657,6 +657,16 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3}};diff --git a/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patch b/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patchnew file mode 100644index 00000000..109b0c6d--- /dev/null+++ b/llama.cpp.patches/patches/src_llama-memory-hybrid-idx.cpp.patch@@ -0,0 +1,53 @@+diff --git a/src/llama-memory-hybrid-idx.cpp b/src/llama-memory-hybrid-idx.cpp+--- a/llama.cpp/src/llama-memory-hybrid-idx.cpp++++ b/llama.cpp/src/llama-memory-hybrid-idx.cpp+@@ -16,6 +16,25 @@+ // llama_memory_hybrid_idx+ //+++// llamafile: defined before the constructor, which instantiates ~unique_ptr<kpool_layout>;++// with -std=gnu++23 that destructor is constexpr and clang instantiates it on the spot.++// Which cells of a sequence make up which pool, for the whole cache.++struct llama_memory_hybrid_idx::kpool_layout {++ struct seq {++ llama_pos pos_min = 0;++ uint32_t strm = 0; // Stream holding this sequence's cells++ std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position.++ std::vector<uint32_t> pools;++++ // Where the pool scan stopped, so an append resumes instead of starting over.++ size_t j_next = 0;++ };++++ std::array<seq, LLAMA_MAX_SEQ> seqs;++++ uint32_t n_pool_real = 0;++};+++ llama_memory_hybrid_idx::llama_memory_hybrid_idx(+ const llama_model & model,+ bool no_alloc,+@@ -329,23 +348,6 @@ static std::vector<uint32_t> llama_memory_hybrid_idx_ns(const llama_kv_cache::sl+ return res;+ }++-// Which cells of a sequence make up which pool, for the whole cache.+-struct llama_memory_hybrid_idx::kpool_layout {+- struct seq {+- llama_pos pos_min = 0;+- uint32_t strm = 0; // Stream holding this sequence's cells+- std::vector<std::pair<llama_pos, uint32_t>> cells; // Position and stream local cell pairs, sorted by position.+- std::vector<uint32_t> pools;+-+- // Where the pool scan stopped, so an append resumes instead of starting over.+- size_t j_next = 0;+- };+-+- std::array<seq, LLAMA_MAX_SEQ> seqs;+-+- uint32_t n_pool_real = 0;+-};+-+ // Which pools of the layout the current ubatch must re-pool, in the layout's pool order.+ struct llama_memory_hybrid_idx_context::kpool_state {+ std::vector<uint32_t> is_new;diff --git a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch b/llama.cpp.patches/patches/src_llama-mmap.cpp.patchindex f0208df5..4e7080dc 100644--- a/llama.cpp.patches/patches/src_llama-mmap.cpp.patch+++ b/llama.cpp.patches/patches/src_llama-mmap.cpp.patch@@ -1,18 +1,19 @@diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp--- a/llama.cpp/src/llama-mmap.cpp+++ b/llama.cpp/src/llama-mmap.cpp-@@ -4,6 +4,10 @@+@@ -4,6 +4,11 @@#include "ggml.h"+#ifdef COSMOCC+#include "llamafile/llamafile.h"++#include <cosmo.h>+#endif+#include <cstring>#include <climits>#include <cstdlib>-@@ -182,6 +186,15 @@ struct llama_file::impl {+@@ -182,6 +187,15 @@ struct llama_file::impl {}#elseimpl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {@@ -28,7 +29,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp#ifdef __linux__// Try unbuffered I/O for read onlyif (use_direct_io && std::strcmp(mode, "rb") == 0) {-@@ -234,6 +247,9 @@ struct llama_file::impl {+@@ -234,6 +248,9 @@ struct llama_file::impl {}size_t tell() const {@@ -38,7 +39,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppif (fd == -1) {off_t ret = llama_mmap_ftell(fp);if (ret == -1) {-@@ -251,6 +267,11 @@ struct llama_file::impl {+@@ -251,6 +268,11 @@ struct llama_file::impl {}void seek(size_t offset, int whence) const {@@ -50,7 +51,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppoff_t ret = 0;if (fd == -1) {ret = llama_mmap_fseek(fp, offset, whence);-@@ -260,12 +281,22 @@ struct llama_file::impl {+@@ -260,12 +282,22 @@ struct llama_file::impl {if (ret == -1) {throw std::runtime_error(format("seek error: %s", strerror(errno)));}@@ -73,7 +74,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpperrno = 0;if (fd == -1) {const size_t curr_off = tell();-@@ -315,6 +346,7 @@ struct llama_file::impl {+@@ -315,6 +347,7 @@ struct llama_file::impl {bytes_read += (size_t) ret;}}@@ -81,7 +82,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp}void read_aligned_chunk(void * dest, size_t size) {-@@ -375,13 +407,20 @@ struct llama_file::impl {+@@ -388,13 +421,20 @@ struct llama_file::impl {}~impl() {@@ -102,7 +103,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppstd::string fname;#endif-@@ -410,6 +449,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }+@@ -423,6 +463,10 @@ size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }int llama_file::file_id() const {@@ -113,7 +114,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp#ifdef _WIN32return _fileno(pimpl->fp);#else-@@ -417,9 +460,10 @@ int llama_file::file_id() const {+@@ -430,9 +474,10 @@ int llama_file::file_id() const {return pimpl->fd;}#if defined(fileno)@@ -126,7 +127,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp#endif#endif}-@@ -437,6 +481,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }+@@ -450,6 +495,31 @@ uint32_t llama_file::read_u32() { return pimpl->read_u32(); }void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }@@ -158,7 +159,7 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp// llama_mmap#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)-@@ -465,9 +534,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t li+@@ -478,9 +548,26 @@ static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t listruct llama_mmap::impl {#ifdef _POSIX_MAPPED_FILESstd::vector<std::pair<size_t, size_t>> mapped_fragments;@@ -185,18 +186,26 @@ diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppint fd = file->file_id();int flags = MAP_SHARED;if (numa) { prefetch = 0; }-@@ -528,6 +614,10 @@ struct llama_mmap::impl {+@@ -541,6 +628,18 @@ struct llama_mmap::impl {}void unmap_fragment(size_t first, size_t last) {+ if (!is_owned) {+ return;+ }++#ifdef COSMOCC++ // Windows can only unmap a whole view (cosmo's munmap fails with ENOTSUP or++ // EINVAL on a sub-range), so do what the _WIN32 branch below does: keep the++ // fragments, and let the destructor unmap the whole file in one call.++ if (IsWindows()) {++ return;++ }++#endif+int page_size = sysconf(_SC_PAGESIZE);align_range(&first, &last, page_size);size_t len = last - first;-@@ -564,6 +654,14 @@ struct llama_mmap::impl {+@@ -577,6 +676,14 @@ struct llama_mmap::impl {}~impl() {diff --git a/llama.cpp.patches/patches/src_llama-mmap.h.patch b/llama.cpp.patches/patches/src_llama-mmap.h.patchindex 6b6ba5eb..ef31f21e 100644--- a/llama.cpp.patches/patches/src_llama-mmap.h.patch+++ b/llama.cpp.patches/patches/src_llama-mmap.h.patch@@ -1,7 +1,7 @@diff --git a/src/llama-mmap.h b/src/llama-mmap.h--- a/llama.cpp/src/llama-mmap.h+++ b/llama.cpp/src/llama-mmap.h-@@ -36,6 +36,11 @@ struct llama_file {+@@ -39,6 +39,11 @@ struct llama_file {size_t read_alignment() const;bool has_direct_io() const;diff --git a/llama.cpp.patches/patches/src_models_t5.cpp.patch b/llama.cpp.patches/patches/src_models_t5.cpp.patchindex 6bb5ae7d..37389c22 100644--- a/llama.cpp.patches/patches/src_models_t5.cpp.patch+++ b/llama.cpp.patches/patches/src_models_t5.cpp.patch@@ -1,7 +1,7 @@diff --git a/src/models/t5.cpp b/src/models/t5.cpp--- a/llama.cpp/src/models/t5.cpp+++ b/llama.cpp/src/models/t5.cpp-@@ -357,6 +357,12 @@ llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_pa+@@ -355,6 +355,12 @@ llama_model_t5::graph<true>::graph(const llama_model & model, const llm_graph_paggml_build_forward_expand(gf, cur);}diff --git a/llama.cpp.patches/patches/tools_server_server-http.cpp.patch b/llama.cpp.patches/patches/tools_server_server-http.cpp.patchindex c3319387..21ea880a 100644--- a/llama.cpp.patches/patches/tools_server_server-http.cpp.patch+++ b/llama.cpp.patches/patches/tools_server_server-http.cpp.patch@@ -1,16 +1,16 @@diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp--- a/llama.cpp/tools/server/server-http.cpp+++ b/llama.cpp/tools/server/server-http.cpp-@@ -104,7 +104,7 @@ bool server_http_context::init(const common_params & params) {-- auto & srv = pimpl->srv;+@@ -159,7 +159,7 @@ bool server_http_context::init(const common_params & params) {+ bool server_http_context::init_listener(const common_params & params) {+ auto & srv = pimpl->servers.back();-#ifdef CPPHTTPLIB_OPENSSL_SUPPORT+#ifdef CPPHTTPLIB_SSL_ENABLEDif (!params.ssl_file_key.empty() && !params.ssl_file_cert.empty()) {SRV_TRC("running with SSL: key = %s, cert = %s\n", params.ssl_file_key.c_str(), params.ssl_file_cert.c_str());srv = std::make_unique<httplib::SSLServer>(-@@ -429,12 +429,15 @@ bool server_http_context::init(const common_params & params) {+@@ -491,7 +491,11 @@ self.addEventListener('activate', (e) => e.waitUntil((async () => {return true;}@@ -22,14 +22,9 @@ diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp+// Bind and listen- const auto & srv = pimpl->srv;- auto was_bound = false;-- auto is_sock = false;- if (string_ends_with(std::string(hostname), ".sock")) {- is_sock = true;- SRV_TRC("%s", "setting address family to AF_UNIX\n");-@@ -461,6 +464,17 @@ bool server_http_context::start() {- return false;+ listening_addresses.clear();+@@ -524,6 +528,15 @@ bool server_http_context::start() {+ : string_format("%s://%s:%d", is_ssl ? "https" : "http", common_http_format_host(host).c_str(), port));}+ is_bound = true;@@ -41,8 +36,6 @@ diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp+ return false;+ }+-+ const auto & srv = pimpl->srv;-+- // run the HTTP server in a thread- thread = std::thread([this] { pimpl->srv->listen_after_bind(); });- srv->wait_until_ready();+ // n_threads_http fixed threads (always alive), plus up to 1024 dynamic threads destroyed after each request+ // ref: https://github.com/yhirose/cpp-httplib/pull/2368+ pimpl->pool = std::make_unique<httplib::ThreadPool>(pimpl->n_threads_http, pimpl->n_threads_http + 1024);diff --git a/llama.cpp.patches/patches/tools_server_server-http.h.patch b/llama.cpp.patches/patches/tools_server_server-http.h.patchindex 39c7a976..07e3dce5 100644--- a/llama.cpp.patches/patches/tools_server_server-http.h.patch+++ b/llama.cpp.patches/patches/tools_server_server-http.h.patch@@ -1,21 +1,20 @@diff --git a/tools/server/server-http.h b/tools/server/server-http.h--- a/llama.cpp/tools/server/server-http.h+++ b/llama.cpp/tools/server/server-http.h-@@ -79,11 +79,17 @@ struct server_http_context {- std::string hostname;- int port = 8080;+@@ -91,11 +91,16 @@ struct server_http_context {+ std::string path_prefix;+ int port = 9931;bool is_ssl = false;-+ bool is_sock = false; // listening on a Unix domain socket+ bool is_bound = false;server_http_context();~server_http_context();bool init(const common_params & params);-+ // creates and binds the listening socket; start() calls it if needed.-+ // llamafile binds early so the socket exists before the sandbox is++ // creates and binds the listening sockets; start() calls it if needed.++ // llamafile binds early so the sockets exist before the sandbox is+ // installed (a Unix domain socket can't be created under pledge "anet")+ bool bind_socket();bool start();void stop() const;-+ void join();diff --git a/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patchnew file mode 100644index 00000000..6ad203e9--- /dev/null+++ b/llama.cpp.patches/patches/tools_server_server-models-sched.cpp.patch@@ -0,0 +1,33 @@+diff --git a/tools/server/server-models-sched.cpp b/tools/server/server-models-sched.cpp+--- a/llama.cpp/tools/server/server-models-sched.cpp++++ b/llama.cpp/tools/server/server-models-sched.cpp+@@ -277,7 +277,12 @@ void server_models_sched::make_room(const std::string & name) {+ };+ if (!measured()) {+ SRV_INF("waiting for the memory measurement of model name=%s\n", name.c_str());+- models.cv.wait(lk, measured);++ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug++ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,++ // causing condition_variable::wait() to throw ETIMEDOUT.++ while (!measured()) {++ models.cv.wait_for(lk, std::chrono::seconds(30));++ }+ }+ }++@@ -298,9 +303,12 @@ void server_models_sched::make_room(const std::string & name) {+ // wait for unload to complete+ {+ std::unique_lock<std::mutex> lk(models.mutex);+- models.cv.wait(lk, [this, &lru_model_name]() {+- return models.mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED;+- });++ // Use wait_for() instead of wait() to work around a Cosmopolitan libc++ // bug where untimed futex waits on XNU (macOS) expire after ~72 minutes,++ // causing condition_variable::wait() to throw ETIMEDOUT.++ while (models.mapping[lru_model_name].meta.status != SERVER_MODEL_STATUS_UNLOADED) {++ models.cv.wait_for(lk, std::chrono::seconds(30));++ }+ }+ }+ }diff --git a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch b/llama.cpp.patches/patches/tools_server_server-models.cpp.patchindex ab633263..3ee67458 100644--- a/llama.cpp.patches/patches/tools_server_server-models.cpp.patch+++ b/llama.cpp.patches/patches/tools_server_server-models.cpp.patch@@ -1,7 +1,7 @@diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp--- a/llama.cpp/tools/server/server-models.cpp+++ b/llama.cpp/tools/server/server-models.cpp-@@ -31,6 +31,7 @@+@@ -34,6 +34,7 @@#include <cstring>#ifndef _WIN32@@ -9,7 +9,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cppextern char **environ;#endif-@@ -61,7 +62,25 @@ static void request_child_exit(server_subproc & proc) {+@@ -51,7 +52,25 @@ static void request_child_exit(server_subproc & proc) {// single-threaded, watching all child processes at oncestruct server_monitor {server_monitor(server_models & models) : models(models) {@@ -36,7 +36,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp}~server_monitor() {-@@ -893,13 +912,19 @@ void server_models::load_models() {+@@ -947,13 +966,19 @@ void server_models::load_models() {}// wait for all targeted models to reach UNLOADED; cv.wait handles unlock/relock@@ -58,23 +58,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp// erase models no longer in any sourcefor (auto it = mapping.begin(); it != mapping.end(); ) {-@@ -1096,9 +1121,12 @@ void server_models::unload_lru() {- // wait for unload to complete- {- std::unique_lock<std::mutex> lk(mutex);-- cv.wait(lk, [this, &lru_model_name]() {-- return mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED;-- });-+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc-+ // bug where untimed futex waits on XNU (macOS) expire after ~72 minutes,-+ // causing condition_variable::wait() to throw ETIMEDOUT.-+ while (mapping[lru_model_name].meta.status != SERVER_MODEL_STATUS_UNLOADED) {-+ cv.wait_for(lk, std::chrono::seconds(30));-+ }- }- }- }-@@ -1123,7 +1151,12 @@ void server_models::load(const std::string & name, const load_options & opts) {+@@ -1158,7 +1183,12 @@ void server_models::load(const std::string & name, const load_options & opts) {std::unique_lock<std::mutex> lk(mutex);// edge case: block until any in-progress reload has finished so we always load// against the freshest preset and a consistent mapping state@@ -86,20 +70,21 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp+ cv.wait_for(lk, std::chrono::seconds(30));+ }- auto meta = opts.custom_meta.has_value() ? *opts.custom_meta : mapping[name].meta;- if (meta.status != SERVER_MODEL_STATUS_UNLOADED) {-@@ -1283,14 +1316,20 @@ void server_models::unload_all() {+ // a download over an existing entry is decided under the lock, so a load of the+ // same model cannot start in between: a paused download or an idle cached model+@@ -1381,7 +1411,10 @@ void server_models::unload_all() {}}// wait for every child to exit, the monitor force-kills the ones that ignore the exit command-- cv.wait(lk, [this]() {+- cv.wait(lk, [this, &lk]() {+ // Use wait_for() instead of wait() to work around a Cosmopolitan libc bug+ // where untimed futex waits on XNU (macOS) expire after ~72 minutes,+ // causing condition_variable::wait() to throw ETIMEDOUT.-+ auto all_stopped = [this]() {- for (const auto & [name, inst] : mapping) {- if (inst.meta.is_running() || inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) {- return false;++ auto all_stopped = [this, &lk]() {+ if (sched->is_measuring(lk)) {+ return false;+ }+@@ -1391,7 +1424,10 @@ void server_models::unload_all() {}}return true;@@ -111,7 +96,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp}void server_models::update_status(const std::string & name, const update_status_args & args) {-@@ -1422,16 +1461,22 @@ void server_models::wait(const std::string & name, std::function<bool(const serv+@@ -1552,16 +1588,22 @@ void server_models::wait(const std::string & name, std::function<bool(const serv}void server_models::wait(std::unique_lock<std::mutex> & lk, const std::string & name, std::function<bool(const server_model_meta &)> predicate) {@@ -141,7 +126,7 @@ diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp}bool server_models::ensure_model_ready(const std::string & name, const std::function<bool()> & should_stop) {-@@ -2448,10 +2493,10 @@ server_http_proxy::server_http_proxy(+@@ -2631,10 +2673,10 @@ server_http_proxy::server_http_proxy(auto pipe = std::make_shared<server_pipe<msg_t>>();if (scheme == "https") {diff --git a/llama.cpp.patches/patches/tools_server_server.cpp.patch b/llama.cpp.patches/patches/tools_server_server.cpp.patchindex 21e6da5d..c7fe27e9 100644--- a/llama.cpp.patches/patches/tools_server_server.cpp.patch+++ b/llama.cpp.patches/patches/tools_server_server.cpp.patch@@ -1,7 +1,7 @@diff --git a/tools/server/server.cpp b/tools/server/server.cpp--- a/llama.cpp/tools/server/server.cpp+++ b/llama.cpp/tools/server/server.cpp-@@ -15,6 +15,7 @@+@@ -16,6 +16,7 @@#include <atomic>#include <clocale>#include <exception>@@ -9,7 +9,7 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp#include <signal.h>#include <thread> // for std::thread::hardware_concurrency-@@ -22,6 +23,13 @@+@@ -23,6 +24,13 @@#include <windows.h>#endif@@ -23,15 +23,23 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cppstatic std::function<void(int)> shutdown_handler;static std::atomic_flag is_terminating = ATOMIC_FLAG_INIT;-@@ -40,7 +48,19 @@ static inline void signal_handler(int signal) {- int llama_server(int argc, char ** argv);+@@ -41,6 +49,12 @@ static inline void signal_handler(int signal) {+ struct server_run_opts {+ bool install_signal_handlers = true; // off when embedded in llama-cli, which handles them itself+ bool resolve_model = true; // download / resolve -hf and docker models; off when the caller did it already++++ // llamafile combined mode (TUI + server in one process), see server_main()++ std::function<void(const std::string &)> on_ready; // the server is loaded and accepting requests++ std::function<void(std::function<void()>)> on_shutdown_available; // receives a function that stops the server++ char ** argv = nullptr; // the sandbox detects --rpc on argv, it has no params field++ bool combined_mode() const { return on_ready || on_shutdown_available; }+ };++ // satisfies -Wmissing-declarations (used by llama command)+@@ -48,6 +62,31 @@ int llama_server(int argc, char ** argv);// to be used via CLI (argc / argv are used by router mode only)--int llama_server(common_params & params, int argc, char ** argv);-+// on_ready / on_shutdown_available are llamafile combined-mode (TUI) callbacks.-+int llama_server(common_params & params, int argc, char ** argv,-+ std::function<void(const std::string &)> on_ready = nullptr,-+ std::function<void(std::function<void()>)> on_shutdown_available = nullptr);+ int llama_server(common_params & params, int argc, char ** argv);++// Core server logic - can be called from llamafile main.cpp or standalone main()+// Optional callbacks for combined mode:@@ -40,11 +48,181 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp+int server_main(int argc, char ** argv,+ std::function<void(const std::string &)> on_ready,+ std::function<void(std::function<void()>)> on_shutdown_available);++++#ifdef LLAMAFILE_TUI++// By now the program can safely exit. In combined mode return normally instead,++// so the caller can handle cleanup and join the TUI thread.++static void server_tui_exit(const server_run_opts & opts) {++ if (opts.combined_mode()) {++ return;++ }++ common_log_flush(common_log_main());++ // Metal backend has async logging that llama_synchronize() doesn't wait for.++ // Without this delay, _exit() might truncate llama_memory_breakdown_print's output++ sleep(1);++ // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)++ _exit(0);++}++#endif+void llama_server_terminate();void llama_server_terminate() {if (shutdown_handler) {-@@ -86,6 +106,14 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t+@@ -602,6 +641,9 @@ static int server_run_router(common_params & params, int argc, char ** argv, con+ // when the HTTP server stops, clean up and exit+ clean_up();+++#ifdef LLAMAFILE_TUI++ server_tui_exit(opts);++#endif+ return 0;+ }++@@ -655,6 +697,107 @@ static int server_run_instance(common_params & params, server_role & role, commo+ llama_backend_free();+ };+++#ifdef COSMOCC++ // Sandbox (pledge()/unveil(), llamafile issue #930). The whole++ // derivation lives in llamafile/sandbox.c so it stays unit-testable++ // and off the vendored diff. Placement is constrained on every side:++ // - after the -hf / docker model resolve above: downloads need connect()++ // - after ctx_http.init(): the TLS certificate/key are read there++ // - before ctx_http.start(): filters only cover threads created++ // after installation, so the HTTP listener/pool must not exist++ // - before load_model(): untrusted GGUF data is parsed sandboxed++ //++ // Combined mode hosts an in-process HTTP client (the TUI) that needs++ // connect(), which the accept-only sandbox forbids, so it's skipped.++ //++ // --connect spawns llama-connect once the server listens, i.e. after the++ // lock; the sandbox forbids exec, so it is skipped there too.++ if (opts.combined_mode()) {++ SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");++ } else if (params.server_connect) {++ SRV_WRN("%s", "sandbox: disabled with --connect (it spawns llama-connect, which the sandbox forbids)\n");++ } else {++ // pledge() (no outbound network / writes / exec) is applied by++ // default. unveil() path confinement is opt-in (--confine-reads),++ // because a server opens files by path at request time (multimodal++ // media, etc.) that can't be known when the ruleset locks.++ //++ // cosmo installs the SECCOMP/Landlock filters on the calling thread++ // only (no TSYNC), so quiesce every pre-existing background thread++ // across the call: the llama.cpp log worker (pause() joins it,++ // resume() respawns it under the filter) and the stream-session GC++ // started at the top of server_main() (stop joins it, start++ // respawns it; no sessions exist yet, so stopping is a no-op++ // beyond the join).++ //++ // When --confine-reads is on, unveil() confines reads to these++ // directories, so collect every on-disk path the loader opens++ // after the lock. Add to this list when llama.cpp gains a new file++ // parameter (else it'd be denied under confinement). Paths read++ // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf++ // downloads in the model resolve above) need no entry.++ std::vector<const char *> read_paths, rw_paths;++ read_paths.push_back(params.model.path.c_str());++ read_paths.push_back(params.mmproj.path.c_str());++ read_paths.push_back(params.media_path.c_str()); // request-time media++ read_paths.push_back(params.speculative.draft.mparams.path.c_str());++ read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());++ read_paths.push_back(params.public_path.c_str());++ for (const auto & la : params.lora_adapters)++ read_paths.push_back(la.path.c_str());++ for (const auto & cv : params.control_vectors)++ read_paths.push_back(cv.fname.c_str());++ rw_paths.push_back(params.slot_save_path.c_str());++ rw_paths.push_back(params.path_prompt_cache.c_str());++ rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());++++ // Features that dial out at runtime need connect(), which the++ // accept-only default forbids; relax to full sockets for them++ // (still no writes/exec). --rpc has no params field -- its handler++ // registers backends directly -- so detect it on argv/env.++ const bool needs_outbound =++ params.ui_mcp_proxy ||++ !params.server_tools.empty() ||++ (opts.argv && llamafile_has(opts.argv, "--rpc")) ||++ (std::getenv("LLAMA_ARG_RPC") != nullptr);++++ llamafile_sandbox_spec spec = {++ read_paths.data(), (int) read_paths.size(),++ rw_paths.data(), (int) rw_paths.size(),++ FLAG_confine_reads, needs_outbound,++ };++ // Bind the listening sockets now: pledge "anet" only covers++ // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be++ // created once the sandbox is on, while accept() on one that is++ // already bound stays allowed (llamafile issue #971). Binding++ // spawns no threads; ctx_http.start() reuses the sockets.++ if (!ctx_http.bind_socket()) {++ clean_up();++ SRV_ERR("%s", "exiting due to HTTP server error\n");++ return 1;++ }++++ char promises[64];++ common_log_pause(common_log_main());++ server_stream_session_manager_stop();++ int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));++ int sandbox_errno = errno; // capture before the respawns spawn threads++ server_stream_session_manager_start();++ common_log_resume(common_log_main());++ if (sandbox == LLAMAFILE_SANDBOX_FAILED) {++ SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",++ strerror(sandbox_errno));++ clean_up();++ return 1;++ } else if (llamafile_sandbox_is_active(sandbox)) {++ SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,++ llamafile_sandbox_describe(sandbox));++ } else {++ SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));++ }++ }++#endif+++ // start the HTTP server before loading the model to be able to serve /health requests+ if (!ctx_http.start()) {+ clean_up();+@@ -687,6 +830,13 @@ static int server_run_instance(common_params & params, server_role & role, commo+ ctx_server.terminate();+ };+++ // Provide shutdown function to caller (for combined mode)++ if (opts.on_shutdown_available) {++ opts.on_shutdown_available([&ctx_server]() {++ ctx_server.terminate();++ });++ }+++ if (opts.install_signal_handlers) {+ server_install_signal_handlers();+ }+@@ -699,6 +849,11 @@ static int server_run_instance(common_params & params, server_role & role, commo+ return 1;+ }+++ // Notify caller that server is ready (for combined mode TUI startup)++ if (opts.on_ready) {++ opts.on_ready(ctx_http.listening_addresses.front());++ }+++ // optionally, notify router server that this instance is ready+ std::thread monitor_thread;+ if (role.is_managed_by_router()) {+@@ -725,6 +880,9 @@ static int server_run_instance(common_params & params, server_role & role, commo+ common_memory_breakdown_print(ll_ctx);+ }+++#ifdef LLAMAFILE_TUI++ server_tui_exit(opts);++#endif+ return 0;+ }++@@ -770,6 +928,14 @@ static int server_run(common_params & params, int argc, char ** argv, server_rol}int llama_server(int argc, char ** argv) {@@ -56,10 +234,10 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cpp+int server_main(int argc, char ** argv,+ std::function<void(const std::string &)> on_ready,+ std::function<void(std::function<void()>)> on_shutdown_available) {- std::setlocale(LC_NUMERIC, "C");+ server_role role;- #ifndef _WIN32-@@ -96,6 +124,10 @@ int llama_server(int argc, char ** argv) {+ std::setlocale(LC_NUMERIC, "C");+@@ -782,6 +948,10 @@ int llama_server(int argc, char ** argv) {// own arguments required by this examplecommon_params params;@@ -70,170 +248,23 @@ diff --git a/tools/server/server.cpp b/tools/server/server.cppcommon_init();// start the stream session manager GC right after common init, before any HTTP route can-@@ -111,10 +143,12 @@ int llama_server(int argc, char ** argv) {+@@ -797,7 +967,12 @@ int llama_server(int argc, char ** argv) {llama_backend_init();llama_numa_init(params.numa);-- return llama_server(params, argc, argv);-+ return llama_server(params, argc, argv, on_ready, on_shutdown_available);- }---int llama_server(common_params & params, int argc, char ** argv) {-+int llama_server(common_params & params, int argc, char ** argv,-+ std::function<void(const std::string &)> on_ready,-+ std::function<void(std::function<void()>)> on_shutdown_available) {- bool is_run_by_cli = (argv == nullptr);-- common_models_handler models_handler;-@@ -457,6 +491,103 @@ int llama_server(common_params & params, int argc, char ** argv) {- llama_backend_free();- };--+#ifdef COSMOCC-+ // Sandbox (pledge()/unveil(), llamafile issue #930). The whole-+ // derivation lives in llamafile/sandbox.c so it stays unit-testable-+ // and off the vendored diff. Placement is constrained on every side:-+ // - after common_params_parse(): -hf downloads need connect()-+ // - after ctx_http.init(): the TLS certificate/key are read there-+ // - before ctx_http.start(): filters only cover threads created-+ // after installation, so the HTTP listener/pool must not exist-+ // - before load_model(): untrusted GGUF data is parsed sandboxed-+ //-+ // Combined mode hosts an in-process HTTP client (the TUI) that needs-+ // connect(), which the accept-only sandbox forbids, so it's skipped.-+ const bool combined_mode = on_ready || on_shutdown_available;-+ if (combined_mode) {-+ SRV_WRN("%s", "sandbox: disabled in combined mode; use --server for a sandboxed server\n");-+ } else {-+ // pledge() (no outbound network / writes / exec) is applied by-+ // default. unveil() path confinement is opt-in (--confine-reads),-+ // because a server opens files by path at request time (multimodal-+ // media, etc.) that can't be known when the ruleset locks.-+ //-+ // cosmo installs the SECCOMP/Landlock filters on the calling thread-+ // only (no TSYNC), so quiesce every pre-existing background thread-+ // across the call: the llama.cpp log worker (pause() joins it,-+ // resume() respawns it under the filter) and the stream-session GC-+ // started at the top of server_main() (stop joins it, start-+ // respawns it; no sessions exist yet, so stopping is a no-op-+ // beyond the join).-+ //-+ // When --confine-reads is on, unveil() confines reads to these-+ // directories, so collect every on-disk path the loader opens-+ // after the lock. Add to this list when llama.cpp gains a new file-+ // parameter (else it'd be denied under confinement). Paths read-+ // before the sandbox (ssl_file_key/cert in ctx_http.init(), -hf-+ // downloads in parse) need no entry.-+ std::vector<const char *> read_paths, rw_paths;-+ read_paths.push_back(params.model.path.c_str());-+ read_paths.push_back(params.mmproj.path.c_str());-+ read_paths.push_back(params.media_path.c_str()); // request-time media-+ read_paths.push_back(params.speculative.draft.mparams.path.c_str());-+ read_paths.push_back(params.speculative.ngram_cache.lookup_cache_static.c_str());-+ read_paths.push_back(params.public_path.c_str());-+ for (const auto & la : params.lora_adapters)-+ read_paths.push_back(la.path.c_str());-+ for (const auto & cv : params.control_vectors)-+ read_paths.push_back(cv.fname.c_str());-+ rw_paths.push_back(params.slot_save_path.c_str());-+ rw_paths.push_back(params.path_prompt_cache.c_str());-+ rw_paths.push_back(params.speculative.ngram_cache.lookup_cache_dynamic.c_str());-+-+ // Features that dial out at runtime need connect(), which the-+ // accept-only default forbids; relax to full sockets for them-+ // (still no writes/exec). --rpc has no params field -- its handler-+ // registers backends directly -- so detect it on argv/env.-+ const bool needs_outbound =-+ params.ui_mcp_proxy ||-+ !params.server_tools.empty() ||-+ llamafile_has(argv, "--rpc") ||-+ (std::getenv("LLAMA_ARG_RPC") != nullptr);-+-+ llamafile_sandbox_spec spec = {-+ read_paths.data(), (int) read_paths.size(),-+ rw_paths.data(), (int) rw_paths.size(),-+ FLAG_confine_reads, needs_outbound,-+ };-+ // Bind the listening socket now: pledge "anet" only covers-+ // IPv4/IPv6, so a Unix domain socket (--host /path.sock) can't be-+ // created once the sandbox is on, while accept() on one that is-+ // already bound stays allowed (llamafile issue #971). Binding-+ // spawns no threads; ctx_http.start() reuses the socket.-+ if (!ctx_http.bind_socket()) {-+ clean_up();-+ SRV_ERR("%s", "exiting due to HTTP server error\n");-+ return 1;-+ }-+-+ char promises[64];-+ common_log_pause(common_log_main());-+ server_stream_session_manager_stop();-+ int sandbox = llamafile_sandbox_server(&spec, promises, sizeof(promises));-+ int sandbox_errno = errno; // capture before the respawns spawn threads-+ server_stream_session_manager_start();-+ common_log_resume(common_log_main());-+ if (sandbox == LLAMAFILE_SANDBOX_FAILED) {-+ SRV_ERR("sandbox: pledge failed: %s; retry with --unsecure to disable sandboxing\n",-+ strerror(sandbox_errno));-+ clean_up();-+ return 1;-+ } else if (llamafile_sandbox_is_active(sandbox)) {-+ SRV_INF("sandbox: pledge(\"%s\") %s\n", promises,-+ llamafile_sandbox_describe(sandbox));-+ } else {-+ SRV_WRN("sandbox: %s\n", llamafile_sandbox_describe(sandbox));-+ }-+ }-+#endif-+- // start the HTTP server before loading the model to be able to serve /health requests- if (!ctx_http.start()) {- clean_up();-@@ -490,6 +621,13 @@ int llama_server(common_params & params, int argc, char ** argv) {- // this will unblock start_loop()- ctx_server.terminate();- };-+-+ // Provide shutdown function to caller (for combined mode)-+ if (on_shutdown_available) {-+ on_shutdown_available([&ctx_server]() {-+ ctx_server.terminate();-+ });-+ }- }-- // register signal handler if not running by CLI-@@ -530,6 +668,11 @@ int llama_server(common_params & params, int argc, char ** argv) {- // when the HTTP server stops, clean up and exit- clean_up();- } else {-+ // Notify caller that server is ready (for combined mode TUI startup)-+ if (on_ready) {-+ on_ready(ctx_http.listening_address);-+ }+- const int result = server_run(params, argc, argv, role, server_run_opts{});++ server_run_opts opts;++ opts.on_ready = std::move(on_ready);++ opts.on_shutdown_available = std::move(on_shutdown_available);++ opts.argv = argv;+- // optionally, notify router server that this instance is ready- std::thread monitor_thread;- if (child.is_child()) {-@@ -554,5 +697,56 @@ int llama_server(common_params & params, int argc, char ** argv) {- }- }--+#ifdef LLAMAFILE_TUI-+ // In combined mode (callbacks provided), return normally so the caller-+ // can handle cleanup and TUI thread joining-+ if (on_ready || on_shutdown_available) {-+ return 0;-+ }-+ // By now the program can safely exit:-+ // Metal backend has async logging that llama_synchronize() doesn't wait for.-+ // Without this delay, _exit() might truncate llama_memory_breakdown_print's output-+ sleep(1);-+ // Use _exit() to avoid Metal cleanup crash (dangling refs with TUI + Metal + server)-+ _exit(0);-+#else- return 0;-+#endif++ const int result = server_run(params, argc, argv, role, opts);+ common_log_flush(common_log_main());+ return result;+ }+@@ -811,3 +986,40 @@ int llama_server(common_params & params, int argc, char ** argv) {+ server_role role;+ return server_run(params, argc, argv, role, opts);}++// Standalone entry point for llama-server executablediff --git a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patchindex 16c52d3e..3868571f 100644--- a/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch+++ b/llama.cpp.patches/patches/vendor_cpp-httplib_httplib.cpp.patch@@ -1,7 +1,7 @@diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp--- a/llama.cpp/vendor/cpp-httplib/httplib.cpp+++ b/llama.cpp/vendor/cpp-httplib/httplib.cpp-@@ -2382,6 +2382,16 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,+@@ -2551,6 +2551,16 @@ socket_t create_socket(const std::string &host, const std::string &ip, int port,hints.ai_addr = reinterpret_cast<sockaddr *>(&addr);hints.ai_addrlen = static_cast<socklen_t>(sizeof(addr) - sizeof(addr.sun_path) + addrlen);@@ -18,7 +18,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp#ifndef SOCK_CLOEXEC#ifndef _WIN32-@@ -7244,6 +7254,22 @@ void ThreadPool::cleanup_finished_threads() {+@@ -7723,6 +7733,22 @@ void ThreadPool::cleanup_finished_threads() {}void ThreadPool::worker(bool is_dynamic) {@@ -41,7 +41,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cppfor (;;) {std::function<void()> fn;{-@@ -7261,7 +7287,13 @@ void ThreadPool::worker(bool is_dynamic) {+@@ -7740,7 +7766,13 @@ void ThreadPool::worker(bool is_dynamic) {break;}} else {@@ -56,7 +56,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp}idle_thread_count_--;-@@ -14003,6 +14035,13 @@ bool enumerate_macos_keychain_certs(Callback cb) {+@@ -14684,6 +14716,13 @@ bool enumerate_macos_keychain_certs(Callback cb) {#if !defined(_WIN32) && !(defined(__APPLE__) && \defined(CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN))@@ -70,7 +70,7 @@ diff --git a/vendor/cpp-httplib/httplib.cpp b/vendor/cpp-httplib/httplib.cpp// Common CA certificate file paths on Linux/Unixconst char **system_ca_paths() {static const char *paths[] = {-@@ -14020,6 +14059,10 @@ const char **system_ca_dirs() {+@@ -14701,6 +14740,10 @@ const char **system_ca_dirs() {static const char *dirs[] = {"/etc/ssl/certs", // Debian/Ubuntu"/etc/pki/tls/certs", // RHEL/CentOS"/usr/share/ca-certificates", // Otherdiff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 3a0a2ef1..be5091ff 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -245,11 +245,15 @@ LLAMAFILE_HIGHLIGHT_KEYWORDS := $(LLAMAFILE_HIGHLIGHT_GPERF_FILES:%.gperf=o/$(MOLLAMAFILE_SERVER_SUPPORT_OBJS := \o/$(MODE)/llama.cpp/tools/server/server-chat.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-common.cpp.o \+ o/$(MODE)/llama.cpp/tools/server/server-connect.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-context.cpp.o \+ o/$(MODE)/llama.cpp/tools/server/server-decision.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-http.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-mcp.cpp.o \+ o/$(MODE)/llama.cpp/tools/server/server-models-sched.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-models.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-queue.cpp.o \+ o/$(MODE)/llama.cpp/tools/server/server-realtime.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-schema.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-stream.cpp.o \o/$(MODE)/llama.cpp/tools/server/server-task.cpp.o \@@ -301,11 +305,29 @@ LLAMAFILE_METAL_SOURCES := \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/conv.metal.zip.o \- o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/misc.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal.zip.o \+ o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/norm.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/pool.metal.zip.o \o/$(MODE)/llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal.zip.o \diff --git a/llamafile/chatbot_cli.cpp b/llamafile/chatbot_cli.cppindex 44ec6bfc..fad2dab4 100644--- a/llamafile/chatbot_cli.cpp+++ b/llamafile/chatbot_cli.cpp@@ -82,17 +82,12 @@ static cli_chat_template_result cli_apply_chat_template_full(llama_model *model,inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;auto chat_params = common_chat_templates_apply(templates, inputs);- result.prompt = chat_params.prompt;+ result.prompt = chat_params.prompt.str();- // Initialize parser params from chat_params- result.parser_params.format = chat_params.format;- result.parser_params.generation_prompt = chat_params.generation_prompt;- if (!chat_params.parser.empty()) {- result.parser_params.parser.load(chat_params.parser);- }- // Enable reasoning parsing for thinking models- result.parser_params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;- result.parser_params.reasoning_in_content = false;+ // Initialize parser params from chat_params. Reasoning extraction is+ // built into the parser from inputs.reasoning_format above.+ result.parser_params = common_chat_parser_params(chat_params);+ result.parser_params.parser = std::move(chat_params.parser);return result;}@@ -408,6 +403,8 @@ int cli_main(int argc, char **argv) {common_chat_msg prev_msg; // Previous parse result for diff computationbool think_tag_opened = false; // Track if we've printed <think>bool think_tag_closed = false; // Track if we've printed </think>+ char last_reasoning_char = '\n'; // Last character of the printed reasoning+ int n_generated = 0; // New tokens so far, for -n / --n-predictwhile (n_cur < params.n_ctx) {if (g_got_sigint) {@@ -415,6 +412,12 @@ int cli_main(int argc, char **argv) {break;}+ // -n N stops after N new tokens; -1 (default) means no limit and -2+ // means "until the context is full", which the loop bound covers.+ if (params.n_predict >= 0 && n_generated >= params.n_predict) {+ break;+ }+llama_token id = common_sampler_sample(sampler, ctx, -1);common_sampler_accept(sampler, id, true);@@ -429,7 +432,7 @@ int cli_main(int argc, char **argv) {raw_output += token_str;// Parse incrementally- auto msg = common_chat_parse(raw_output, /*is_partial=*/true, parser_params);+ auto msg = common_chat_parse(common_chat_input(raw_output), /*is_partial=*/true, parser_params);// Compute diffs to find new contentauto diffs = common_chat_msg_diff::compute_diffs(prev_msg, msg);@@ -442,11 +445,17 @@ int cli_main(int argc, char **argv) {think_tag_opened = true;}fputs(diff.reasoning_content_delta.c_str(), stdout);+ last_reasoning_char = diff.reasoning_content_delta.back();fflush(stdout);}// Output final content (close think tag first if needed)if (!diff.content_delta.empty()) {if (think_tag_opened && !think_tag_closed) {+ // The parser trims the newline that ends the reasoning+ // (since llama.cpp v0.8.0), so put </think> on its own line+ if (last_reasoning_char != '\n') {+ fputs("\n", stdout);+ }fputs("</think>\n", stdout);think_tag_closed = true;}@@ -468,6 +477,7 @@ int cli_main(int argc, char **argv) {break;}n_cur++;+ n_generated++;}// Ensure output ends with newlinediff --git a/llamafile/chatbot_direct.cpp b/llamafile/chatbot_direct.cppindex e337898f..6d21f89d 100644--- a/llamafile/chatbot_direct.cpp+++ b/llamafile/chatbot_direct.cpp@@ -61,7 +61,7 @@ public:if (use_chat_parser) {std::string token_str = token_to_piece(g_ctx, id, true);raw_output += token_str;- auto msg = common_chat_parse(raw_output, true, g_chat_syntax);+ auto msg = common_chat_parse(common_chat_input(raw_output), true, g_chat_syntax);auto diffs = common_chat_msg_diff::compute_diffs(prev_msg, msg);for (const auto &diff : diffs) {diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cppindex 080b0cad..b6b6c28e 100644--- a/llamafile/chatbot_main.cpp+++ b/llamafile/chatbot_main.cpp@@ -276,17 +276,11 @@ int main(int argc, char **argv) {try {auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);- g_chat_syntax.format = chat_params.format;- g_chat_syntax.generation_prompt = chat_params.generation_prompt;- // Load the PEG parser if one was provided- if (!chat_params.parser.empty()) {- g_chat_syntax.parser.load(chat_params.parser);- }-- // Copy reasoning format to chat syntax for use by the parser at runtime- g_chat_syntax.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;- g_chat_syntax.reasoning_in_content = false;+ // The PEG parser (with reasoning extraction, from+ // inputs.reasoning_format above) comes built in chat_params.+ g_chat_syntax = common_chat_parser_params(chat_params);+ g_chat_syntax.parser = std::move(chat_params.parser);// Print detected formatif (!FLAG_nologo && g_chat_syntax.format != COMMON_CHAT_FORMAT_CONTENT_ONLY) {diff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cppindex 6c676517..bd0a32bc 100644--- a/llamafile/chatbot_repl.cpp+++ b/llamafile/chatbot_repl.cpp@@ -86,7 +86,7 @@ std::string apply_chat_template_with_thinking(inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;auto chat_params = common_chat_templates_apply(g_chat_templates.get(), inputs);- return chat_params.prompt;+ return chat_params.prompt.str();}void on_sigint(int sig) {diff --git a/llamafile/metal.c b/llamafile/metal.cindex e09800aa..550fc3cf 100644--- a/llamafile/metal.c+++ b/llamafile/metal.c@@ -107,11 +107,29 @@ __static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/quantize.h");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/conv.metal");-__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/misc.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal");+__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/norm.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/pool.metal");__static_yoink("llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal");@@ -181,11 +199,29 @@ static const struct MetalSource {{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/argsort.metal", "kernels/argsort.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/binbcast.metal", "kernels/binbcast.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/conv.metal", "kernels/conv.metal"},- {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa.metal", "kernels/fa.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_aux.metal", "kernels/fa_aux.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_common.metal", "kernels/fa_common.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_f16.metal", "kernels/fa_f16.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_f32.metal", "kernels/fa_f32.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_0.metal", "kernels/fa_q4_0.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q4_1.metal", "kernels/fa_q4_1.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_0.metal", "kernels/fa_q5_0.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q5_1.metal", "kernels/fa_q5_1.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_q8_0.metal", "kernels/fa_q8_0.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_tensor.metal", "kernels/fa_tensor.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_common.metal", "kernels/fa_vec_common.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f16.metal", "kernels/fa_vec_f16.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_f32.metal", "kernels/fa_vec_f32.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_0.metal", "kernels/fa_vec_q4_0.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q4_1.metal", "kernels/fa_vec_q4_1.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_0.metal", "kernels/fa_vec_q5_0.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q5_1.metal", "kernels/fa_vec_q5_1.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/fa_vec_q8_0.metal", "kernels/fa_vec_q8_0.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/gated_delta_net.metal", "kernels/gated_delta_net.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/misc.metal", "kernels/misc.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mm.metal", "kernels/mul_mm.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv.metal", "kernels/mul_mv.metal"},+ {"/zip/llama.cpp/ggml/src/ggml-metal/kernels/mul_mv_mma.metal", "kernels/mul_mv_mma.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/norm.metal", "kernels/norm.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/pool.metal", "kernels/pool.metal"},{"/zip/llama.cpp/ggml/src/ggml-metal/kernels/quantize.metal", "kernels/quantize.metal"},diff --git a/tests/integration/conftest.py b/tests/integration/conftest.pyindex 5e4e993e..d41a3a28 100644--- a/tests/integration/conftest.py+++ b/tests/integration/conftest.py@@ -39,7 +39,7 @@ def pytest_addoption(parser):"--gpu",action="store",default=None,- choices=["auto", "apple", "amd", "nvidia", "disable"],+ choices=["auto", "apple", "amd", "nvidia", "vulkan", "disable"],help="GPU mode to use (disable for CPU-only)",)parser.addoption(diff --git a/tests/integration/tests/test_cli.py b/tests/integration/tests/test_cli.pyindex d636e5fc..0026245f 100644--- a/tests/integration/tests/test_cli.py+++ b/tests/integration/tests/test_cli.py@@ -31,6 +31,25 @@ class TestCLIBasic:assert result.returncode == 0+ def test_cli_n_predict_limits_output(self, llamafile, timeouts):+ """Test that -n caps the number of generated tokens.++ Counting to 200 takes far more than 64 tokens, so without the cap+ both runs would print the same long answer.+ """+ prompt = "Count from 1 to 200, separated by commas."+ short = llamafile.run_cli(prompt, thinking=False, timeout=timeouts.cli,+ extra_args=["-n", "8", "--temp", "0"])+ longer = llamafile.run_cli(prompt, thinking=False, timeout=timeouts.cli,+ extra_args=["-n", "64", "--temp", "0"])++ assert short.returncode == 0, f"CLI failed: {short.stderr}"+ assert longer.returncode == 0, f"CLI failed: {longer.stderr}"+ assert 0 < len(short.stdout.strip()) < len(longer.stdout.strip()), (+ f"-n 8 output ({len(short.stdout)} chars) is not shorter than "+ f"-n 64 output ({len(longer.stdout)} chars)"+ )+@pytest.mark.cli@pytest.mark.thinkingdiff --git a/tests/integration/tests/test_gpu.py b/tests/integration/tests/test_gpu.pyindex 2749fa7e..a6c0885c 100644--- a/tests/integration/tests/test_gpu.py+++ b/tests/integration/tests/test_gpu.py@@ -219,7 +219,8 @@ class TestGPUAcceleration:complete``, i.e. graphs actually engaged for this model. ggml maylegitimately refuse them (pre-Volta GPU, unsupported node type); itthen logs ``disabling CUDA graphs due to ...`` and the runtime check- is skipped rather than failed.+ is skipped rather than failed. Since llama.cpp v0.8.0 (#30327) ggml+ prints both lines only with ``GGML_CUDA_GRAPH_DEBUG`` set.The module's own log lines go to stderr, not to --log-file (they arewritten by the ggml logger inside the DSO), so stderr is captured too.@@ -243,7 +244,8 @@ class TestGPUAcceleration:proc = runner.start_server(port=server_port, log_file=log_path,stderr_file=stderr_path,- extra_args=["--verbose"])+ extra_args=["--verbose"],+ env={"GGML_CUDA_GRAPH_DEBUG": "1"})try:ready = LlamafileRunner.wait_for_server(server_port, timeout=timeouts.server_ready, proc=procdiff --git a/tests/integration/utils/llamafile.py b/tests/integration/utils/llamafile.pyindex 3e71fd50..128c12b1 100644--- a/tests/integration/utils/llamafile.py+++ b/tests/integration/utils/llamafile.py@@ -238,7 +238,9 @@ class LlamafileRunner:Args:executable: Path to llamafile binary or pre-built .llamafilemodel: Path to model file (None for pre-built llamafiles)- gpu: GPU mode - "auto", "apple", "amd", "nvidia", or None for CPU+ gpu: GPU mode passed as --gpu - "auto", "apple", "amd", "nvidia",+ "vulkan", or "disable" for CPU only; None passes no flag,+ so llamafile picks a GPU itselfdefault_thinking: Default thinking mode when callers don't specifyone. Set to True for tests that exercise thinkingbehavior; False otherwise so small reasoning@@ -409,6 +411,7 @@ class LlamafileRunner:extra_args: list[str] | None = None,log_file: str | None = None,stderr_file: str | None = None,+ env: dict[str, str] | None = None,) -> subprocess.Popen:"""Start llamafile in server mode.@@ -424,6 +427,8 @@ class LlamafileRunner:file. Messages logged by a GPU backend module(ggml-cuda.so, ...) are emitted by the module's own ggmllogger and reach stderr, not --log-file.+ env: Extra environment variables for the server process, on top+ of the inherited environment.Returns:Popen process handle (caller must terminate)@@ -451,6 +456,7 @@ class LlamafileRunner:stdout=subprocess.DEVNULL,stderr=stderr,text=True,+ env={**os.environ, **env} if env else None,)finally:if stderr_file:diff --git a/transcribefile/main.cpp b/transcribefile/main.cppindex 0429472f..7942008a 100644--- a/transcribefile/main.cpp+++ b/transcribefile/main.cpp@@ -65,9 +65,8 @@ static void print_wrapper_help(void) {" --verbose show GPU loader and backend logging\n"" (suppressed by default)\n""transcribefile notes:\n"- " GPU backends wired up in this build: metal (macOS on Apple\n"- " Silicon; used by --backend auto/metal, first run compiles\n"- " ggml-metal.dylib and caches it under ~/.transcribefile).\n"+ " No GPU backend in this build: runs on CPU. Metal is disabled\n"+ " until transcribe.cpp's ggml matches llamafile's again;\n"" vulkan and cuda are not available yet.\n"" Default arguments are read from .args in the executable's zip\n"" store (one per line), so models can be bundled with zipalign.\n");@@ -86,7 +85,21 @@ static void print_wrapper_help(void) {// FLAG_gpu stays AUTO (not APPLE) even for --backend metal so a failed// load degrades silently here; transcribe.cpp reports the missing// backend itself, with wording that matches its own CLI.+// Metal is disabled for now. The dylib is built from llamafile's bundled+// llama.cpp ggml-metal sources and registered with transcribe.cpp's own+// ggml (0.15.2), which only works while both trees agree on the backend+// interface structs. llama.cpp v0.8.0 (ggml 0.28) added alloc_buffer_n+// and get_alloc_size_n to ggml_backend_buffer_type_i, so transcribe.cpp+// called a NULL slot in ggml_backend_buft_get_alignment and crashed on+// model load. Re-enable together with a transcribe.cpp update whose ggml+// matches llama.cpp's.+static constexpr bool kMetalEnabled = false;+static void load_gpu_backends(char ** argv) {+ if (!kMetalEnabled) {+ FLAG_gpu = LLAMAFILE_GPU_DISABLE;+ return;+ }const char * backend = "auto";for (char ** p = argv + 1; p && *p; ++p) {if (!strcmp(*p, "--backend") && p[1]) {